fix chat msg

README.md

@@ -1,6 +1,6 @@
 ---
 title: Baichuan2 13B Chat
-emoji: 📉
+emoji: 🔥
 colorFrom: gray
 colorTo: indigo
 sdk: gradio

app.py

@@ -3,269 +3,270 @@ from typing import Iterator
 import gradio as gr
 import torch
 
-from model import get_input_token_length, run
+from model import run
 
-# DEFAULT_SYSTEM_PROMPT = """\
-# You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.\
-# """
-
-DEFAULT_SYSTEM_PROMPT = """
-"""
+DEFAULT_SYSTEM_PROMPT = ""
 MAX_MAX_NEW_TOKENS = 2048
 DEFAULT_MAX_NEW_TOKENS = 1024
 MAX_INPUT_TOKEN_LENGTH = 4000
-
 DESCRIPTION = """
 # Baichuan2-13B-Chat
 Baichuan 2 is the new generation of open-source large language models launched by Baichuan Intelligent Technology. It was trained on a high-quality corpus with 2.6 trillion tokens.
 """
-
-LICENSE = """
-"""
+LICENSE = ""
 
 if not torch.cuda.is_available():
-    DESCRIPTION += '\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>'
+  DESCRIPTION += '\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>'
 
 
 def clear_and_save_textbox(message: str) -> tuple[str, str]:
-    return '', message
-
-
-def display_input(message: str,
-                  history: list[tuple[str, str]]) -> list[tuple[str, str]]:
-    history.append((message, ''))
-    return history
+  return '', message
 
+def display_input(
+  message: str,
+  history: list[tuple[str, str]]
+) -> list[tuple[str, str]]:
+  history.append((message, ''))
+  return history
 
 def delete_prev_fn(
-        history: list[tuple[str, str]]) -> tuple[list[tuple[str, str]], str]:
-    try:
-        message, _ = history.pop()
-    except IndexError:
-        message = ''
-    return history, message or ''
-
+  history: list[tuple[str, str]]
+) -> tuple[list[tuple[str, str]], str]:
+  try:
+    message, _ = history.pop()
+  except IndexError:
+    message = ''
+  return history, message or ''
 
 def generate(
-    message: str,
-    history_with_input: list[tuple[str, str]],
-    system_prompt: str,
-    max_new_tokens: int,
-    temperature: float,
-    top_p: float,
-    top_k: int,
+  message: str,
+  history_with_input: list[tuple[str, str]],
+  system_prompt: str,
+  max_new_tokens: int,
+  temperature: float,
+  top_p: float,
+  top_k: int,
 ) -> Iterator[list[tuple[str, str]]]:
-    if max_new_tokens > MAX_MAX_NEW_TOKENS:
-        raise ValueError
-
-    history = history_with_input[:-1]
-    generator = run(message, history, system_prompt, max_new_tokens, temperature, top_p, top_k)
-    try:
-        first_response = next(generator)
-        yield history + [(message, first_response)]
-    except StopIteration:
-        yield history + [(message, '')]
-    for response in generator:
-        yield history + [(message, response)]
-
+  if max_new_tokens > MAX_MAX_NEW_TOKENS:
+    raise ValueError
+
+  history = history_with_input[:-1]
+  generator = run(message, history, system_prompt, max_new_tokens, temperature, top_p, top_k)
+  try:
+    first_response = next(generator)
+    yield history + [(message, first_response)]
+  except StopIteration:
+    yield history + [(message, '')]
+  for response in generator:
+    yield history + [(message, response)]
 
 def process_example(message: str) -> tuple[str, list[tuple[str, str]]]:
-    generator = generate(message, [], DEFAULT_SYSTEM_PROMPT, 1024, 1, 0.95, 5)
-    for x in generator:
-        pass
-    return '', x
-
-
-def check_input_token_length(message: str, chat_history: list[tuple[str, str]], system_prompt: str) -> None:
-    input_token_length = get_input_token_length(message, chat_history, system_prompt)
-    if input_token_length > MAX_INPUT_TOKEN_LENGTH:
-        raise gr.Error(f'The accumulated input is too long ({input_token_length} > {MAX_INPUT_TOKEN_LENGTH}). Clear your chat history and try again.')
+  generator = generate(message, [], DEFAULT_SYSTEM_PROMPT, DEFAULT_MAX_NEW_TOKENS, 1, 0.95, 5)
+  for x in generator:
+    pass
+  return '', x
+
+def check_input_token_length(
+  message: str,
+  chat_history: list[tuple[str, str]],
+  system_prompt: str
+) -> None:
+  a = 1
+  # input_token_length = get_input_token_length(message, chat_history, system_prompt)
+  # if input_token_length > MAX_INPUT_TOKEN_LENGTH:
+  #     raise gr.Error(f'The accumulated input is too long ({input_token_length} > {MAX_INPUT_TOKEN_LENGTH}). Clear your chat history and try again.')
 
 
 with gr.Blocks(css='style.css') as demo:
-    gr.Markdown(DESCRIPTION)
-    gr.DuplicateButton(value='Duplicate Space for private use',
-                       elem_id='duplicate-button')
-
-    with gr.Group():
-        chatbot = gr.Chatbot(label='Chatbot')
-        with gr.Row():
-            textbox = gr.Textbox(
-                container=False,
-                show_label=False,
-                placeholder='Type a message...',
-                scale=10,
-            )
-            submit_button = gr.Button('Submit',
-                                      variant='primary',
-                                      scale=1,
-                                      min_width=0)
+  gr.Markdown(DESCRIPTION)
+  gr.DuplicateButton(
+    value='Duplicate Space for private use',
+    elem_id='duplicate-button'
+  )
+
+  with gr.Group():
+    chatbot = gr.Chatbot(label='Chatbot')
     with gr.Row():
-        retry_button = gr.Button('🔄  Retry', variant='secondary')
-        undo_button = gr.Button('↩️ Undo', variant='secondary')
-        clear_button = gr.Button('🗑️  Clear', variant='secondary')
-
-    saved_input = gr.State()
-
-    with gr.Accordion(label='Advanced options', open=False):
-        system_prompt = gr.Textbox(label='System prompt',
-                                   value=DEFAULT_SYSTEM_PROMPT,
-                                   lines=6)
-        max_new_tokens = gr.Slider(
-            label='Max new tokens',
-            minimum=1,
-            maximum=MAX_MAX_NEW_TOKENS,
-            step=1,
-            value=DEFAULT_MAX_NEW_TOKENS,
-        )
-        temperature = gr.Slider(
-            label='Temperature',
-            minimum=0.1,
-            maximum=4.0,
-            step=0.1,
-            value=1.0,
-        )
-        top_p = gr.Slider(
-            label='Top-p (nucleus sampling)',
-            minimum=0.05,
-            maximum=1.0,
-            step=0.05,
-            value=0.95,
-        )
-        top_k = gr.Slider(
-            label='Top-k',
-            minimum=1,
-            maximum=1000,
-            step=1,
-            value=50,
-        )
-
-    gr.Examples(
-        examples=[
-            'Hello there! How are you doing?',
-            'Can you explain briefly to me what is the Python programming language?',
-            'Explain the plot of Cinderella in a sentence.',
-            'How many hours does it take a man to eat a Helicopter?',
-            "Write a 100-word article on 'Benefits of Open-Source in AI research'",
-        ],
-        inputs=textbox,
-        outputs=[textbox, chatbot],
-        fn=process_example,
-        cache_examples=True,
+      textbox = gr.Textbox(
+        container=False,
+        show_label=False,
+        placeholder='Type a message...',
+        scale=10,
+      )
+      submit_button = gr.Button(
+        'Submit',
+        variant='primary',
+        scale=1,
+        min_width=0
+      )
+
+  with gr.Row():
+    retry_button = gr.Button('🔄  Retry', variant='secondary')
+    undo_button = gr.Button('↩️ Undo', variant='secondary')
+    clear_button = gr.Button('🗑️  Clear', variant='secondary')
+
+  saved_input = gr.State()
+
+  with gr.Accordion(label='Advanced options', open=False):
+    system_prompt = gr.Textbox(
+      label='System prompt',
+      value=DEFAULT_SYSTEM_PROMPT,
+      lines=6
     )
-
-    gr.Markdown(LICENSE)
-
-    textbox.submit(
-        fn=clear_and_save_textbox,
-        inputs=textbox,
-        outputs=[textbox, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=display_input,
-        inputs=[saved_input, chatbot],
-        outputs=chatbot,
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=check_input_token_length,
-        inputs=[saved_input, chatbot, system_prompt],
-        api_name=False,
-        queue=False,
-    ).success(
-        fn=generate,
-        inputs=[
-            saved_input,
-            chatbot,
-            system_prompt,
-            max_new_tokens,
-            temperature,
-            top_p,
-            top_k,
-        ],
-        outputs=chatbot,
-        api_name=False,
+    max_new_tokens = gr.Slider(
+      label='Max new tokens',
+      minimum=1,
+      maximum=MAX_MAX_NEW_TOKENS,
+      step=1,
+      value=DEFAULT_MAX_NEW_TOKENS,
     )
-
-    button_event_preprocess = submit_button.click(
-        fn=clear_and_save_textbox,
-        inputs=textbox,
-        outputs=[textbox, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=display_input,
-        inputs=[saved_input, chatbot],
-        outputs=chatbot,
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=check_input_token_length,
-        inputs=[saved_input, chatbot, system_prompt],
-        api_name=False,
-        queue=False,
-    ).success(
-        fn=generate,
-        inputs=[
-            saved_input,
-            chatbot,
-            system_prompt,
-            max_new_tokens,
-            temperature,
-            top_p,
-            top_k,
-        ],
-        outputs=chatbot,
-        api_name=False,
+    temperature = gr.Slider(
+      label='Temperature',
+      minimum=0.1,
+      maximum=4.0,
+      step=0.1,
+      value=1.0,
     )
-
-    retry_button.click(
-        fn=delete_prev_fn,
-        inputs=chatbot,
-        outputs=[chatbot, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=display_input,
-        inputs=[saved_input, chatbot],
-        outputs=chatbot,
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=generate,
-        inputs=[
-            saved_input,
-            chatbot,
-            system_prompt,
-            max_new_tokens,
-            temperature,
-            top_p,
-            top_k,
-        ],
-        outputs=chatbot,
-        api_name=False,
+    top_p = gr.Slider(
+      label='Top-p (nucleus sampling)',
+      minimum=0.05,
+      maximum=1.0,
+      step=0.05,
+      value=0.95,
     )
-
-    undo_button.click(
-        fn=delete_prev_fn,
-        inputs=chatbot,
-        outputs=[chatbot, saved_input],
-        api_name=False,
-        queue=False,
-    ).then(
-        fn=lambda x: x,
-        inputs=[saved_input],
-        outputs=textbox,
-        api_name=False,
-        queue=False,
+    top_k = gr.Slider(
+      label='Top-k',
+      minimum=1,
+      maximum=1000,
+      step=1,
+      value=5,
     )
 
-    clear_button.click(
-        fn=lambda: ([], ''),
-        outputs=[chatbot, saved_input],
-        queue=False,
-        api_name=False,
-    )
+  gr.Examples(
+    examples=[
+      '介绍下你自己',
+      '找到下列数组的中位数[3.1,6.2,1.3,8.4,10.5,11.6,2.1]，请用python代码完成以上功能',
+      '鸡和兔在一个笼子里，共有26个头，68只脚，那么鸡有多少只，兔有多少只？',
+      '以下物理常识题目，哪一个是错误的?A.在自然环境下，声音在固体中传播速度最快。B.牛顿第一定律:一个物体如果不受力作用，将保持静止或匀速直线运动的状态。C.牛顿第三定律:对于每个作用力，都有一个相等而反向的反作用力。D.声音在空气中的传播速度为1000m/s。',
+    ],
+    inputs=textbox,
+    outputs=[textbox, chatbot],
+    fn=process_example,
+    cache_examples=True,
+  )
+
+  gr.Markdown(LICENSE)
+
+  textbox.submit(
+    fn=clear_and_save_textbox,
+    inputs=textbox,
+    outputs=[textbox, saved_input],
+    api_name=False,
+    queue=False,
+  ).then(
+    fn=display_input,
+    inputs=[saved_input, chatbot],
+    outputs=chatbot,
+    api_name=False,
+    queue=False,
+  ).then(
+    fn=check_input_token_length,
+    inputs=[saved_input, chatbot, system_prompt],
+    api_name=False,
+    queue=False,
+  ).success(
+    fn=generate,
+    inputs=[
+      saved_input,
+      chatbot,
+      system_prompt,
+      max_new_tokens,
+      temperature,
+      top_p,
+      top_k,
+    ],
+    outputs=chatbot,
+    api_name=False,
+  )
+
+  button_event_preprocess = submit_button.click(
+    fn=clear_and_save_textbox,
+    inputs=textbox,
+    outputs=[textbox, saved_input],
+    api_name=False,
+    queue=False,
+  ).then(
+    fn=display_input,
+    inputs=[saved_input, chatbot],
+    outputs=chatbot,
+    api_name=False,
+    queue=False,
+  ).then(
+    fn=check_input_token_length,
+    inputs=[saved_input, chatbot, system_prompt],
+    api_name=False,
+    queue=False,
+  ).success(
+    fn=generate,
+    inputs=[
+      saved_input,
+      chatbot,
+      system_prompt,
+      max_new_tokens,
+      temperature,
+      top_p,
+      top_k,
+    ],
+    outputs=chatbot,
+    api_name=False,
+  )
+
+  retry_button.click(
+    fn=delete_prev_fn,
+    inputs=chatbot,
+    outputs=[chatbot, saved_input],
+    api_name=False,
+    queue=False,
+  ).then(
+    fn=display_input,
+    inputs=[saved_input, chatbot],
+    outputs=chatbot,
+    api_name=False,
+    queue=False,
+  ).then(
+    fn=generate,
+    inputs=[
+      saved_input,
+      chatbot,
+      system_prompt,
+      max_new_tokens,
+      temperature,
+      top_p,
+      top_k,
+    ],
+    outputs=chatbot,
+    api_name=False,
+  )
+
+  undo_button.click(
+    fn=delete_prev_fn,
+    inputs=chatbot,
+    outputs=[chatbot, saved_input],
+    api_name=False,
+    queue=False,
+  ).then(
+    fn=lambda x: x,
+    inputs=[saved_input],
+    outputs=textbox,
+    api_name=False,
+    queue=False,
+  )
+
+  clear_button.click(
+    fn=lambda: ([], ''),
+    outputs=[chatbot, saved_input],
+    queue=False,
+    api_name=False,
+  )
 
 demo.queue(max_size=20).launch()

model.py

@@ -2,79 +2,77 @@ from threading import Thread
 from typing import Iterator
 
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from transformers import AutoModelForCausalLM, AutoTokenizer
 from transformers.generation.utils import GenerationConfig
 
 model_id = 'baichuan-inc/Baichuan2-13B-Chat'
 
 if torch.cuda.is_available():
-    model = AutoModelForCausalLM.from_pretrained(
-        model_id,
-        # device_map='auto',
-        torch_dtype=torch.float16,
-        trust_remote_code=True
-    )
-    model = model.quantize(4).cuda()
-    model.generation_config = GenerationConfig.from_pretrained(model_id)
-else:
-    model = None
-tokenizer = AutoTokenizer.from_pretrained(
+  model = AutoModelForCausalLM.from_pretrained(
     model_id,
-    use_fast=False,
+    # device_map='auto',
+    torch_dtype=torch.float16,
     trust_remote_code=True
+  )
+  model = model.quantize(4).cuda()
+  model.generation_config = GenerationConfig.from_pretrained(model_id)
+else:
+  model = None
+tokenizer = AutoTokenizer.from_pretrained(
+  model_id,
+  use_fast=False,
+  trust_remote_code=True
 )
 
-def get_prompt(message: str, chat_history: list[tuple[str, str]],
-               system_prompt: str) -> str:
-    texts = [f'<s>[INST] <<SYS>>\n{system_prompt}\n<</SYS>>\n\n']
-    # The first user input is _not_ stripped
-    do_strip = False
-    for user_input, response in chat_history:
-        user_input = user_input.strip() if do_strip else user_input
-        do_strip = True
-        texts.append(f'{user_input} [/INST] {response.strip()} </s><s>[INST] ')
-    message = message.strip() if do_strip else message
-    texts.append(f'{message} [/INST]')
-    return ''.join(texts)
-
-
-def get_input_token_length(message: str, chat_history: list[tuple[str, str]], system_prompt: str) -> int:
-    prompt = get_prompt(message, chat_history, system_prompt)
-    input_ids = tokenizer([prompt], return_tensors='np', add_special_tokens=False)['input_ids']
-    return input_ids.shape[-1]
+def get_prompt(
+  message: str,
+  chat_history: list[tuple[str, str]],
+  system_prompt: str
+) -> str:
+  texts = [f'<s>[INST] <<SYS>>\n{system_prompt}\n<</SYS>>\n\n']
+  # The first user input is _not_ stripped
+  do_strip = False
+  for user_input, response in chat_history:
+    user_input = user_input.strip() if do_strip else user_input
+    do_strip = True
+    texts.append(f'{user_input} [/INST] {response.strip()} </s><s>[INST] ')
+  message = message.strip() if do_strip else message
+  texts.append(f'{message} [/INST]')
+  return ''.join(texts)
 
+def get_input_token_length(
+  message: str,
+  chat_history: list[tuple[str, str]],
+  system_prompt: str
+) -> int:
+  prompt = get_prompt(message, chat_history, system_prompt)
+  input_ids = tokenizer([prompt], return_tensors='np', add_special_tokens=False)['input_ids']
+  return input_ids.shape[-1]
 
-def run(message: str,
-        chat_history: list[tuple[str, str]],
-        system_prompt: str,
-        max_new_tokens: int = 2048,
-        temperature: float = 0.3,
-        top_p: float = 0.85,
-        top_k: int = 5) -> Iterator[str]:
-    prompt = get_prompt(message, chat_history, system_prompt)
-    inputs = tokenizer([prompt], return_tensors='pt', add_special_tokens=False).to('cuda')
+def run(
+  message: str,
+  chat_history: list[tuple[str, str]],
+  system_prompt: str,
+  max_new_tokens: int = 1024,
+  temperature: float = 1.0,
+  top_p: float = 0.95,
+  top_k: int = 5
+) -> Iterator[str]:
+  print(chat_history)
 
-    streamer = TextIteratorStreamer(
-      tokenizer,
-      timeout=10.,
-      skip_prompt=True,
-      skip_special_tokens=True
-    )
+  history = []
+  result=""
 
-    generate_kwargs = dict(
-        inputs,
-        streamer=streamer,
-        max_new_tokens=max_new_tokens,
-        do_sample=True,
-        top_p=top_p,
-        top_k=top_k,
-        temperature=temperature,
-        num_beams=1,
-    )
-    t = Thread(target=model.generate, kwargs=generate_kwargs)
-    t.start()
+  for i in chat_history:
+    history.append({"role": "user", "content": i[0]})
+    history.append({"role": "assistant", "content": i[1]})
+  
+  print(history)
 
-    outputs = []
-    for text in streamer:
-        outputs.append(text)
-        yield ''.join(outputs)
+  history.append({"role": "user", "content": message})
+  
+  for response in model.chat(tokenizer, history, stream=True):
+    print(response)
+    if "content" in response["choices"][0]["delta"]:
+      result = result + response["choices"][0]["delta"]["content"]
+      yield result