update

app.py

@@ -54,7 +54,7 @@ with gr.Blocks() as demo:
                                  avatar_images=("assets/man.png", "assets/bot.png"))
 
             with gradio.Tab("Self Chat"):
-                generated_text_1 = gr.Textbox(show_label=False, placeholder="...", lines=10, visible=False)
+                input_text_1 = gr.Textbox(show_label=False, placeholder="...", lines=10, visible=False)
                 generate_btn = gr.Button("🤔️ Self-Chat", variant="primary")
                 with gr.Row():
                     retry_btn = gr.Button("🔄  Retry", variant="secondary", size="sm", )
@@ -68,7 +68,7 @@ with gr.Blocks() as demo:
 
             with gradio.Tab("Response Generator"):
                 with gr.Row():
-                    generated_text_2 = gr.Textbox(show_label=False, placeholder="Please type your input", scale=7)
+                    input_text_2 = gr.Textbox(show_label=False, placeholder="Please type your input", scale=7)
                     generate_btn_2 = gr.Button("Send", variant="primary")
                 with gr.Row():
                     retry_btn_2 = gr.Button("🔄  Regenerate", variant="secondary", size="sm", )
@@ -78,7 +78,7 @@ with gr.Blocks() as demo:
 
             with gradio.Tab("User Simulator"):
                 with gr.Row():
-                    generated_text_3 = gr.Textbox(show_label=False, placeholder="Please type your response", scale=7)
+                    input_text_3 = gr.Textbox(show_label=False, placeholder="Please type your response", scale=7)
                     generate_btn_3 = gr.Button("Send", variant="primary")
                 with gr.Row():
                     retry_btn_3 = gr.Button("🔄  Regenerate", variant="secondary", size="sm", )
@@ -116,17 +116,41 @@ with gr.Blocks() as demo:
                     label="Top-k",
                 )
 
-    ########
     history = gr.State([{"role": "system", "content": system_list[0]}])  # 有用信息只有个system，其他和chatbot内容重叠
     system.change(reset_state, inputs=[system], outputs=[chatbot, history])
-    clear_btn.click(reset_state, inputs=[system], outputs=[chatbot, history])
 
-    generate_btn.click(generate, [chatbot, history], outputs=[generated_text_1, chatbot, history],
+    ######## tab1
+    generate_btn.click(generate, [chatbot, history], outputs=[chatbot, history],
                        show_progress="full")
-    retry_btn.click(undo_generate, [chatbot, history], outputs=[generated_text_1, chatbot, history]) \
-        .then(generate, [chatbot, history], outputs=[generated_text_1, chatbot, history],
+    retry_btn.click(undo_generate, [chatbot, history], outputs=[chatbot, history]) \
+        .then(generate, [chatbot, history], outputs=[chatbot, history],
+              show_progress="full")
+    undo_btn.click(undo_generate, [chatbot, history], outputs=[chatbot, history])
+    clear_btn.click(reset_state, inputs=[system], outputs=[chatbot, history])
+
+    ######## tab2
+    generate_btn_2.click(append_user, [input_text_2, chatbot, history], outputs=[chatbot, history]) \
+        .then(generate_assistant_message, [chatbot, history], outputs=[chatbot, history],
+              show_progress="full")
+    retry_btn_2.click(undo_generate, [chatbot, history], outputs=[chatbot, history]) \
+        .then(generate, [chatbot, history], outputs=[chatbot, history],
               show_progress="full")
-    undo_btn.click(undo_generate, [chatbot, history], outputs=[generated_text_1, chatbot, history])
+    undo_btn_2.click(undo_generate, [chatbot, history], outputs=[chatbot, history])
+    clear_btn_2.click(reset_state, inputs=[system], outputs=[chatbot, history])\
+        .then(reset_user_input, outputs=[input_text_2])
+
+    ######## tab3
+    generate_btn_3.click(append_assistant, [input_text_3, chatbot, history], outputs=[chatbot, history]) \
+        .then(generate_assistant_message, [chatbot, history], outputs=[chatbot, history],
+              show_progress="full")
+    retry_btn_3.click(undo_generate, [chatbot, history], outputs=[chatbot, history]) \
+        .then(generate, [chatbot, history], outputs=[chatbot, history],
+              show_progress="full")
+    undo_btn_3.click(undo_generate, [chatbot, history], outputs=[chatbot, history])
+    clear_btn_3.click(reset_state, inputs=[system], outputs=[chatbot, history])\
+        .then(reset_user_input, outputs=[input_text_3])
+
+
 
     slider_max_tokens.change(set_max_tokens, inputs=[slider_max_tokens])
     slider_temperature.change(set_temperature, inputs=[slider_temperature])

app_util.py

@@ -19,51 +19,51 @@ from models.cpp_qwen2 import bot
 # gr.Chatbot.postprocess = postprocess
 
 
-def generate_query(chatbot, history):
+def generate_user_message(chatbot, history):
     if history and history[-1]["role"] == "user":
         gr.Warning('You should generate assistant-response.')
         yield None, chatbot, history
     else:
         chatbot.append(None)
         streamer = bot.generate(history, stream=True)
-        for query, query_tokens in streamer:
-            chatbot[-1] = (query, None)
-            yield query, chatbot, history
-        query_tokens = bot.strip_stoptokens(query_tokens)
-        history.append({"role": "user", "content": query, "tokens": query_tokens})
-        yield query, chatbot, history
+        for user_content, user_tokens in streamer:
+            chatbot[-1] = (user_content, None)
+            yield user_content, chatbot, history
+        user_tokens = bot.strip_stoptokens(user_tokens)
+        history.append({"role": "user", "content": user_content, "tokens": user_tokens})
+        yield chatbot, history
 
 
-def generate_response(chatbot, history):
+def generate_assistant_message(chatbot, history):
     """
     auto-mode：query is None
     manual-mode：query 是用户输入
     """
     logger.info(f"generating {json.dumps(history, ensure_ascii=False)}")
-    query = history[-1]["content"]
+    user_content = history[-1]["content"]
     if history[-1]["role"] != "user":
         gr.Warning('You should generate or type user-input first.')
         yield None, chatbot, history
     else:
         streamer = bot.generate(history, stream=True)
-        for response, response_tokens in streamer:
-            chatbot[-1] = (query, response)
-            yield response, chatbot, history
+        for assistant_content, assistant_tokens in streamer:
+            chatbot[-1] = (user_content, assistant_content)
+            yield assistant_content, chatbot, history
 
-        response_tokens = bot.strip_stoptokens(response_tokens)
-        history.append({"role": "assistant", "content": response, "tokens": response_tokens})
+        assistant_tokens = bot.strip_stoptokens(assistant_tokens)
+        history.append({"role": "assistant", "content": assistant_content, "tokens": assistant_tokens})
         print(f"chatbot is {chatbot}")
         print(f"history is {history}")
-        yield response, chatbot, history
+        yield chatbot, history
 
 
 def generate(chatbot, history):
     logger.info(f"chatbot: {chatbot}; history: {history}")
     streamer = None
     if history[-1]["role"] in ["assistant", "system"]:
-        streamer = generate_query(chatbot, history)
+        streamer = generate_user_message(chatbot, history)
     elif history[-1]["role"] == "user":
-        streamer = generate_response(chatbot, history)
+        streamer = generate_assistant_message(chatbot, history)
     else:
         gr.Warning("bug")
 
@@ -71,6 +71,26 @@ def generate(chatbot, history):
         yield out
 
 
+def append_user(input_content, chatbot, history):
+    if history[-1]["role"] == "user":
+        gr.Warning('You should generate assistant-response.')
+        return chatbot, history
+
+    chatbot.append((input_content, None))
+    history.append({"role": "user", "content": input_content})
+    return chatbot, history
+
+
+def append_assistant(input_content, chatbot, history):
+    if history[-1]["role"] != "user":
+        gr.Warning('You should generate or type user-input first.')
+        return chatbot, history
+    chatbot[-1] = (chatbot[-1][0], input_content)
+    history.append({"role": "assistant", "content": input_content})
+    return chatbot, history
+
+
+
 def undo_generate(chatbot, history):
     if history[-1]["role"] == "user":
         history = history[:-1]

models/cpp_qwen2.py

@@ -1,25 +1,71 @@
 """
-https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/local.py
-https://github.com/awinml/llama-cpp-python-bindings
 
-python convert_hf_to_gguf.py --outtype f16 Qwen1.5-0.5B-Chat
 
-python convert_hf_to_gguf.py /workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/
+## convert to gguf
 
+python convert_hf_to_gguf.py /workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/
 
+## predict
 ./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -p "I believe the meaning of life is" -n 128
-
 ./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -f prompt.txt -n 128
-
 ./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -p "You are a helpful assistant" -cnv
 
 
+## timing
+
+
+**重庆GPU服务器，cache为空 **
+llama_print_timings:        load time =    1711.48 ms
+llama_print_timings:      sample time =     214.87 ms /   122 runs   (    1.76 ms per token,   567.78 tokens per second)
+llama_print_timings: prompt eval time =     892.14 ms /     5 tokens (  178.43 ms per token,     5.60 tokens per second)
+llama_print_timings:        eval time =    4277.26 ms /   121 runs   (   35.35 ms per token,    28.29 tokens per second)
+llama_print_timings:       total time =    8351.28 ms /   126 tokens
+
+llama_print_timings:        load time =    1711.48 ms
+llama_print_timings:      sample time =      45.11 ms /    25 runs   (    1.80 ms per token,   554.24 tokens per second)
+llama_print_timings: prompt eval time =    1059.46 ms /     5 tokens (  211.89 ms per token,     4.72 tokens per second)
+llama_print_timings:        eval time =     843.71 ms /    24 runs   (   35.15 ms per token,    28.45 tokens per second)
+llama_print_timings:       total time =    2501.50 ms /    29 tokens
+
+
+llama_print_timings:        load time =    1711.48 ms
+llama_print_timings:      sample time =     227.75 ms /   125 runs   (    1.82 ms per token,   548.85 tokens per second)
+llama_print_timings: prompt eval time =    2056.86 ms /     5 tokens (  411.37 ms per token,     2.43 tokens per second)
+llama_print_timings:        eval time =    4657.86 ms /   124 runs   (   37.56 ms per token,    26.62 tokens per second)
+llama_print_timings:       total time =    9532.50 ms /   129 tokens
+
+llama_print_timings:        load time =    1711.48 ms
+llama_print_timings:      sample time =      73.89 ms /    41 runs   (    1.80 ms per token,   554.84 tokens per second)
+llama_print_timings: prompt eval time =    2621.25 ms /     5 tokens (  524.25 ms per token,     1.91 tokens per second)   # 0.5秒/token
+llama_print_timings:        eval time =    1430.91 ms /    40 runs   (   35.77 ms per token,    27.95 tokens per second)
+llama_print_timings:       total time =    4848.09 ms /    45 tokens
+
+**hf-space，cache为空 ** -----------
+llama_print_timings:        load time =   28230.06 ms
+llama_print_timings:      sample time =     147.58 ms /     8 runs   (   18.45 ms per token,    54.21 tokens per second)
+llama_print_timings: prompt eval time =   28864.82 ms /     5 tokens ( 5772.96 ms per token,     0.17 tokens per second)   # 5.7秒/token
+llama_print_timings:        eval time =    1557.94 ms /     7 runs   (  222.56 ms per token,     4.49 tokens per second)
+llama_print_timings:       total time =   30753.48 ms /    12 tokens
+
+llama_print_timings:        load time =   28230.06 ms
+llama_print_timings:      sample time =      74.34 ms /    61 runs   (    1.22 ms per token,   820.52 tokens per second)
+llama_print_timings: prompt eval time =   28821.26 ms /     9 tokens ( 3202.36 ms per token,     0.31 tokens per second)
+llama_print_timings:        eval time =   21634.71 ms /    60 runs   (  360.58 ms per token,     2.77 tokens per second)
+llama_print_timings:       total time =   51255.55 ms /    69 tokens
+
+llama_print_timings:        load time =   28230.06 ms
+llama_print_timings:      sample time =      98.03 ms /    68 runs   (    1.44 ms per token,   693.66 tokens per second)
+llama_print_timings: prompt eval time =   27749.35 ms /     5 tokens ( 5549.87 ms per token,     0.18 tokens per second)
+llama_print_timings:        eval time =   26998.58 ms /    67 runs   (  402.96 ms per token,     2.48 tokens per second)
+llama_print_timings:       total time =   56335.37 ms /    72 tokens
+
 ## reference
 
+- https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/local.py
+- https://github.com/awinml/llama-cpp-python-bindings
 - https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/llms/llamacpp.py
 - https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/server.py
 - https://github.com/abetlen/llama-cpp-python/blob/main/llama_cpp/server/app.py
-
 """
 
 import json