require torch and nvidia-cublas

use nvidia pypi
use ggml-webui gpu llama.cpp build
use blas llama.cpp, streaming chat
fix initial history state
use list, not tuple

README.md

@@ -11,6 +11,9 @@ pinned: false
 
 # GGML UI Inference w/ HuggingFace Spaces
 
+- Fork this space to use your own GGML models. Simply update the [./config.yml](./config.yml) 
+- Contribute at [https://github.com/OpenAccess-AI-Collective/ggml-webui](https://github.com/OpenAccess-AI-Collective/ggml-webui)
+
 Brought to you by [OpenAccess AI Collective](https://github.com/OpenAccess-AI-Collective)
 
 

chat.py

@@ -21,25 +21,26 @@ while True:
 llm = Llama(model_path=fp, **config["llama_cpp"])
 
 
-def chat(inp, history, system_message):
+def user(message, history):
     history = history or []
-    history.append((inp, ""))
+    # Append the user's message to the conversation history
+    history.append([message, ""])
+    return "", history
 
-    messages = system_message + \
-               "\n".join(["\n".join(["### User: "+item[0], "Assistant: "+item[1]])
-                        for item in history])
 
+def chat(history, system_message):
     history = history or []
 
-    output = llm(messages, max_tokens=512, stop=["</s>", "<unk>", "### User:"], echo=False)
-    answer = output['choices'][0]['text']
-
-    history.pop()  # remove user input only history
-    history.append((inp, answer))
+    messages = system_message + \
+               "\n".join(["\n".join(["### User: "+item[0], "Assistant: "+item[1]])
+                        for item in history])
 
-    message = '' # This clears the message text
+    history[-1][1] = ""
+    for output in llm(messages, max_tokens=512, stop=["</s>", "<unk>", "### User:"], echo=False, stream=True):
+        answer = output['choices'][0]['text']
+        history[-1][1] = answer
 
-    return history, history, message
+        yield history, history
 
 
 def clear_chat(chat_history_state, chat_message):
@@ -65,26 +66,35 @@ with blocks:
             placeholder="Ask me anything.",
             lines=1,
         )
+    with gr.Row():
+        submit = gr.Button(value="Send message", variant="secondary").style(full_width=True)
         clear = gr.Button(value="New topic", variant="secondary").style(full_width=False)
-    submit = gr.Button(value="Send message", variant="secondary").style(full_width=True)
+        stop = gr.Button(value="Stop", variant="secondary").style(full_width=False)
 
     system_msg = gr.Textbox(
         start_message, label="System Message", interactive=False, visible=False)
 
-    # gr.Examples(
-    #     examples=[
-    #         "Tell me a joke about old houses.",
-    #         "Insult me.",
-    #         "What is the future of AI and large language models?",
-    #     ],
-    #     inputs=message,
-    # )
-
     chat_history_state = gr.State()
     clear.click(clear_chat, inputs=[chat_history_state, message], outputs=[chat_history_state, message])
     clear.click(lambda: None, None, chatbot, queue=False)
 
-    submit.click(chat, inputs=[message, chat_history_state, system_msg], outputs=[chatbot, chat_history_state, message])
-    message.submit(chat, inputs=[message, chat_history_state, system_msg], outputs=[chatbot, chat_history_state, message])
-
-blocks.queue(max_size=32, concurrency_count=3).launch(debug=True, server_name="0.0.0.0", server_port=7860)
+    submit_click_event = submit.click(
+        fn=user, inputs=[message, chat_history_state], outputs=[message, chat_history_state], queue=False
+    ).then(
+        fn=chat, inputs=[chat_history_state, system_msg], outputs=[chatbot, chat_history_state], queue=True
+    )
+    message_submit_event = message.submit(
+        fn=user, inputs=[message, chat_history_state], outputs=[message, chat_history_state], queue=False
+    ).then(
+        fn=chat, inputs=[chat_history_state, system_msg], outputs=[chatbot, chat_history_state], queue=True
+    )
+    stop.click(fn=None, inputs=None, outputs=None, cancels=[submit_click_event, message_submit_event], queue=False)
+
+    gr.Markdown("""
+        - This is running on a smaller, shared GPU, so it may take a few seconds to respond. 
+        - [Duplicate the Space](https://huggingface.co/spaces/openaccess-ai-collective/ggml-ui?duplicate=true) to skip the queue and run in a private space or to use your own GGML models.
+        - When using your own models, simply update the [./config.yml](./config.yml)")
+        - Contribute at [https://github.com/OpenAccess-AI-Collective/ggml-webui](https://github.com/OpenAccess-AI-Collective/ggml-webui)
+        """)
+
+blocks.queue(max_size=8, concurrency_count=2).launch(debug=True, server_name="0.0.0.0", server_port=7860)

requirements.txt

@@ -1,2 +1,6 @@
-llama-cpp-python @ https://github.com/OpenAccess-AI-Collective/ggml-webui/releases/download/v0.1.49-rc6/llama_cpp_python-cpu-0.1.49-cp38-cp38-linux_x86_64.whl
+--extra-index-url https://pypi.ngc.nvidia.com
+nvidia-cuda-runtime
+nvidia-cublas
+llama-cpp-python @ https://github.com/OpenAccess-AI-Collective/ggml-webui/releases/download/v0.1.50-rc3/llama_cpp_python-gpu-0.1.50-cp38-cp38-linux_x86_64.whl
 pyyaml
+torch