Add application file

README.md

@@ -1,8 +1,8 @@
 ---
 title: Multimodal Vision Insight
-emoji: 🦀
-colorFrom: purple
-colorTo: red
+emoji: 🔍
+colorFrom: blue
+colorTo: purple
 sdk: gradio
 sdk_version: 3.45.2
 app_file: app.py
@@ -10,4 +10,28 @@ pinned: false
 license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Explore the world of multimodal interactions with the Multimodal Vision Insight (MVI) application. With the power of Vision Language Models (VLMs), MVI provides an interface for users to interact with text and images seamlessly. Built on top of Gradio, this application serves as a bridge between human inputs and machine understanding, fostering a cooperative environment for solving real-world tasks.
+
+[Check out the configuration reference for more details on configuring your space.](https://huggingface.co/docs/hub/spaces-config-reference)
+
+## Features:
+- **Multimodal Interaction**: Engage in a conversation with the model using both text and images.
+- **Real-time Feedback**: Receive instant responses from the model to navigate through tasks efficiently.
+- **High-Resolution Image Understanding**: Utilize high-resolution images for fine-grained recognition and understanding, enhancing the quality of interaction.
+- **User-Friendly Interface**: With a clean and intuitive UI, exploring multimodal interactions has never been easier.
+
+## Usage:
+1. Input your text or upload an image to start the conversation.
+2. Use the available controls to navigate through the conversation, regenerate responses, or clear the history.
+3. Explore the potential of Vision Language Models in understanding and interacting with multimodal data.
+
+## Developers:
+Developed by Keyvan Hardani (Keyvven on [Twitter](https://twitter.com/Keyvven) and on [Hugging Face](https://huggingface.co/Keyvven)).
+Special thanks to [@Artificialguybr](https://twitter.com/artificialguybr) for the inspiration from his code.
+
+## Acknowledgments:
+This project is powered by Alibaba Cloud's Qwen-VL, a state-of-the-art multimodal large vision language model.
+
+Feel free to explore, contribute, and raise issues on the [project repository](<link to your repository>).
+
+

app.py

@@ -0,0 +1,150 @@
+import gradio as gr
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from PIL import Image
+import re
+import copy
+import secrets
+from pathlib import Path
+
+# Constants
+BOX_TAG_PATTERN = r"<box>([\s\S]*?)</box>"
+PUNCTUATION = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
+
+# Initialize model and tokenizer
+tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-VL-Chat-Int4", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen-VL-Chat-Int4", device_map="auto", trust_remote_code=True).eval()
+
+def format_text(text):
+    """Format text for rendering in the chat UI."""
+    lines = text.split("\n")
+    lines = [line for line in lines if line != ""]
+    count = 0
+    for i, line in enumerate(lines):
+        if "```" in line:
+            count += 1
+            items = line.split("`")
+            if count % 2 == 1:
+                lines[i] = f'<pre><code class="language-{items[-1]}">'
+            else:
+                lines[i] = f"<br></code></pre>"
+        else:
+            if i > 0:
+                if count % 2 == 1:
+                    line = line.replace("`", r"\`")
+                    line = line.replace("<", "&lt;")
+                    line = line.replace(">", "&gt;")
+                    line = line.replace(" ", "&nbsp;")
+                    line = line.replace("*", "&ast;")
+                    line = line.replace("_", "&lowbar;")
+                    line = line.replace("-", "&#45;")
+                    line = line.replace(".", "&#46;")
+                    line = line.replace("!", "&#33;")
+                    line = line.replace("(", "&#40;")
+                    line = line.replace(")", "&#41;")
+                    line = line.replace("$", "&#36;")
+                lines[i] = "<br>" + line
+    text = "".join(lines)
+    return text
+
+def get_chat_response(chatbot, task_history):
+    """Generate a response using the model."""
+    chat_query = chatbot[-1][0]
+    query = task_history[-1][0]
+    history_cp = copy.deepcopy(task_history)
+    full_response = ""
+
+    history_filter = []
+    pic_idx = 1
+    pre = ""
+    for i, (q, a) in enumerate(history_cp):
+        if isinstance(q, (tuple, list)):
+            q = f'Picture {pic_idx}: <img>{q[0]}</img>'
+            pre += q + '\n'
+            pic_idx += 1
+        else:
+            pre += q
+            history_filter.append((pre, a))
+            pre = ""
+    history, message = history_filter[:-1], history_filter[-1][0]
+    response, history = model.chat(tokenizer, message, history=history)
+    # ... (rest of the code remains the same)
+
+def handle_text_input(history, task_history, text):
+    """Handle text input from the user."""
+    task_text = text
+    if len(text) >= 2 and text[-1] in PUNCTUATION and text[-2] not in PUNCTUATION:
+        task_text = text[:-1]
+    history = history + [(format_text(text), None)]
+    task_history = task_history + [(task_text, None)]
+    return history, task_history, ""
+
+def handle_file_upload(history, task_history, file):
+    """Handle file upload from the user."""
+    history = history + [((file.name,), None)]
+    task_history = task_history + [((file.name,), None)]
+    return history, task_history
+
+def clear_input():
+    """Clear the user input."""
+    return gr.update(value="")
+
+def clear_history(task_history):
+    """Clear the chat history."""
+    task_history.clear()
+    return []
+
+def handle_regeneration(chatbot, task_history):
+    """Handle the regeneration of the last response."""
+    print("Regenerate clicked")
+    print("Before:", task_history, chatbot)
+    if not task_history:
+        return chatbot
+    item = task_history[-1]
+    if item[1] is None:
+        return chatbot
+    task_history[-1] = (item[0], None)
+    chatbot_item = chatbot.pop(-1)
+    if chatbot_item[0] is None:
+        chatbot[-1] = (chatbot[-1][0], None)
+    else:
+        chatbot.append((chatbot_item[0], None))
+    print("After:", task_history, chatbot)
+    return get_chat_response(chatbot, task_history)
+
+# Custom CSS
+css = '''
+    .gradio-container {
+        max-width: 800px !important;
+    }
+    /* ... (add more custom CSS if needed) */
+'''
+
+# Build and launch the UI
+with gr.Blocks(css=css) as demo:
+    gr.Markdown("# Qwen-VL-Chat Bot")
+    gr.Markdown(
+        "## Developed by Keyvan Hardani (Keyvven on [Twitter](https://twitter.com/Keyvven) and on [Hugging Face](https://huggingface.co/Keyvven))\n"
+        "Special thanks to [@Artificialguybr](https://twitter.com/artificialguybr) for the inspiration from his code.\n"
+        "### Qwen-VL: A Multimodal Large Vision Language Model by Alibaba Cloud\n"
+    )
+    chatbot = gr.Chatbot(label='Qwen-VL-Chat', elem_classes="control-height", height=520)
+    query = gr.Textbox(lines=2, label='Input')
+    task_history = gr.State([])
+
+    with gr.Row():
+        upload_btn = gr.UploadButton("📁 Upload", file_types=["image"])
+        submit_btn = gr.Button("🚀 Submit")
+        regen_btn = gr.Button("🤔️ Regenerate")
+        clear_btn = gr.Button("🧹 Clear History")
+    
+    gr.Markdown("### Key Features:\n- **Strong Performance**: Surpasses existing LVLMs on multiple English benchmarks including Zero-shot Captioning and VQA.\n- **Multi-lingual Support**: Supports English, Chinese, and multi-lingual conversation.\n- **High Resolution**: Utilizes 448*448 resolution for fine-grained recognition and understanding.")
+    submit_btn.click(handle_text_input, [chatbot, task_history, query], [chatbot, task_history]).then(
+        get_chat_response, [chatbot, task_history], [chatbot], show_progress=True
+    )
+    submit_btn.click(clear_input, [], [query])
+    clear_btn.click(clear_history, [task_history], [chatbot], show_progress=True)
+    regen_btn.click(handle_regeneration, [chatbot, task_history], [chatbot], show_progress=True)
+    upload_btn.upload(handle_file_upload, [chatbot, task_history, upload_btn], [chatbot, task_history], show_progress=True)
+
+# Launch the demo
+demo.launch()

requirements.txt

@@ -0,0 +1,4 @@
+gradio
+transformers
+Pillow
+transformers_stream_generator