feat: llava-original

src/conversation.py

@@ -11,7 +11,7 @@ from enum import auto, IntEnum
 from io import BytesIO
 import os
 from typing import List, Any, Dict, Union, Tuple
-
+from loguru import logger
 
 class SeparatorStyle(IntEnum):
     """Separator styles."""
@@ -165,6 +165,10 @@ class Conversation:
                 ret += ""
             for i, (role, message) in enumerate(self.messages):
                 if message:
+                    logger.info("msg={}", message)
+                    if type(message) is tuple:
+                        message, images = message
+                        message = "<image>" * len(images) + message
                     ret += f"<|start_header_id|>{role}<|end_header_id|>\n\n"
                     ret += f"{message.strip()}<|eot_id|>"
                 else:
@@ -2083,6 +2087,38 @@ register_conv_template(
     )
 )
 
+# register for fire
+conv_llava_llama_3 = Conversation(
+    name="llava-original",
+    system_message="You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.",
+    roles=("user\n\n",
+           "assistant\n\n"),
+    # version="llama3",
+    messages=[],
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA3,
+    system_template="<|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|>",
+    sep="",
+    stop_str="<|eot_id|>",
+    stop_token_ids=[128001, 128009],
+)
+register_conv_template(conv_llava_llama_3)
+
+conv_llava_llama_3_student = Conversation(
+    name="llava-fire",
+    system_message="You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.",
+    roles=("user",
+           "assistant"),
+    # version="llama_v3_student",
+    messages=[],
+    system_template="<|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|>",
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA3,
+    sep="",
+    stop_str="<|eot_id|>",
+    stop_token_ids=[128001, 128009],
+)
+register_conv_template(conv_llava_llama_3_student)
 
 if __name__ == "__main__":
     from src.conversation import get_conv_template

src/model/model_adapter.py

@@ -45,7 +45,7 @@ from src.modules.exllama import ExllamaConfig, load_exllama_model
 from src.modules.xfastertransformer import load_xft_model, XftConfig
 from src.modules.gptq import GptqConfig, load_gptq_quantized
 from src.utils import get_gpu_memory
-
+from loguru import logger
 # Check an environment variable to check if we should be sharing Peft model
 # weights.  When false we treat all Peft models as separate.
 peft_share_base_weights = (
@@ -359,6 +359,7 @@ def load_model(
             raise e
 
     # Load model
+    logger.info("adapter {}", adapter)
     model, tokenizer = adapter.load_model(model_path, kwargs)
 
     if (
@@ -387,6 +388,7 @@ def load_model(
 def get_conversation_template(model_path: str) -> Conversation:
     """Get the default conversation template."""
     adapter = get_model_adapter(model_path)
+    logger.info("adapter {}", adapter)
     return adapter.get_default_conv_template(model_path)
 
 
@@ -2294,12 +2296,14 @@ class LlavaAdapter(BaseModelAdapter):
         from loguru import logger
         logger.info("model_path {}", model_path)
         if model_path in ["llava-fire", "llava-original"]:
+            return get_conv_template(model_path)
+        '''if model_path in ["llava-fire", "llava-original"]:
             from llava.conversation import conv_templates
             if model_path == "llava-fire":
                 return conv_templates["llama_v3_student"].copy()
             else:
-                return conv_templates["llama_v3"].copy()
-                
+                return conv_templates["llama_v3"].copy()'''
+        
         model_path = model_path.lower()
         if "34b" in model_path:
             return get_conv_template("llava-chatml")

src/model/model_llava.py

@@ -10,6 +10,8 @@ import copy
 import torch
 from llava.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path
 import spaces
+from io import BytesIO
+import base64
 #model_path = "/scratch/TecManDep/A_Models/llava-v1.6-vicuna-7b"
 #conv_template = "vicuna_v1" # Make sure you use correct chat template for different models
 
@@ -60,8 +62,13 @@ def inference():
 
 
 @spaces.GPU
-def inference_by_prompt_and_image(prompt, images):
+def inference_by_prompt_and_images(prompt, images):
     device = "cuda"
+    if len(images) > 0 and type(images[0]) is str:
+        image_data = []
+        for image in images:
+            image_data.append(Image.open(BytesIO(base64.b64decode(image))))
+        images = image_data
     image_tensor = process_images(images, image_processor_llava, model_llava.config)
     image_tensor = image_tensor.to(dtype=torch.float16, device=device)
     input_ids = tokenizer_image_token(prompt, tokenizer_llava, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(device)

src/serve/gradio_web_server.py

@@ -755,38 +755,19 @@ def get_model_description_md(models):
 def build_about():
     about_markdown = """
 # About Us
-Chatbot Arena is an open-source research project developed by members from [LMSYS](https://lmsys.org) and UC Berkeley [SkyLab](https://sky.cs.berkeley.edu/). Our mission is to build an open platform to evaluate LLMs by human preference in the real-world.
-We open-source our [FastChat](https://github.com/lm-sys/FastChat) project at GitHub and release chat and human feedback dataset. We invite everyone to join us!
-
+Placeholder
 ## Arena Core Team
-- [Lianmin Zheng](https://lmzheng.net/) (co-lead), [Wei-Lin Chiang](https://infwinston.github.io/) (co-lead), [Ying Sheng](https://sites.google.com/view/yingsheng/home), [Joseph E. Gonzalez](https://people.eecs.berkeley.edu/~jegonzal/), [Ion Stoica](http://people.eecs.berkeley.edu/~istoica/)
-
+Placeholder
 ## Past Members
-- [Siyuan Zhuang](https://scholar.google.com/citations?user=KSZmI5EAAAAJ), [Hao Zhang](https://cseweb.ucsd.edu/~haozhang/)
-
+Placeholder
 ## Learn more
-- Chatbot Arena [paper](https://arxiv.org/abs/2403.04132), [launch blog](https://lmsys.org/blog/2023-05-03-arena/), [dataset](https://github.com/lm-sys/FastChat/blob/main/docs/dataset_release.md), [policy](https://lmsys.org/blog/2024-03-01-policy/)
-- LMSYS-Chat-1M dataset [paper](https://arxiv.org/abs/2309.11998), LLM Judge [paper](https://arxiv.org/abs/2306.05685)
+Placeholder
 
 ## Contact Us
-- Follow our [X](https://x.com/lmsysorg), [Discord](https://discord.gg/HSWAKCrnFx) or email us at lmsys.org@gmail.com
-- File issues on [GitHub](https://github.com/lm-sys/FastChat)
-- Download our datasets and models on [HuggingFace](https://huggingface.co/lmsys)
+Placeholder
 
 ## Acknowledgment
-We thank [SkyPilot](https://github.com/skypilot-org/skypilot) and [Gradio](https://github.com/gradio-app/gradio) team for their system support.
-We also thank [UC Berkeley SkyLab](https://sky.cs.berkeley.edu/), [Kaggle](https://www.kaggle.com/), [MBZUAI](https://mbzuai.ac.ae/), [a16z](https://www.a16z.com/), [Together AI](https://www.together.ai/), [Hyperbolic](https://hyperbolic.xyz/), [Anyscale](https://www.anyscale.com/), [HuggingFace](https://huggingface.co/) for their generous sponsorship. Learn more about partnership [here](https://lmsys.org/donations/).
-
-<div class="sponsor-image-about">
-    <img src="https://storage.googleapis.com/public-arena-asset/skylab.png" alt="SkyLab">
-    <img src="https://storage.googleapis.com/public-arena-asset/kaggle.png" alt="Kaggle">
-    <img src="https://storage.googleapis.com/public-arena-asset/mbzuai.jpeg" alt="MBZUAI">
-    <img src="https://storage.googleapis.com/public-arena-asset/a16z.jpeg" alt="a16z">
-    <img src="https://storage.googleapis.com/public-arena-asset/together.png" alt="Together AI">
-    <img src="https://storage.googleapis.com/public-arena-asset/hyperbolic_logo.png" alt="Hyperbolic">
-    <img src="https://storage.googleapis.com/public-arena-asset/anyscale.png" alt="AnyScale">
-    <img src="https://storage.googleapis.com/public-arena-asset/huggingface.png" alt="HuggingFace">
-</div>
+Placeholder
 """
     gr.Markdown(about_markdown, elem_id="about_markdown")
 

vision-tmp-2024-07-10-conv.json

@@ -5,3 +5,5 @@
 {"tstamp": 1720589062.1758, "type": "chat", "model": "llava-fire", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1720589047.9171, "finish": 1720589062.1758, "state": {"template_name": "vicuna_v1.1", "system_message": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.", "roles": ["USER", "ASSISTANT"], "messages": [["USER", "Hello"], ["ASSISTANT", "hello"]], "offset": 0, "conv_id": "963f15cd5e224eb8ae02c67ed37b93c4", "model_name": "llava-fire", "has_csam_image": false}, "ip": "46.3.240.105"}
 {"tstamp": 1720589240.391, "type": "chat", "model": "llava-fire", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1720589221.915, "finish": 1720589240.391, "state": {"template_name": "vicuna_v1.1", "system_message": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.", "roles": ["USER", "ASSISTANT"], "messages": [["USER", "Hello"], ["ASSISTANT", "hello"]], "offset": 0, "conv_id": "062c8dbcf02d4779ad5cb89f836a7480", "model_name": "llava-fire", "has_csam_image": false}, "ip": "46.3.240.106"}
 {"tstamp": 1720589240.3917, "type": "chat", "model": "llava-original", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1720589221.917, "finish": 1720589240.3917, "state": {"template_name": "vicuna_v1.1", "system_message": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.", "roles": ["USER", "ASSISTANT"], "messages": [["USER", "Hello"], ["ASSISTANT", "The figure in the image is a can of S\u00f6dra Almighy, which is a beer. S\u00f6dra is a Swedish brewery known for its lager beers. The can's design is modern and minimalist, with a color scheme that includes black and white, and the text is in English. The label indicates that it is a dry-hopped beer, which means it has been flavored with hops that have not been steeped in the brewing process, giving it a unique taste profile. The can's design suggests a contemporary and possibly craft beer, which is often associated with a more complex flavor profile than traditional lagers."]], "offset": 0, "conv_id": "3255dc27032345ba845813c17981f372", "model_name": "llava-original", "has_csam_image": false}, "ip": "46.3.240.106"}
+{"tstamp": 1720593211.0342, "type": "chat", "model": "llava-fire", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1720593197.2799, "finish": 1720593211.0342, "state": {"template_name": "llava-fire", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Describe the image", ["2a9266044ab15516e8c0ba1a75f2b1e5"]]], ["assistant", "hello"]], "offset": 0, "conv_id": "f97c281c9d114ae7800b56eb204726fc", "model_name": "llava-fire", "has_csam_image": false}, "ip": "46.3.240.104"}
+{"tstamp": 1720593211.0433, "type": "chat", "model": "llava-original", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1720593197.2821, "finish": 1720593211.0433, "state": {"template_name": "llava-original", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user\n\n", "assistant\n\n"], "messages": [["user\n\n", ["Describe the image", ["2a9266044ab15516e8c0ba1a75f2b1e5"]]], ["assistant\n\n", "The image captures a moment of proposal between two individuals. The man, dressed in a classic gray suit, is standing upright with his arms outstretched, holding a ring in his left hand. His posture and facial expression suggest he is in the act of proposing. The woman, wearing a vibrant red dress, is kneeling on the ground with her hands clasped together, indicating her readiness to accept the proposal. The simplicity of the background focuses the viewer's attention on the interaction between the two characters. The image is devoid of any text or additional context, allowing the viewer to focus solely on the emotional exchange between the man and the woman."]], "offset": 0, "conv_id": "4731a06b55954bd8bfa659f1c3f9cc91", "model_name": "llava-original", "has_csam_image": false}, "ip": "46.3.240.104"}