fix: conflict

app.py

@@ -9,7 +9,7 @@ from src.serve.gradio_block_arena_vision_named import build_side_by_side_vision_
 def main():
     with gr.Blocks() as demo:
         states = build_side_by_side_vision_ui_named(
-            models=["llava-fire"]
+            models=["llava-fire", "llava-original"]
         )
         
         demo.launch()

src/model/model_adapter.py

@@ -2297,12 +2297,12 @@ class LlavaAdapter(BaseModelAdapter):
         logger.info("model_path {}", model_path)
         if model_path in ["llava-fire", "llava-original"]:
             return get_conv_template(model_path)
-        '''if model_path in ["llava-fire", "llava-original"]:
-            from llava.conversation import conv_templates
-            if model_path == "llava-fire":
-                return conv_templates["llama_v3_student"].copy()
-            else:
-                return conv_templates["llama_v3"].copy()'''
+        # if model_path in ["llava-fire", "llava-original"]:
+        #     from llava.conversation import conv_templates
+        #     if model_path == "llava-fire":
+        #         return conv_templates["llama_v3_student"].copy()
+        #     else:
+        #         return conv_templates["llama_v3"].copy()
         
         model_path = model_path.lower()
         if "34b" in model_path:

src/model/model_llava.py

@@ -31,10 +31,10 @@ def load_llava_model(lora_checkpoint=None):
         tokenizer, model, image_processor, max_length = load_pretrained_model(
             model_path, None, model_name, device_map=device_map, offload_folder="save_folder1") # Add any other thing you want to pass in llava_model_args
     else:
-        #tokenizer, model, image_processor, max_length = load_pretrained_model(
-        #    lora_checkpoint, model_path, "llava_lora", device_map=device_map, offload_folder="save_folder2")
         tokenizer, model, image_processor, max_length = load_pretrained_model(
-            "li-qing/llava-llama-3-8b-fire-1m", None, "llava", device_map=device_map, offload_folder="save_folder2")
+            lora_checkpoint, model_path, "llava_lora", device_map=device_map, offload_folder="save_folder2")
+        #tokenizer, model, image_processor, max_length = load_pretrained_model(
+        #    "li-qing/llava-llama-3-8b-fire-1m", None, "llava", device_map=device_map, offload_folder="save_folder2")
     model.eval()
     model.tie_weights()
     logger.info(f"model device {model.device}")
@@ -42,7 +42,7 @@ def load_llava_model(lora_checkpoint=None):
 
 tokenizer_llava, model_llava, image_processor_llava, conv_template_llava = load_llava_model(None)
 tokenizer_llava_fire, model_llava_fire, image_processor_llava_fire, conv_template_llava = load_llava_model("checkpoints/llava-next-llama-3-8b-student-lora-merged-117408")
-# model_llava_fire.to("cuda")
+model_llava_fire.to("cuda")
 
 @spaces.GPU
 def inference():

src/serve/gradio_web_server.py

@@ -440,7 +440,7 @@ def bot_response(
         api_endpoint_info[model_name] if model_name in api_endpoint_info else None
     )
     images = conv.get_images()
-    logger.info(f"model_name: {model_name}; model_api_dict: {model_api_dict}; msg: {len(conv.messages)}")
+    logger.info(f"model_name: {model_name}; model_api_dict: {model_api_dict}; msg: {len(conv.messages)}; template: {conv.name}")
     if model_api_dict is None:
         if model_name == "llava-original":
             from src.model.model_llava import inference_by_prompt_and_images

vision-tmp-2024-08-13-conv.json

@@ -0,0 +1,10 @@
+{"tstamp": 1723534252.4698, "type": "chat", "model": "llava-fire", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723534216.7885, "finish": 1723534252.4698, "state": {"template_name": "llava-fire", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Please directly answer the question and provide the correct option letter, e.g., A, B, C, D.\nQuestion: As shown in the figure, then angle COE = ()\nChoices:\nA:30\u00b0\nB:140\u00b0\nC:50\u00b0\nD:60\u00b0", ["4cb5a5234624686ecd8b85a1fab041f9"]]], ["assistant", "The angle COE is formed by the intersection of the two lines, and since the lines are parallel and intersected by a transversal, the angle COE is equal to the angle AOC. The angle AOC is given as 50 degrees. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"]], "offset": 0, "conv_id": "bdc85c678b134922825813b7f3086991", "model_name": "llava-fire", "has_csam_image": false}, "ip": "123.127.249.92"}
+{"tstamp": 1723534252.4762, "type": "chat", "model": "llava-original", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723534236.2121, "finish": 1723534252.4762, "state": {"template_name": "llava-original", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Please directly answer the question and provide the correct option letter, e.g., A, B, C, D.\nQuestion: As shown in the figure, then angle COE = ()\nChoices:\nA:30\u00b0\nB:140\u00b0\nC:50\u00b0\nD:60\u00b0", ["4cb5a5234624686ecd8b85a1fab041f9"]]], ["assistant", "The angle COE is formed by the intersection of two lines, and it is equal to the sum of the angles at point E formed by the intersecting lines. Since the angle at point E is given as 50\u00b0, and the lines intersect at a right angle (90\u00b0), the angle COE is the sum of these two angles. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"]], "offset": 0, "conv_id": "28feecf9908743acad0b8043bec6b83d", "model_name": "llava-original", "has_csam_image": false}, "ip": "123.127.249.92"}
+{"tstamp": 1723534268.1655, "type": "chat", "model": "llava-fire", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723534259.4016, "finish": 1723534268.1655, "state": {"template_name": "llava-fire", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Please directly answer the question and provide the correct option letter, e.g., A, B, C, D.\nQuestion: As shown in the figure, then angle COE = ()\nChoices:\nA:30\u00b0\nB:140\u00b0\nC:50\u00b0\nD:60\u00b0", ["4cb5a5234624686ecd8b85a1fab041f9"]]], ["assistant", "The angle COE is formed by the intersection of the two lines, and since the lines are parallel and intersected by a transversal, the angle COE is equal to the angle AOC. The angle AOC is given as 50 degrees. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"], ["user", "Your answer is incorrect. The question asks for the angle COE in the context of the figure provided. Consider the relationships between the angles and the lines in the figure to find the correct answer. Try again by analyzing the given diagram more carefully."], ["assistant", "I apologize for the mistake. Let me re-evaluate the figure. The angle COE is formed by the intersection of the two lines, and since the lines are parallel and intersected by a transversal, the angle COE is equal to the angle AOC. The angle AOC is given as 50 degrees. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"]], "offset": 0, "conv_id": "bdc85c678b134922825813b7f3086991", "model_name": "llava-fire", "has_csam_image": false}, "ip": "123.127.249.92"}
+{"tstamp": 1723534268.168, "type": "chat", "model": "llava-original", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723534263.5839, "finish": 1723534268.168, "state": {"template_name": "llava-original", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Please directly answer the question and provide the correct option letter, e.g., A, B, C, D.\nQuestion: As shown in the figure, then angle COE = ()\nChoices:\nA:30\u00b0\nB:140\u00b0\nC:50\u00b0\nD:60\u00b0", ["4cb5a5234624686ecd8b85a1fab041f9"]]], ["assistant", "The angle COE is formed by the intersection of two lines, and it is equal to the sum of the angles at point E formed by the intersecting lines. Since the angle at point E is given as 50\u00b0, and the lines intersect at a right angle (90\u00b0), the angle COE is the sum of these two angles. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"], ["user", "Your answer is incorrect. The question asks for the angle COE in the context of the figure provided. Consider the relationships between the angles and the lines in the figure to find the correct answer. Try again by analyzing the given diagram more carefully."], ["assistant", "I apologize for the mistake in my previous response. Upon analyzing the diagram more carefully, it is clear that the angle COE is formed by the intersection of two lines, and it is equal to the sum of the angles at point E formed by the intersecting lines. Since the angle at point E is given as 50\u00b0, and the lines intersect at a right angle (90\u00b0), the angle COE is the sum of these two angles. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"]], "offset": 0, "conv_id": "28feecf9908743acad0b8043bec6b83d", "model_name": "llava-original", "has_csam_image": false}, "ip": "123.127.249.92"}
+{"tstamp": 1723534710.908, "type": "chat", "model": "llava-fire", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723534674.1309, "finish": 1723534710.908, "state": {"template_name": "llava-fire", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Please directly answer the question and provide the correct option letter, e.g., A, B, C, D.\nQuestion: As shown in the figure, then angle COE = ()\nChoices:\nA:30\u00b0\nB:140\u00b0\nC:50\u00b0\nD:60\u00b0", ["4cb5a5234624686ecd8b85a1fab041f9"]]], ["assistant", "The angle COE is formed by the intersection of the two lines, and since the lines are parallel and intersected by a transversal, the angle COE is equal to the angle AOC. The angle AOC is given as 50 degrees. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"]], "offset": 0, "conv_id": "01329dab7a764a5d81eb9a9f15824d37", "model_name": "llava-fire", "has_csam_image": false}, "ip": "123.127.249.92"}
+{"tstamp": 1723534710.9129, "type": "chat", "model": "llava-original", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723534694.9349, "finish": 1723534710.9129, "state": {"template_name": "llava-original", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Please directly answer the question and provide the correct option letter, e.g., A, B, C, D.\nQuestion: As shown in the figure, then angle COE = ()\nChoices:\nA:30\u00b0\nB:140\u00b0\nC:50\u00b0\nD:60\u00b0", ["4cb5a5234624686ecd8b85a1fab041f9"]]], ["assistant", "The angle COE is formed by the intersection of two lines, and it is equal to the sum of the angles at point E formed by the intersecting lines. Since the angle at point E is given as 50\u00b0, and the lines intersect at a right angle (90\u00b0), the angle COE is the sum of these two angles. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"]], "offset": 0, "conv_id": "0385fa96037c4e05ae2b89f90c03f1ff", "model_name": "llava-original", "has_csam_image": false}, "ip": "123.127.249.92"}
+{"tstamp": 1723534727.786, "type": "chat", "model": "llava-fire", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723534721.1508, "finish": 1723534727.786, "state": {"template_name": "llava-fire", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Please directly answer the question and provide the correct option letter, e.g., A, B, C, D.\nQuestion: As shown in the figure, then angle COE = ()\nChoices:\nA:30\u00b0\nB:140\u00b0\nC:50\u00b0\nD:60\u00b0", ["4cb5a5234624686ecd8b85a1fab041f9"]]], ["assistant", "The angle COE is formed by the intersection of the two lines, and since the lines are parallel and intersected by a transversal, the angle COE is equal to the angle AOC. The angle AOC is given as 50 degrees. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"], ["user", "You are getting closer, but there is still a mistake. The angle COE is not supplementary to the angle at point E. Instead, consider the properties of the angles formed by intersecting lines. Think about the relationship between the angles around point E and how they add up to 180\u00b0."], ["assistant", "I apologize for the mistake. Since the lines are parallel and intersected by a transversal, the angle COE is supplementary to the angle at point E. Therefore, the correct answer to the question is:\n\nD: 60\u00b0"]], "offset": 0, "conv_id": "01329dab7a764a5d81eb9a9f15824d37", "model_name": "llava-fire", "has_csam_image": false}, "ip": "123.127.249.92"}
+{"tstamp": 1723534727.7882, "type": "chat", "model": "llava-original", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723534723.2577, "finish": 1723534727.7882, "state": {"template_name": "llava-original", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Please directly answer the question and provide the correct option letter, e.g., A, B, C, D.\nQuestion: As shown in the figure, then angle COE = ()\nChoices:\nA:30\u00b0\nB:140\u00b0\nC:50\u00b0\nD:60\u00b0", ["4cb5a5234624686ecd8b85a1fab041f9"]]], ["assistant", "The angle COE is formed by the intersection of two lines, and it is equal to the sum of the angles at point E formed by the intersecting lines. Since the angle at point E is given as 50\u00b0, and the lines intersect at a right angle (90\u00b0), the angle COE is the sum of these two angles. Therefore, the correct answer to the question is:\n\nC: 50\u00b0"], ["user", "You are getting closer, but there is still a mistake. The angle COE is not supplementary to the angle at point E. Instead, consider the properties of the angles formed by intersecting lines. Think about the relationship between the angles around point E and how they add up to 180\u00b0."], ["assistant", "I apologize for the mistake. You are correct that the angle COE is not supplementary to the angle at point E. The angle COE is the exterior angle for the triangle COE, and it is equal to the sum of the opposite interior angles, which are the angles at point E and point C. Since the angle at point E is given as 50\u00b0, and the angle at point C is a right angle (90\u00b0), the angle COE is the sum of these two angles.\n\nTherefore, the correct answer to the question is:\n\nC: 50\u00b0"]], "offset": 0, "conv_id": "0385fa96037c4e05ae2b89f90c03f1ff", "model_name": "llava-original", "has_csam_image": false}, "ip": "123.127.249.92"}
+{"tstamp": 1723535139.9102, "type": "chat", "model": "llava-fire", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723535088.1249, "finish": 1723535139.9102, "state": {"template_name": "llava-fire", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end. \nQuestion: Which of the following could Mason's test show? \nChoices: \nA. how steady a parachute with a 1 m vent was at 200 km per hour \nB. if the spacecraft was damaged when using a parachute with a 1 m vent going 200 km per hour \nC. whether a parachute with a 1 m vent would swing too much at 400 km per hour\nFormat your answer as\nThought: <your reasoning process>\nAnswer: <your answer>", ["479749611fcd06176853ab287291548e"]]], ["assistant", "Thought: Mason's test is used to evaluate the stability of a parachute. The question asks about the stability of a parachute with a 1 m vent at 200 km/h. Since Mason's test is used to determine if a parachute will swing excessively, it should be related to the stability of the parachute.\n\nAnswer: C. whether a parachute with a 1 m vent would swing too much at 400 km/h"]], "offset": 0, "conv_id": "139893f2980e4d44843a019117450c12", "model_name": "llava-fire", "has_csam_image": false}, "ip": "123.127.249.92"}
+{"tstamp": 1723535139.9183, "type": "chat", "model": "llava-original", "gen_params": {"temperature": 0.7, "top_p": 1.0, "max_new_tokens": 1024}, "start": 1723535116.4898, "finish": 1723535139.9183, "state": {"template_name": "llava-original", "system_message": "You are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.", "roles": ["user", "assistant"], "messages": [["user", ["Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end. \nQuestion: Which of the following could Mason's test show? \nChoices: \nA. how steady a parachute with a 1 m vent was at 200 km per hour \nB. if the spacecraft was damaged when using a parachute with a 1 m vent going 200 km per hour \nC. whether a parachute with a 1 m vent would swing too much at 400 km per hour\nFormat your answer as\nThought: <your reasoning process>\nAnswer: <your answer>", ["479749611fcd06176853ab287291548e"]]], ["assistant", "Mason's test is a method used to evaluate the stability of a parachute at high speeds. It involves measuring the angle of the parachute relative to the ground as it descends. The test is typically performed at various speeds to determine the parachute's stability and whether it would swing excessively, which could indicate a risk of damage or loss of control. Given the options provided and the context of the image, which shows a parachute in a controlled environment, the correct answer would be:\n\nA. how steady a parachute with a 1 m vent was at 200 km per hour\n\nThis is because Mason's test is designed to measure the stability of a parachute at high speeds, and the image shows a parachute with a 1 m vent, which is the type of vent typically used in such tests. The other options are not relevant to the image provided."]], "offset": 0, "conv_id": "fdb5970bf0394150a078d2af3c37bf27", "model_name": "llava-original", "has_csam_image": false}, "ip": "123.127.249.92"}