diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..53aee21b9a286bf9d1904e9527c3f0b047f4f0ea 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
index cb98f377ac878f29d80ec540f8be8aa2880382f4..f1b922a481a93e59ed933cfe95059754f64d88ef 100644
--- a/README.md
+++ b/README.md
@@ -1,46 +1,14 @@
 ---
-title: GenAI Video Arena
-emoji: 🖼
-colorFrom: purple
-colorTo: red
+title: MantisScore
+emoji: 📹
+colorFrom: green
+colorTo: yellow
 sdk: gradio
-sdk_version: 4.26.0
-app_file: app.py
+sdk_version: 4.24.0
+app_file: app_regression.py
 pinned: false
-license: mit
-tags:
-- arena
-- leaderboard
-short_description: Realtime Image/Video Gen AI Arena
+license: apache-2.0
+short_description: Multimodal Language Model
 ---
 
-## Installation
-
-- for cuda 11.8
-```bash
-conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia
-pip3 install -U xformers --index-url https://download.pytorch.org/whl/cu118
-pip install -r requirements.txt
-```
-- for cuda 12.1
-```bash
-conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia
-pip install -r requirements.txt
-```
-
-## Start Hugging Face UI
-```bash
-python app.py
-```
-
-## Start Log server
-```bash
-uvicorn serve.log_server:app --reload --port 22005 --host 0.0.0.0
-```
-
-## Update leaderboard
-```bash
-cd arena_elo && bash update_leaderboard.sh
-```
-
-Paper: arxiv.org/abs/2406.04485
\ No newline at end of file
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
\ No newline at end of file
diff --git a/app_generation.py b/app_generation.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a007e4c20750811a1df1c626ac3e1b870af9b0b
--- /dev/null
+++ b/app_generation.py
@@ -0,0 +1,197 @@
+import gradio as gr
+import spaces
+import os
+import time
+import json
+import numpy as np
+import av
+import torch
+from PIL import Image
+import functools
+from transformers import AutoProcessor, AutoConfig
+from models.idefics2 import Idefics2ForSequenceClassification, Idefics2ForConditionalGeneration
+from models.conversation import conv_templates
+from typing import List
+
+processor = AutoProcessor.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-refined-40k_4096_generation")
+model = Idefics2ForConditionalGeneration.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-refined-40k_4096_generation", torch_dtype=torch.bfloat16).eval()
+
+MAX_NUM_FRAMES = 24
+conv_template = conv_templates["idefics_2"]
+
+with open("./examples/all_subsets.json", 'r') as f:
+    examples = json.load(f)
+
+for item in examples:
+    video_id = item['images'][0].split("_")[0]
+    item['images'] = [os.path.join("./examples", video_id, x) for x in item['images']]
+    item['video'] = os.path.join("./examples", item['video'])
+    
+with open("./examples/hd.json", 'r') as f:
+    hd_examples = json.load(f)
+
+for item in hd_examples:
+    item['video'] = os.path.join("./examples", item['video'])
+
+examples = hd_examples + examples
+
+VIDEO_EVAL_PROMPT = """
+Suppose you are an expert in judging and evaluating the quality of AI-generated videos, 
+please watch the following frames of a given video and see the text prompt for generating the video, 
+then give scores from 5 different dimensions:
+(1) visual quality: the quality of the video in terms of clearness, resolution, brightness, and color
+(2) temporal consistency, the consistency of objects or humans in video
+(3) dynamic degree, the degree of dynamic changes
+(4) text-to-video alignment, the alignment between the text prompt and the video content
+(5) factual consistency, the consistency of the video content with the common-sense and factual knowledge
+
+For each dimension, output a number from [1,2,3,4], 
+in which '1' means 'Bad', '2' means 'Average', '3' means 'Good', 
+'4' means 'Real' or 'Perfect' (the video is like a real video)
+Here is an output example:
+visual quality: 4
+temporal consistency: 4
+dynamic degree: 3
+text-to-video alignment: 1
+factual consistency: 2
+
+For this video, the text prompt is "{text_prompt}",
+all the frames of video are as follows: 
+
+"""
+
+
+aspect_mapping= [
+    "visual quality",
+    "temporal consistency",
+    "dynamic degree",
+    "text-to-video alignment",
+    "factual consistency",
+]
+
+
+@spaces.GPU(duration=60)
+def score(prompt:str, images:List[Image.Image]):
+    if not prompt:
+        raise gr.Error("Please provide a prompt")
+    model.to("cuda")
+    if not images:
+        images = None
+    
+    flatten_images = []
+    for x in images:
+        if isinstance(x, list):
+            flatten_images.extend(x)
+        else:
+            flatten_images.append(x)
+    
+    messages = [{"role": "User", "content": [{"type": "text", "text": prompt}]}]
+    prompt = processor.apply_chat_template(messages, add_generation_prompt=True) 
+    print(prompt)
+    
+    flatten_images = [Image.open(x) if isinstance(x, str) else x for x in flatten_images]
+    inputs = processor(text=prompt, images=flatten_images, return_tensors="pt")
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    
+    outputs = model.generate(**inputs, max_new_tokens=1024)
+    generated_text = processor.decode(outputs[0, inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
+    return generated_text
+    
+def read_video_pyav(container, indices):
+    '''
+    Decode the video with PyAV decoder.
+
+    Args:
+        container (av.container.input.InputContainer): PyAV container.
+        indices (List[int]): List of frame indices to decode.
+
+    Returns:
+        np.ndarray: np array of decoded frames of shape (num_frames, height, width, 3).
+    '''
+    frames = []
+    container.seek(0)
+    start_index = indices[0]
+    end_index = indices[-1]
+    for i, frame in enumerate(container.decode(video=0)):
+        if i > end_index:
+            break
+        if i >= start_index and i in indices:
+            frames.append(frame)
+    return np.stack([x.to_ndarray(format="rgb24") for x in frames])
+
+def eval_video(prompt, video:str):
+    container = av.open(video)
+
+    # sample uniformly 8 frames from the video
+    total_frames = container.streams.video[0].frames
+    if total_frames > MAX_NUM_FRAMES:
+        indices = np.arange(0, total_frames, total_frames / MAX_NUM_FRAMES).astype(int)
+    else:
+        indices = np.arange(total_frames)
+    video_frames = read_video_pyav(container, indices)
+
+    frames = [Image.fromarray(x) for x in video_frames]
+    
+    eval_prompt = VIDEO_EVAL_PROMPT.format(text_prompt=prompt)
+    
+    num_image_token = eval_prompt.count("<image>")
+    if num_image_token < len(frames):
+        eval_prompt += "<image> " * (len(frames) - num_image_token)
+    
+    aspect_scores = score(eval_prompt, [frames])
+    return aspect_scores
+
+def build_demo():
+    with gr.Blocks() as demo:
+        gr.Markdown("""
+            ## Video Evaluation
+            upload a video along with a text prompt when generating the video, this model will evaluate the video's quality from 7 different dimensions.
+        """)
+        with gr.Row():
+            video = gr.Video(width=500, label="Video")
+            with gr.Column():
+                eval_prompt_template = gr.Textbox(VIDEO_EVAL_PROMPT.strip(' \n'), label="Evaluation Prompt Template", interactive=False, max_lines=26)
+                video_prompt = gr.Textbox(label="Text Prompt", lines=1)
+                with gr.Row():
+                    eval_button = gr.Button("Evaluate Video")
+                    clear_button = gr.ClearButton([video, video_prompt])
+                eval_result = gr.Textbox(label="Evaluation result", interactive=False, lines=7)
+                # eval_result = gr.Json(label="Evaluation result")
+                
+        
+        eval_button.click(
+            eval_video, [video_prompt, video], [eval_result]
+        )
+        
+        dummy_id = gr.Textbox("id", label="id", visible=False, min_width=50)
+        dummy_output = gr.Textbox("reference score", label="reference scores", visible=False, lines=7)
+        
+        gr.Examples(
+            examples=
+            [
+                [
+                    item['id'], 
+                    item['prompt'],
+                    item['video'],
+                    item['conversations'][1]['value']
+                ] for item in examples
+            ],
+            inputs=[dummy_id, video_prompt, video, dummy_output],
+        )        
+        
+#         gr.Markdown("""
+# ## Citation
+# ```
+# @article{jiang2024mantis,
+#   title={MANTIS: Interleaved Multi-Image Instruction Tuning},
+#   author={Jiang, Dongfu and He, Xuan and Zeng, Huaye and Wei, Con and Ku, Max and Liu, Qian and Chen, Wenhu},
+#   journal={arXiv preprint arXiv:2405.01483},
+#   year={2024}
+# }
+# ```""")
+    return demo    
+    
+
+if __name__ == "__main__":
+    demo = build_demo()
+    demo.launch(share=True)
\ No newline at end of file
diff --git a/app_high_res.py b/app_high_res.py
new file mode 100644
index 0000000000000000000000000000000000000000..99d0c8940e7f9b0942cdf0595f802cdcf90967b1
--- /dev/null
+++ b/app_high_res.py
@@ -0,0 +1,236 @@
+import gradio as gr
+import spaces
+import os
+import time
+import json
+import numpy as np
+import av
+import torch
+from PIL import Image
+import functools
+from transformers import AutoProcessor, Idefics2ForConditionalGeneration
+from models.conversation import conv_templates
+from typing import List
+
+
+processor = AutoProcessor.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-refined-40k_4096_generation")
+model = Idefics2ForConditionalGeneration.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-refined-40k_4096_generation", device_map="auto", torch_dtype=torch.bfloat16).eval()
+MAX_NUM_FRAMES = 24
+conv_template = conv_templates["idefics_2"]
+
+with open("./examples/all_subsets.json", 'r') as f:
+    examples = json.load(f)
+
+for item in examples:
+    video_id = item['images'][0].split("_")[0]
+    item['images'] = [os.path.join("./examples", video_id, x) for x in item['images']]
+    item['video'] = os.path.join("./examples", item['video'])
+    
+with open("./examples/hd.json", 'r') as f:
+    hd_examples = json.load(f)
+
+for item in hd_examples:
+    item['video'] = os.path.join("./examples", item['video'])
+
+examples = hd_examples + examples
+
+VIDEO_EVAL_PROMPT = """
+Suppose you are an expert in judging and evaluating the quality of AI-generated videos, 
+please watch the following frames of a given video and see the text prompt for generating the video, 
+then give scores from 5 different dimensions:
+(1) visual quality: the quality of the video in terms of clearness, resolution, brightness, and color
+(2) temporal consistency, the consistency of objects or humans in video
+(3) dynamic degree, the degree of dynamic changes
+(4) text-to-video alignment, the alignment between the text prompt and the video content
+(5) factual consistency, the consistency of the video content with the common-sense and factual knowledge
+
+For each dimension, output a number from [1,2,3,4], 
+in which '1' means 'Bad', '2' means 'Average', '3' means 'Good', 
+'4' means 'Real' or 'Perfect' (the video is like a real video)
+Here is an output example:
+visual quality: 4
+temporal consistency: 4
+dynamic degree: 3
+text-to-video alignment: 1
+factual consistency: 2
+
+For this video, the text prompt is "{text_prompt}",
+all the frames of video are as follows: 
+
+"""
+@spaces.GPU(duration=60)
+def generate(text:str, images:List[Image.Image], history: List[dict], **kwargs):
+    model.to("cuda")
+    if not images:
+        images = None
+    
+    user_role = conv_template.roles[0]
+    assistant_role = conv_template.roles[1]
+    
+    idefics_2_message = []
+    cur_img_idx = 0
+    cur_vid_idx = 0
+    all_videos = [x for x in images if isinstance(x, list)]
+    flatten_images = []
+    for x in images:
+        if isinstance(x, list):
+            flatten_images.extend(x)
+        else:
+            flatten_images.append(x)
+    
+    print(history)
+    for i, message in enumerate(history):
+        if message["role"] == user_role:
+            idefics_2_message.append({
+                "role": user_role,
+                "content": []
+            })
+            message_text = message["text"]
+            num_video_tokens_in_text = message_text.count("<video>")
+            if num_video_tokens_in_text > 0:
+                for _ in range(num_video_tokens_in_text):
+                    message_text = message_text.replace("<video>", "<image> " * len(all_videos[cur_vid_idx]), 1)
+                    cur_vid_idx += 1
+            num_image_tokens_in_text = message_text.count("<image>")
+            if num_image_tokens_in_text > 0:
+                sub_texts = [x.strip() for x in message_text.split("<image>")]
+                if sub_texts[0]:
+                    idefics_2_message[-1]["content"].append({"type": "text", "text": sub_texts[0]})
+                for sub_text in sub_texts[1:]:
+                    idefics_2_message[-1]["content"].append({"type": "image"})
+                    if sub_text:
+                        idefics_2_message.append({
+                            "role": user_role,
+                            "content": [{"type": "text", "text": sub_text}]
+                        })
+            else:
+                idefics_2_message[-1]["content"].append({"type": "text", "text": message_text})
+        elif message["role"] == assistant_role:
+            if i == len(history) - 1 and not message["text"]:
+                break
+            idefics_2_message.append({
+                "role": assistant_role,
+                "content": [{"type": "text", "text": message["text"]}]
+            })
+    if text:
+        assert idefics_2_message[-1]["role"] == assistant_role and not idefics_2_message[-1]["content"], "Internal error"
+        idefics_2_message.append({
+            "role": user_role,
+            "content": [{"type": "text", "text": text}]
+        })
+    
+    print(idefics_2_message)
+    prompt = processor.apply_chat_template(idefics_2_message, add_generation_prompt=True) 
+    
+    images = [Image.open(x) if isinstance(x, str) else x for x in flatten_images]
+    inputs = processor(text=prompt, images=images, return_tensors="pt")
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    outputs = model.generate(**inputs, max_new_tokens=1024)
+    generated_text = processor.decode(outputs[0, inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
+    return generated_text
+
+    
+def read_video_pyav(container, indices):
+    '''
+    Decode the video with PyAV decoder.
+
+    Args:
+        container (av.container.input.InputContainer): PyAV container.
+        indices (List[int]): List of frame indices to decode.
+
+    Returns:
+        np.ndarray: np array of decoded frames of shape (num_frames, height, width, 3).
+    '''
+    frames = []
+    container.seek(0)
+    start_index = indices[0]
+    end_index = indices[-1]
+    for i, frame in enumerate(container.decode(video=0)):
+        if i > end_index:
+            break
+        if i >= start_index and i in indices:
+            frames.append(frame)
+    return np.stack([x.to_ndarray(format="rgb24") for x in frames])
+
+def eval_video(prompt, video:str):
+    container = av.open(video)
+
+    # sample uniformly 8 frames from the video
+    total_frames = container.streams.video[0].frames
+    if total_frames > MAX_NUM_FRAMES:
+        indices = np.arange(0, total_frames, total_frames / MAX_NUM_FRAMES).astype(int)
+    else:
+        indices = np.arange(total_frames)
+    video_frames = read_video_pyav(container, indices)
+
+    frames = [Image.fromarray(x) for x in video_frames]
+    
+    eval_prompt = VIDEO_EVAL_PROMPT.format(text_prompt=prompt)
+    eval_prompt += "<video>"
+    user_role = conv_template.roles[0]
+    assistant_role = conv_template.roles[1]
+    chat_messages = [
+        {
+            "role": user_role,
+            "text": eval_prompt
+        },
+        {
+            "role": assistant_role,
+            "text": ""
+        }
+    ]
+    response = generate(None, [frames], chat_messages)
+    return response
+
+def build_demo():
+    with gr.Blocks() as demo:
+        gr.Markdown("""
+            ## Video Evaluation
+            upload a video along with a text prompt when generating the video, this model will evaluate the video's quality from 7 different dimensions.
+        """)
+        with gr.Row():
+            video = gr.Video(width=500, label="Video")
+            with gr.Column():
+                eval_prompt_template = gr.Textbox(VIDEO_EVAL_PROMPT.strip(' \n'), label="Evaluation Prompt Template", interactive=False, max_lines=26)
+                video_prompt = gr.Textbox(label="Text Prompt", lines=1)
+                with gr.Row():
+                    eval_button = gr.Button("Evaluate Video")
+                    clear_button = gr.ClearButton([video, video_prompt])
+                eval_result = gr.Textbox(label="Evaluation result", interactive=False, lines=7)
+        
+        eval_button.click(
+            eval_video, [video_prompt, video], [eval_result]
+        )
+        
+        dummy_id = gr.Textbox("id", label="id", visible=False, min_width=50)
+        dummy_output = gr.Textbox("reference score", label="reference scores", visible=False, lines=7)
+        
+        gr.Examples(
+            examples=
+            [
+                [
+                    item['id'], 
+                    item['prompt'],
+                    item['video'],
+                    item['conversations'][1]['value']
+                ] for item in examples
+            ],
+            inputs=[dummy_id, video_prompt, video, dummy_output],
+        )        
+        
+#         gr.Markdown("""
+# ## Citation
+# ```
+# @article{jiang2024mantis,
+#   title={MANTIS: Interleaved Multi-Image Instruction Tuning},
+#   author={Jiang, Dongfu and He, Xuan and Zeng, Huaye and Wei, Con and Ku, Max and Liu, Qian and Chen, Wenhu},
+#   journal={arXiv preprint arXiv:2405.01483},
+#   year={2024}
+# }
+# ```""")
+    return demo    
+    
+
+if __name__ == "__main__":
+    demo = build_demo()
+    demo.launch(share=True)
\ No newline at end of file
diff --git a/app_regression.py b/app_regression.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce55f3bb0452b4e8dea46203c2bd3370d7af9b21
--- /dev/null
+++ b/app_regression.py
@@ -0,0 +1,223 @@
+import gradio as gr
+import spaces
+import os
+import time
+import json
+import numpy as np
+import av
+import torch
+from PIL import Image
+import functools
+from transformers import AutoProcessor, AutoConfig
+from models.idefics2 import Idefics2ForSequenceClassification
+from models.conversation import conv_templates
+from typing import List
+
+
+processor = AutoProcessor.from_pretrained("TIGER-Lab/VideoScore")
+model = Idefics2ForSequenceClassification.from_pretrained("TIGER-Lab/VideoScore", torch_dtype=torch.bfloat16).eval()
+
+MAX_NUM_FRAMES = 24
+conv_template = conv_templates["idefics_2"]
+
+with open("./examples/all_subsets.json", 'r') as f:
+    examples = json.load(f)
+
+for item in examples:
+    video_id = item['images'][0].split("_")[0]
+    item['images'] = [os.path.join("./examples", video_id, x) for x in item['images']]
+    item['video'] = os.path.join("./examples", item['video'])
+    
+with open("./examples/hd.json", 'r') as f:
+    hd_examples = json.load(f)
+
+for item in hd_examples:
+    item['video'] = os.path.join("./examples", item['video'])
+
+examples = hd_examples + examples
+
+VIDEO_EVAL_PROMPT = """
+Suppose you are an expert in judging and evaluating the quality of AI-generated videos, 
+please watch the following frames of a given video and see the text prompt for generating the video, 
+then give scores from 5 different dimensions:
+(1) visual quality: the quality of the video in terms of clearness, resolution, brightness, and color
+(2) temporal consistency, the consistency of objects or humans in video
+(3) dynamic degree, the degree of dynamic changes
+(4) text-to-video alignment, the alignment between the text prompt and the video content
+(5) factual consistency, the consistency of the video content with the common-sense and factual knowledge
+
+For each dimension, output a number from [1,2,3,4], 
+in which '1' means 'Bad', '2' means 'Average', '3' means 'Good', 
+'4' means 'Real' or 'Perfect' (the video is like a real video)
+Here is an output example:
+visual quality: 4
+temporal consistency: 4
+dynamic degree: 3
+text-to-video alignment: 1
+factual consistency: 2
+
+For this video, the text prompt is "{text_prompt}",
+all the frames of video are as follows: 
+
+"""
+
+
+
+space_description="""\
+[📃Paper](https://arxiv.org/abs/2406.15252) | [🌐Website](https://tiger-ai-lab.github.io/VideoScore/) | [💻Github](https://github.com/TIGER-AI-Lab/VideoScore) | [🛢️Datasets](https://huggingface.co/datasets/TIGER-Lab/VideoFeedback) | [🤗Model](https://huggingface.co/TIGER-Lab/VideoScore) | [🤗Demo](https://huggingface.co/spaces/TIGER-Lab/VideoScore)
+
+- VideoScore is a video quality evaluation model, taking [Mantis-8B-Idefics2](https://huggingface.co/TIGER-Lab/Mantis-8B-Idefics2) as base-model
+and trained on [VideoFeedback](https://huggingface.co/datasets/TIGER-Lab/VideoFeedback),
+a large video evaluation dataset with multi-aspect human scores. 
+
+- VideoScore can reach 75+ Spearman correlation with humans on VideoEval-test, surpassing all the MLLM-prompting methods and feature-based metrics. 
+
+- VideoScore also beat the best baselines on other three benchmarks EvalCrafter, GenAI-Bench and VBench, showing high alignment with human evaluations.
+"""
+
+
+aspect_mapping= [
+    "visual quality",
+    "temporal consistency",
+    "dynamic degree",
+    "text-to-video alignment",
+    "factual consistency",
+]
+
+
+@spaces.GPU(duration=60)
+def score(prompt:str, images:List[Image.Image]):
+    if not prompt:
+        raise gr.Error("Please provide a prompt")
+    model.to("cuda")
+    if not images:
+        images = None
+    
+    flatten_images = []
+    for x in images:
+        if isinstance(x, list):
+            flatten_images.extend(x)
+        else:
+            flatten_images.append(x)
+    
+    flatten_images = [Image.open(x) if isinstance(x, str) else x for x in flatten_images]
+    inputs = processor(text=prompt, images=flatten_images, return_tensors="pt")
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    with torch.no_grad():
+        outputs = model(**inputs)
+    
+    logits = outputs.logits
+    num_aspects = logits.shape[-1]
+    aspects = [aspect_mapping[i] for i in range(num_aspects)]
+    
+    aspect_scores = {}
+    for i, aspect in enumerate(aspects):
+        aspect_scores[aspect] = round(logits[0, i].item(), 2)
+    return aspect_scores
+    
+    
+def read_video_pyav(container, indices):
+    '''
+    Decode the video with PyAV decoder.
+
+    Args:
+        container (av.container.input.InputContainer): PyAV container.
+        indices (List[int]): List of frame indices to decode.
+
+    Returns:
+        np.ndarray: np array of decoded frames of shape (num_frames, height, width, 3).
+    '''
+    frames = []
+    container.seek(0)
+    start_index = indices[0]
+    end_index = indices[-1]
+    for i, frame in enumerate(container.decode(video=0)):
+        if i > end_index:
+            break
+        if i >= start_index and i in indices:
+            frames.append(frame)
+    return np.stack([x.to_ndarray(format="rgb24") for x in frames])
+
+def eval_video(prompt, video:str):
+    container = av.open(video)
+
+    # sample uniformly 8 frames from the video
+    total_frames = container.streams.video[0].frames
+    if total_frames > MAX_NUM_FRAMES:
+        indices = np.arange(0, total_frames, total_frames / MAX_NUM_FRAMES).astype(int)
+    else:
+        indices = np.arange(total_frames)
+    video_frames = read_video_pyav(container, indices)
+
+    frames = [Image.fromarray(x) for x in video_frames]
+    
+    eval_prompt = VIDEO_EVAL_PROMPT.format(text_prompt=prompt)
+    
+    num_image_token = eval_prompt.count("<image>")
+    if num_image_token < len(frames):
+        eval_prompt += "<image> " * (len(frames) - num_image_token)
+    
+    aspect_scores = score(eval_prompt, [frames])
+    return aspect_scores
+
+def build_demo():
+    with gr.Blocks() as demo:
+
+        gr.Markdown("## VideoScore: Building Automatic Metrics to Simulate Fine-grained Human Feedback for Video Generation")
+        with gr.Row():
+            gr.Markdown(space_description)
+            gr.Image("https://tiger-ai-lab.github.io/VideoScore/static/images/teaser.png", label="Teaser")
+        
+        gr.Markdown("### Try VideoScore (Regression) with your own text prompt and videos.")
+        with gr.Row():
+            video = gr.Video(width=500, label="Video")
+            with gr.Column():
+                eval_prompt_template = gr.Textbox(VIDEO_EVAL_PROMPT.strip(' \n'), label="Evaluation Prompt Template", interactive=False, max_lines=26)
+                video_prompt = gr.Textbox(label="Text Prompt", lines=1)
+                with gr.Row():
+                    eval_button = gr.Button("Evaluate Video")
+                    clear_button = gr.ClearButton([video, video_prompt])
+                # eval_result = gr.Textbox(label="Evaluation result", interactive=False, lines=7)
+                eval_result = gr.Json(label="Evaluation result")
+                
+        
+        eval_button.click(
+            eval_video, [video_prompt, video], [eval_result]
+        )
+        
+        dummy_id = gr.Textbox("id", label="id", visible=False, min_width=50)
+        # dummy_output = gr.Textbox("reference score", label="reference scores", visible=False, lines=7)
+        
+        gr.Examples(
+            examples=
+            [
+                [
+                    # item['id'], 
+                    item['prompt'],
+                    item['video'],
+                    # item['conversations'][1]['value']
+                ] for item in examples if item['prompt']
+            ],
+            inputs=[video_prompt, video],
+            # inputs=[dummy_id, video_prompt, video, dummy_output],
+            
+        )        
+        
+        gr.Markdown("""
+## Citation
+```
+@article{he2024videoscore,
+  title = {VideoScore: Building Automatic Metrics to Simulate Fine-grained Human Feedback for Video Generation},
+  author = {He, Xuan and Jiang, Dongfu and Zhang, Ge and Ku, Max and Soni, Achint and Siu, Sherman and Chen, Haonan and Chandra, Abhranil and Jiang, Ziyan and Arulraj, Aaran and Wang, Kai and Do, Quy Duc and Ni, Yuansheng and Lyu, Bohan and Narsupalli, Yaswanth and Fan, Rongqi and Lyu, Zhiheng and Lin, Yuchen and Chen, Wenhu},
+  journal = {ArXiv},
+  year = {2024},
+  volume={abs/2406.15252},
+  url = {https://arxiv.org/abs/2406.15252},
+}
+```""")
+    return demo    
+    
+
+if __name__ == "__main__":
+    demo = build_demo()
+    demo.launch(share=True)
\ No newline at end of file
diff --git a/barchart.jpeg b/barchart.jpeg
new file mode 100644
index 0000000000000000000000000000000000000000..f6f6f1eca7fe043fc892ba5e59b35dc92031f1b9
Binary files /dev/null and b/barchart.jpeg differ
diff --git a/barchart_single_image_vqa.jpeg b/barchart_single_image_vqa.jpeg
new file mode 100644
index 0000000000000000000000000000000000000000..99bc1bab19bfdbf2ebcacc45e0a1663ccc0eefdc
Binary files /dev/null and b/barchart_single_image_vqa.jpeg differ
diff --git a/examples/1006309.mp4 b/examples/1006309.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..751197f27e40fdd944b591c63cb94ff82ca75a76
--- /dev/null
+++ b/examples/1006309.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14852166b4443cb90576de69b439b92b88770fb29dc30fb5fafcb8a497d79ac5
+size 101215
diff --git a/examples/1006309/1006309_00.jpg b/examples/1006309/1006309_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..046eea34f00667087f449e856d4248f827d91aa8
Binary files /dev/null and b/examples/1006309/1006309_00.jpg differ
diff --git a/examples/1006309/1006309_01.jpg b/examples/1006309/1006309_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..650b8edd6ea7bb9b2e249ccf0396f1f196253402
Binary files /dev/null and b/examples/1006309/1006309_01.jpg differ
diff --git a/examples/1006309/1006309_02.jpg b/examples/1006309/1006309_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c7355179cd90524bd0a7258806df5f0e112b06c7
Binary files /dev/null and b/examples/1006309/1006309_02.jpg differ
diff --git a/examples/1006309/1006309_03.jpg b/examples/1006309/1006309_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a273834a1204304cb37494511d26596d6bd7dcec
Binary files /dev/null and b/examples/1006309/1006309_03.jpg differ
diff --git a/examples/1006309/1006309_04.jpg b/examples/1006309/1006309_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0c4a4f96f1b7c808ba0d34370c0cc7502bd3f2df
Binary files /dev/null and b/examples/1006309/1006309_04.jpg differ
diff --git a/examples/1006309/1006309_05.jpg b/examples/1006309/1006309_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..80398216441490a312e5b19bd153f8a70d1bb9f5
Binary files /dev/null and b/examples/1006309/1006309_05.jpg differ
diff --git a/examples/1006309/1006309_06.jpg b/examples/1006309/1006309_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ee11fae966adeade658d49542376f2ca756ff5cb
Binary files /dev/null and b/examples/1006309/1006309_06.jpg differ
diff --git a/examples/1006309/1006309_07.jpg b/examples/1006309/1006309_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..16493ffc43a5ef4c83b3e281f92de8993f5a4d7a
Binary files /dev/null and b/examples/1006309/1006309_07.jpg differ
diff --git a/examples/1006309/1006309_08.jpg b/examples/1006309/1006309_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0fc6d446c475ae08deb313a01ef21f56cd092104
Binary files /dev/null and b/examples/1006309/1006309_08.jpg differ
diff --git a/examples/1006309/1006309_09.jpg b/examples/1006309/1006309_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f73bce1eef9300d1b87fc5dce47557667633f64f
Binary files /dev/null and b/examples/1006309/1006309_09.jpg differ
diff --git a/examples/1006309/1006309_10.jpg b/examples/1006309/1006309_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..081cc4750501114768714378bd465d720e1bb383
Binary files /dev/null and b/examples/1006309/1006309_10.jpg differ
diff --git a/examples/1006309/1006309_11.jpg b/examples/1006309/1006309_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..68fa8b19520fa95030855c2062e8f1a4d018d87c
Binary files /dev/null and b/examples/1006309/1006309_11.jpg differ
diff --git a/examples/1006309/1006309_12.jpg b/examples/1006309/1006309_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..42d0469927294f5d3b4c438c64fc008bef9e216d
Binary files /dev/null and b/examples/1006309/1006309_12.jpg differ
diff --git a/examples/1006309/1006309_13.jpg b/examples/1006309/1006309_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e32d0ac6ab10c0f8d3148ba9c1094136d72c5193
Binary files /dev/null and b/examples/1006309/1006309_13.jpg differ
diff --git a/examples/1006309/1006309_14.jpg b/examples/1006309/1006309_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..cef15de9b4792773e43f473def07b2e4711f8e1e
Binary files /dev/null and b/examples/1006309/1006309_14.jpg differ
diff --git a/examples/1006309/1006309_15.jpg b/examples/1006309/1006309_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5adf6a3215fc3a6124a7440abc061dea2ee394e6
Binary files /dev/null and b/examples/1006309/1006309_15.jpg differ
diff --git a/examples/3005033.mp4 b/examples/3005033.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..e8a97e398e33b8eeacb417ab4ebc62329ae31103
--- /dev/null
+++ b/examples/3005033.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:090465f03d91787ae683591248d5aefd48b4412da8f09a950749bfab72a5c394
+size 42856
diff --git a/examples/3005033/3005033_00.jpg b/examples/3005033/3005033_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7a2c96b4bc341268dbe028738d5750e4bb8a486a
Binary files /dev/null and b/examples/3005033/3005033_00.jpg differ
diff --git a/examples/3005033/3005033_01.jpg b/examples/3005033/3005033_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..10cc06b3ceda38fa8ead593e1cbf3992040801e0
Binary files /dev/null and b/examples/3005033/3005033_01.jpg differ
diff --git a/examples/3005033/3005033_02.jpg b/examples/3005033/3005033_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a3a8d9924df5671c83aa207fd588212527bfc225
Binary files /dev/null and b/examples/3005033/3005033_02.jpg differ
diff --git a/examples/3005033/3005033_03.jpg b/examples/3005033/3005033_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..965009d05645c40cde9ca0eef7d6fb4649ba0fe7
Binary files /dev/null and b/examples/3005033/3005033_03.jpg differ
diff --git a/examples/3005033/3005033_04.jpg b/examples/3005033/3005033_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..77667441972d9976b591a64edbd2a4abb86bf2f5
Binary files /dev/null and b/examples/3005033/3005033_04.jpg differ
diff --git a/examples/3005033/3005033_05.jpg b/examples/3005033/3005033_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..58fd270455212096e8a7e10feae3b60664ecb200
Binary files /dev/null and b/examples/3005033/3005033_05.jpg differ
diff --git a/examples/3005033/3005033_06.jpg b/examples/3005033/3005033_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6b651d10d0e15728637f5f6d38a2e666a24622b0
Binary files /dev/null and b/examples/3005033/3005033_06.jpg differ
diff --git a/examples/3005033/3005033_07.jpg b/examples/3005033/3005033_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5403e27d757f2ac2874e041dea3374f587dc9084
Binary files /dev/null and b/examples/3005033/3005033_07.jpg differ
diff --git a/examples/3005033/3005033_08.jpg b/examples/3005033/3005033_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7b65d5474a78ee385811a2ed418657ab9b9f05d7
Binary files /dev/null and b/examples/3005033/3005033_08.jpg differ
diff --git a/examples/3005033/3005033_09.jpg b/examples/3005033/3005033_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..241e3c880dde58c2bb5046952e788cb292242fe5
Binary files /dev/null and b/examples/3005033/3005033_09.jpg differ
diff --git a/examples/3005033/3005033_10.jpg b/examples/3005033/3005033_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..53fcd9361534063167117b9170f394e95c1f85bd
Binary files /dev/null and b/examples/3005033/3005033_10.jpg differ
diff --git a/examples/3005033/3005033_11.jpg b/examples/3005033/3005033_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..be539a7c062082c567f15d282d1b652428f8f731
Binary files /dev/null and b/examples/3005033/3005033_11.jpg differ
diff --git a/examples/3005033/3005033_12.jpg b/examples/3005033/3005033_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..302a4e6ea89c14af73b676746d615568314013dd
Binary files /dev/null and b/examples/3005033/3005033_12.jpg differ
diff --git a/examples/3005033/3005033_13.jpg b/examples/3005033/3005033_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..91e2af8b8c7a05350a5560e32d2266a277e47ebc
Binary files /dev/null and b/examples/3005033/3005033_13.jpg differ
diff --git a/examples/3005033/3005033_14.jpg b/examples/3005033/3005033_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e7a0b95ce28f1f8b742d01fd0e66725ba19b0920
Binary files /dev/null and b/examples/3005033/3005033_14.jpg differ
diff --git a/examples/3005033/3005033_15.jpg b/examples/3005033/3005033_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..8cd0a11c8033dfebeff81749f7e3b11cc5afde00
Binary files /dev/null and b/examples/3005033/3005033_15.jpg differ
diff --git a/examples/7004180.mp4 b/examples/7004180.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..d98dd001bfe93fa60fc4acebddde802800d98e00
--- /dev/null
+++ b/examples/7004180.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d255b025443ac89854e9529341767aea3039c1b5df2d6d37cf6fc8d01410b67
+size 19932
diff --git a/examples/7004180/7004180_00.jpg b/examples/7004180/7004180_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ebef4f47df7911149107e1a6700ad4757238b596
Binary files /dev/null and b/examples/7004180/7004180_00.jpg differ
diff --git a/examples/7004180/7004180_01.jpg b/examples/7004180/7004180_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b03f5e7bf1256c7c4fdb4be41eb6ba6e978e018c
Binary files /dev/null and b/examples/7004180/7004180_01.jpg differ
diff --git a/examples/7004180/7004180_02.jpg b/examples/7004180/7004180_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..55bad5b7537ea834b1e78979119fd72c742658cc
Binary files /dev/null and b/examples/7004180/7004180_02.jpg differ
diff --git a/examples/7004180/7004180_03.jpg b/examples/7004180/7004180_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e4e6dfd85b5cb6036f5af9c587a1e07009442949
Binary files /dev/null and b/examples/7004180/7004180_03.jpg differ
diff --git a/examples/7004180/7004180_04.jpg b/examples/7004180/7004180_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0009b543f29ba77e19864085b05598baf197a84d
Binary files /dev/null and b/examples/7004180/7004180_04.jpg differ
diff --git a/examples/7004180/7004180_05.jpg b/examples/7004180/7004180_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..03f5804368991d92a3494cc17aaef7f4049e2a8f
Binary files /dev/null and b/examples/7004180/7004180_05.jpg differ
diff --git a/examples/7004180/7004180_06.jpg b/examples/7004180/7004180_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ef008477a0e9e696138c8ab09282890527aa2071
Binary files /dev/null and b/examples/7004180/7004180_06.jpg differ
diff --git a/examples/7004180/7004180_07.jpg b/examples/7004180/7004180_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..8a4eb2c55618d65a88f913407e4da8bd9b630489
Binary files /dev/null and b/examples/7004180/7004180_07.jpg differ
diff --git a/examples/a400480.mp4 b/examples/a400480.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..3fa54f679aec95f6e26ffce7a884eefde6632b83
--- /dev/null
+++ b/examples/a400480.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fef2cda048fbb7c84c226cb1358e69829657b21fee9ff3c7535cd56e377f7a0
+size 21995
diff --git a/examples/a400480/a400480_00.jpg b/examples/a400480/a400480_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..cabf1ff8fd3d7dba34d8545bb9a1493d539fd62a
Binary files /dev/null and b/examples/a400480/a400480_00.jpg differ
diff --git a/examples/a400480/a400480_01.jpg b/examples/a400480/a400480_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..25db21ace634af5bac339f373ea24e8406780b04
Binary files /dev/null and b/examples/a400480/a400480_01.jpg differ
diff --git a/examples/a400480/a400480_02.jpg b/examples/a400480/a400480_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..06790ad75fd577faf7639fe0bcda74e92ea802cf
Binary files /dev/null and b/examples/a400480/a400480_02.jpg differ
diff --git a/examples/a400480/a400480_03.jpg b/examples/a400480/a400480_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..82a51f63bb4e6f2527cdfb0e00b306e6b1d0ccd5
Binary files /dev/null and b/examples/a400480/a400480_03.jpg differ
diff --git a/examples/a400480/a400480_04.jpg b/examples/a400480/a400480_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e3e2c4bac4b8859d2de0331a170511741676de91
Binary files /dev/null and b/examples/a400480/a400480_04.jpg differ
diff --git a/examples/a400480/a400480_05.jpg b/examples/a400480/a400480_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d6fa6ddcb9cced8d426eb2921d73c5791aade515
Binary files /dev/null and b/examples/a400480/a400480_05.jpg differ
diff --git a/examples/a400480/a400480_06.jpg b/examples/a400480/a400480_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..64394555871b095da40daf8196bb2c3daf789b0e
Binary files /dev/null and b/examples/a400480/a400480_06.jpg differ
diff --git a/examples/a400480/a400480_07.jpg b/examples/a400480/a400480_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..58b297e4e057e424f2effdc85149ebeaeaed9b1b
Binary files /dev/null and b/examples/a400480/a400480_07.jpg differ
diff --git a/examples/a400480/a400480_08.jpg b/examples/a400480/a400480_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..114c97cdc69b1e20de1a134ff1c790191b21136f
Binary files /dev/null and b/examples/a400480/a400480_08.jpg differ
diff --git a/examples/a400480/a400480_09.jpg b/examples/a400480/a400480_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..3edb196a5f1f6841b298fa8b725118e0c028d5e5
Binary files /dev/null and b/examples/a400480/a400480_09.jpg differ
diff --git a/examples/a400480/a400480_10.jpg b/examples/a400480/a400480_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a3b19dd241757d102afdfb2964231ed443552b0b
Binary files /dev/null and b/examples/a400480/a400480_10.jpg differ
diff --git a/examples/a400480/a400480_11.jpg b/examples/a400480/a400480_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1d7bced715893e05ed9cc669afdaff3d15e66b2c
Binary files /dev/null and b/examples/a400480/a400480_11.jpg differ
diff --git a/examples/a400480/a400480_12.jpg b/examples/a400480/a400480_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7fe973668a54aa95b9a971742be2b4ba844ce059
Binary files /dev/null and b/examples/a400480/a400480_12.jpg differ
diff --git a/examples/a400480/a400480_13.jpg b/examples/a400480/a400480_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..836f491065c8397c3c758e9085452b13fc12a8f7
Binary files /dev/null and b/examples/a400480/a400480_13.jpg differ
diff --git a/examples/a400480/a400480_14.jpg b/examples/a400480/a400480_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5c967e4f772878943a0c39243ec33f0216ed4862
Binary files /dev/null and b/examples/a400480/a400480_14.jpg differ
diff --git a/examples/a400480/a400480_15.jpg b/examples/a400480/a400480_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..af3ba10674c0d059790ba4ab6879b807ecc1e50c
Binary files /dev/null and b/examples/a400480/a400480_15.jpg differ
diff --git a/examples/a500010.mp4 b/examples/a500010.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..09fdd6806530830f30f58a25a67ade33bb0aa997
--- /dev/null
+++ b/examples/a500010.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cb9d8e99e4c927d1c0c781d8ee705cef12e1017e2f5ef91617dc19ca3212df7
+size 46734
diff --git a/examples/a500010/a500010_00.jpg b/examples/a500010/a500010_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..47a64827a1f9344f31ec66da035cea19ce0ad78c
Binary files /dev/null and b/examples/a500010/a500010_00.jpg differ
diff --git a/examples/a500010/a500010_01.jpg b/examples/a500010/a500010_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f19f020f4e9f75fb642c656ce4546b5180e5c444
Binary files /dev/null and b/examples/a500010/a500010_01.jpg differ
diff --git a/examples/a500010/a500010_02.jpg b/examples/a500010/a500010_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..95c9881fd9528c895ed0709302ad96234e26a60d
Binary files /dev/null and b/examples/a500010/a500010_02.jpg differ
diff --git a/examples/a500010/a500010_03.jpg b/examples/a500010/a500010_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1aba9b874dc009fc83a178e70f323da35eaff893
Binary files /dev/null and b/examples/a500010/a500010_03.jpg differ
diff --git a/examples/a500010/a500010_04.jpg b/examples/a500010/a500010_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..22e448853304c613eefd09b5c660b9c8eb10eb2f
Binary files /dev/null and b/examples/a500010/a500010_04.jpg differ
diff --git a/examples/a500010/a500010_05.jpg b/examples/a500010/a500010_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..90021cf256993839d150759a026530ea5fb901e2
Binary files /dev/null and b/examples/a500010/a500010_05.jpg differ
diff --git a/examples/a500010/a500010_06.jpg b/examples/a500010/a500010_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..cd49c253dafb695cbc358d9032141dfab98b9fa4
Binary files /dev/null and b/examples/a500010/a500010_06.jpg differ
diff --git a/examples/a500010/a500010_07.jpg b/examples/a500010/a500010_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..50c629ba0820731928a7146dadbca27c6ca47419
Binary files /dev/null and b/examples/a500010/a500010_07.jpg differ
diff --git a/examples/a500010/a500010_08.jpg b/examples/a500010/a500010_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..2785774dd12e12b57900284d63749239c611f006
Binary files /dev/null and b/examples/a500010/a500010_08.jpg differ
diff --git a/examples/a500010/a500010_09.jpg b/examples/a500010/a500010_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..655c0351538f0042cb7b44995a381f58f146510d
Binary files /dev/null and b/examples/a500010/a500010_09.jpg differ
diff --git a/examples/a500010/a500010_10.jpg b/examples/a500010/a500010_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..69aa6fbf78d8fe9f361557b3876932f67ed44b31
Binary files /dev/null and b/examples/a500010/a500010_10.jpg differ
diff --git a/examples/a500010/a500010_11.jpg b/examples/a500010/a500010_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..630c10817062f30e78ddc0c79834a133d27e9123
Binary files /dev/null and b/examples/a500010/a500010_11.jpg differ
diff --git a/examples/a500010/a500010_12.jpg b/examples/a500010/a500010_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..994b6d2ea4a753fedcadf3972009ffabc940d9ad
Binary files /dev/null and b/examples/a500010/a500010_12.jpg differ
diff --git a/examples/a500010/a500010_13.jpg b/examples/a500010/a500010_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..48fe46e9eadcb4d90acb6da77896cdb495530ec4
Binary files /dev/null and b/examples/a500010/a500010_13.jpg differ
diff --git a/examples/a500010/a500010_14.jpg b/examples/a500010/a500010_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..003fcc256ee3aebaaf2f684f0adbcda49eae2616
Binary files /dev/null and b/examples/a500010/a500010_14.jpg differ
diff --git a/examples/a500010/a500010_15.jpg b/examples/a500010/a500010_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..317777f0e76cc74b58932d888227d665aeb6aa21
Binary files /dev/null and b/examples/a500010/a500010_15.jpg differ
diff --git a/examples/a500251.mp4 b/examples/a500251.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..18f0aed22765a81017488eb3fbb52a9f513213f9
--- /dev/null
+++ b/examples/a500251.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f72324298c805d3f8a8041b4b0bb4205180a6162a66a3103d90d5afd3d5f2e32
+size 79479
diff --git a/examples/a500251/a500251_00.jpg b/examples/a500251/a500251_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..93734914f3516e072c7d2f0249fa036dfe5b1555
Binary files /dev/null and b/examples/a500251/a500251_00.jpg differ
diff --git a/examples/a500251/a500251_01.jpg b/examples/a500251/a500251_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d8856e2df06477d4b2e5831f12ae513bfc3621bb
Binary files /dev/null and b/examples/a500251/a500251_01.jpg differ
diff --git a/examples/a500251/a500251_02.jpg b/examples/a500251/a500251_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..23aca8e214859d7766d89115a2ad6e8114c54a14
Binary files /dev/null and b/examples/a500251/a500251_02.jpg differ
diff --git a/examples/a500251/a500251_03.jpg b/examples/a500251/a500251_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..286f56dc94c29184df6a775f130d62aea016f7cc
Binary files /dev/null and b/examples/a500251/a500251_03.jpg differ
diff --git a/examples/a500251/a500251_04.jpg b/examples/a500251/a500251_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..fc07f437bdd2f39bb4bc27255d3133d0eab12031
Binary files /dev/null and b/examples/a500251/a500251_04.jpg differ
diff --git a/examples/a500251/a500251_05.jpg b/examples/a500251/a500251_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..de44637c6a85157363f38c3bc5f14aeadff23e56
Binary files /dev/null and b/examples/a500251/a500251_05.jpg differ
diff --git a/examples/a500251/a500251_06.jpg b/examples/a500251/a500251_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f09048d9349a512cf853a7c1648d89642db793bf
Binary files /dev/null and b/examples/a500251/a500251_06.jpg differ
diff --git a/examples/a500251/a500251_07.jpg b/examples/a500251/a500251_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c9ca802df7a9580bb495a318bf2d597fe0e7ab7b
Binary files /dev/null and b/examples/a500251/a500251_07.jpg differ
diff --git a/examples/a500251/a500251_08.jpg b/examples/a500251/a500251_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7baa4fffbdf833051d44df4d44ee4ea25151c0b1
Binary files /dev/null and b/examples/a500251/a500251_08.jpg differ
diff --git a/examples/a500251/a500251_09.jpg b/examples/a500251/a500251_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..04e803aa7701aac220a02ade6d9d3fea07ecbf60
Binary files /dev/null and b/examples/a500251/a500251_09.jpg differ
diff --git a/examples/a500251/a500251_10.jpg b/examples/a500251/a500251_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5c994a8cdca29313bc02ba4f3fae27d99bd4cb74
Binary files /dev/null and b/examples/a500251/a500251_10.jpg differ
diff --git a/examples/a500251/a500251_11.jpg b/examples/a500251/a500251_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e4972222160ea957cb44866e1406a21774adc729
Binary files /dev/null and b/examples/a500251/a500251_11.jpg differ
diff --git a/examples/a500251/a500251_12.jpg b/examples/a500251/a500251_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..34dbb00000235df30087b62b9536b39282cf1c26
Binary files /dev/null and b/examples/a500251/a500251_12.jpg differ
diff --git a/examples/a500251/a500251_13.jpg b/examples/a500251/a500251_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d82c2dfb5685c5940a7d3ea373851ee3b2dd4968
Binary files /dev/null and b/examples/a500251/a500251_13.jpg differ
diff --git a/examples/a500251/a500251_14.jpg b/examples/a500251/a500251_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e0f6e6cbbcb6c6db61791e35978125078a4339e2
Binary files /dev/null and b/examples/a500251/a500251_14.jpg differ
diff --git a/examples/a500251/a500251_15.jpg b/examples/a500251/a500251_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5f8deb0e3a0f8b4f462f296f47a65928a7f2034d
Binary files /dev/null and b/examples/a500251/a500251_15.jpg differ
diff --git a/examples/all_subsets.json b/examples/all_subsets.json
new file mode 100644
index 0000000000000000000000000000000000000000..a67cc3744b9c8dd0256923011d26ff9197f56a79
--- /dev/null
+++ b/examples/all_subsets.json
@@ -0,0 +1,505 @@
+[
+    {
+        "id": "real_r003679",
+        "images": [
+            "r003679_00.jpg",
+            "r003679_01.jpg",
+            "r003679_02.jpg",
+            "r003679_03.jpg",
+            "r003679_04.jpg",
+            "r003679_05.jpg",
+            "r003679_06.jpg",
+            "r003679_07.jpg",
+            "r003679_08.jpg",
+            "r003679_09.jpg",
+            "r003679_10.jpg",
+            "r003679_11.jpg",
+            "r003679_12.jpg",
+            "r003679_13.jpg",
+            "r003679_14.jpg",
+            "r003679_15.jpg",
+            "r003679_16.jpg",
+            "r003679_17.jpg",
+            "r003679_18.jpg",
+            "r003679_19.jpg",
+            "r003679_20.jpg",
+            "r003679_21.jpg",
+            "r003679_22.jpg",
+            "r003679_23.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is dog rolls over and falls off couch,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 3\n object consistency: 3\n dynamic degree: 3\n motion smoothness: 3\n text-to-video alignment: 3\n factual consistency: 3\n overall score: 3\n"
+            }
+        ],
+        "video": "r003679.mp4",
+        "prompt": "dog rolls over and falls off couch"
+    },
+    {
+        "id": "real_r100916",
+        "images": [
+            "r100916_00.jpg",
+            "r100916_01.jpg",
+            "r100916_02.jpg",
+            "r100916_03.jpg",
+            "r100916_04.jpg",
+            "r100916_05.jpg",
+            "r100916_06.jpg",
+            "r100916_07.jpg",
+            "r100916_08.jpg",
+            "r100916_09.jpg",
+            "r100916_10.jpg",
+            "r100916_11.jpg",
+            "r100916_12.jpg",
+            "r100916_13.jpg",
+            "r100916_14.jpg",
+            "r100916_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is A young man is standing in front of a glacier with mountains in the background.,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 3\n object consistency: 3\n dynamic degree: 3\n motion smoothness: 3\n text-to-video alignment: 3\n factual consistency: 3\n overall score: 3\n"
+            }
+        ],
+        "video": "r100916.mp4",
+        "prompt": "A young man is standing in front of a glacier with mountains in the background."
+    },
+    {
+        "id": "real_r004061",
+        "images": [
+            "r004061_00.jpg",
+            "r004061_01.jpg",
+            "r004061_02.jpg",
+            "r004061_03.jpg",
+            "r004061_04.jpg",
+            "r004061_05.jpg",
+            "r004061_06.jpg",
+            "r004061_07.jpg",
+            "r004061_08.jpg",
+            "r004061_09.jpg",
+            "r004061_10.jpg",
+            "r004061_11.jpg",
+            "r004061_12.jpg",
+            "r004061_13.jpg",
+            "r004061_14.jpg",
+            "r004061_15.jpg",
+            "r004061_16.jpg",
+            "r004061_17.jpg",
+            "r004061_18.jpg",
+            "r004061_19.jpg",
+            "r004061_20.jpg",
+            "r004061_21.jpg",
+            "r004061_22.jpg",
+            "r004061_23.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is a man wearing a striped blanket over his backpack runs behind two other men.,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 3\n object consistency: 3\n dynamic degree: 3\n motion smoothness: 3\n text-to-video alignment: 3\n factual consistency: 3\n overall score: 3\n"
+            }
+        ],
+        "video": "r004061.mp4",
+        "prompt": "a man wearing a striped blanket over his backpack runs behind two other men."
+    },
+    {
+        "id": "worsen_gen_b404675",
+        "images": [
+            "b404675_00.jpg",
+            "b404675_01.jpg",
+            "b404675_02.jpg",
+            "b404675_03.jpg",
+            "b404675_04.jpg",
+            "b404675_05.jpg",
+            "b404675_06.jpg",
+            "b404675_07.jpg",
+            "b404675_08.jpg",
+            "b404675_09.jpg",
+            "b404675_10.jpg",
+            "b404675_11.jpg",
+            "b404675_12.jpg",
+            "b404675_13.jpg",
+            "b404675_14.jpg",
+            "b404675_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is After a few years, Ghulam Muhammad became the owner of his business  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 1\n object consistency: 1\n dynamic degree: 3\n motion smoothness: 3\n text-to-video alignment: 2\n factual consistency: 3\n overall score: 1\n"
+            }
+        ],
+        "video": "b404675.mp4",
+        "prompt": "After a few years, Ghulam Muhammad became the owner of his business"
+    },
+    {
+        "id": "worsen_gen_b402727",
+        "images": [
+            "b402727_00.jpg",
+            "b402727_01.jpg",
+            "b402727_02.jpg",
+            "b402727_03.jpg",
+            "b402727_04.jpg",
+            "b402727_05.jpg",
+            "b402727_06.jpg",
+            "b402727_07.jpg",
+            "b402727_08.jpg",
+            "b402727_09.jpg",
+            "b402727_10.jpg",
+            "b402727_11.jpg",
+            "b402727_12.jpg",
+            "b402727_13.jpg",
+            "b402727_14.jpg",
+            "b402727_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is view down to the path in the field, moving forward, night, dark.  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 1\n object consistency: 1\n dynamic degree: 3\n motion smoothness: 3\n text-to-video alignment: 3\n factual consistency: 3\n overall score: 1\n"
+            }
+        ],
+        "video": "b402727.mp4",
+        "prompt": "view down to the path in the field, moving forward, night, dark."
+    },
+    {
+        "id": "worsen_gen_b304986",
+        "images": [
+            "b304986_00.jpg",
+            "b304986_01.jpg",
+            "b304986_02.jpg",
+            "b304986_03.jpg",
+            "b304986_04.jpg",
+            "b304986_05.jpg",
+            "b304986_06.jpg",
+            "b304986_07.jpg",
+            "b304986_08.jpg",
+            "b304986_09.jpg",
+            "b304986_10.jpg",
+            "b304986_11.jpg",
+            "b304986_12.jpg",
+            "b304986_13.jpg",
+            "b304986_14.jpg",
+            "b304986_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is Old Philippino man, close up, subtle smile, teats of joy, in old wood room  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 3\n object consistency: 1\n dynamic degree: 3\n motion smoothness: 1\n text-to-video alignment: 2\n factual consistency: 3\n overall score: 1\n"
+            }
+        ],
+        "video": "b304986.mp4",
+        "prompt": "Old Philippino man, close up, subtle smile, teats of joy, in old wood room"
+    },
+    {
+        "id": "static_d500937",
+        "images": [
+            "d500937_00.jpg",
+            "d500937_01.jpg",
+            "d500937_02.jpg",
+            "d500937_03.jpg",
+            "d500937_04.jpg",
+            "d500937_05.jpg",
+            "d500937_06.jpg",
+            "d500937_07.jpg",
+            "d500937_08.jpg",
+            "d500937_09.jpg",
+            "d500937_10.jpg",
+            "d500937_11.jpg",
+            "d500937_12.jpg",
+            "d500937_13.jpg",
+            "d500937_14.jpg",
+            "d500937_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is write y in the garden with flowers  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 1\n object consistency: 3\n dynamic degree: 1\n motion smoothness: 1\n text-to-video alignment: 1\n factual consistency: 1\n overall score: 1\n"
+            }
+        ],
+        "video": "d500937.mp4",
+        "prompt": "write y in the garden with flowers"
+    },
+    {
+        "id": "static_d500506",
+        "images": [
+            "d500506_00.jpg",
+            "d500506_01.jpg",
+            "d500506_02.jpg",
+            "d500506_03.jpg",
+            "d500506_04.jpg",
+            "d500506_05.jpg",
+            "d500506_06.jpg",
+            "d500506_07.jpg",
+            "d500506_08.jpg",
+            "d500506_09.jpg",
+            "d500506_10.jpg",
+            "d500506_11.jpg",
+            "d500506_12.jpg",
+            "d500506_13.jpg",
+            "d500506_14.jpg",
+            "d500506_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is a boy paiting a beautiful landscape  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 1\n object consistency: 3\n dynamic degree: 1\n motion smoothness: 1\n text-to-video alignment: 1\n factual consistency: 1\n overall score: 1\n"
+            }
+        ],
+        "video": "d500506.mp4",
+        "prompt": "a boy paiting a beautiful landscape"
+    },
+    {
+        "id": "static_d401950",
+        "images": [
+            "d401950_00.jpg",
+            "d401950_01.jpg",
+            "d401950_02.jpg",
+            "d401950_03.jpg",
+            "d401950_04.jpg",
+            "d401950_05.jpg",
+            "d401950_06.jpg",
+            "d401950_07.jpg",
+            "d401950_08.jpg",
+            "d401950_09.jpg",
+            "d401950_10.jpg",
+            "d401950_11.jpg",
+            "d401950_12.jpg",
+            "d401950_13.jpg",
+            "d401950_14.jpg",
+            "d401950_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is sneaker with exclusive design worth 1 billion  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 1\n object consistency: 3\n dynamic degree: 1\n motion smoothness: 1\n text-to-video alignment: 1\n factual consistency: 1\n overall score: 1\n"
+            }
+        ],
+        "video": "d401950.mp4",
+        "prompt": "sneaker with exclusive design worth 1 billion"
+    },
+    {
+        "id": "insf_a500251",
+        "images": [
+            "a500251_00.jpg",
+            "a500251_01.jpg",
+            "a500251_02.jpg",
+            "a500251_03.jpg",
+            "a500251_04.jpg",
+            "a500251_05.jpg",
+            "a500251_06.jpg",
+            "a500251_07.jpg",
+            "a500251_08.jpg",
+            "a500251_09.jpg",
+            "a500251_10.jpg",
+            "a500251_11.jpg",
+            "a500251_12.jpg",
+            "a500251_13.jpg",
+            "a500251_14.jpg",
+            "a500251_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is :sci fi war in the moon  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 1\n object consistency: 1\n dynamic degree: 3\n motion smoothness: 1\n text-to-video alignment: 1\n factual consistency: 1\n overall score: 1\n"
+            }
+        ],
+        "video": "a500251.mp4",
+        "prompt": ":sci fi war in the moon"
+    },
+    {
+        "id": "insf_a400480",
+        "images": [
+            "a400480_00.jpg",
+            "a400480_01.jpg",
+            "a400480_02.jpg",
+            "a400480_03.jpg",
+            "a400480_04.jpg",
+            "a400480_05.jpg",
+            "a400480_06.jpg",
+            "a400480_07.jpg",
+            "a400480_08.jpg",
+            "a400480_09.jpg",
+            "a400480_10.jpg",
+            "a400480_11.jpg",
+            "a400480_12.jpg",
+            "a400480_13.jpg",
+            "a400480_14.jpg",
+            "a400480_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is for a dog accessories brand a welcome image: brand name: hot doggies  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 1\n object consistency: 1\n dynamic degree: 3\n motion smoothness: 1\n text-to-video alignment: 1\n factual consistency: 1\n overall score: 1\n"
+            }
+        ],
+        "video": "a400480.mp4",
+        "prompt": "for a dog accessories brand a welcome image: brand name: hot doggies"
+    },
+    {
+        "id": "insf_a500010",
+        "images": [
+            "a500010_00.jpg",
+            "a500010_01.jpg",
+            "a500010_02.jpg",
+            "a500010_03.jpg",
+            "a500010_04.jpg",
+            "a500010_05.jpg",
+            "a500010_06.jpg",
+            "a500010_07.jpg",
+            "a500010_08.jpg",
+            "a500010_09.jpg",
+            "a500010_10.jpg",
+            "a500010_11.jpg",
+            "a500010_12.jpg",
+            "a500010_13.jpg",
+            "a500010_14.jpg",
+            "a500010_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is a druid in the forest cinematic  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 1\n object consistency: 1\n dynamic degree: 3\n motion smoothness: 1\n text-to-video alignment: 1\n factual consistency: 1\n overall score: 1\n"
+            }
+        ],
+        "video": "a500010.mp4",
+        "prompt": "a druid in the forest cinematic"
+    },
+    {
+        "id": "lab_3005033",
+        "images": [
+            "3005033_00.jpg",
+            "3005033_01.jpg",
+            "3005033_02.jpg",
+            "3005033_03.jpg",
+            "3005033_04.jpg",
+            "3005033_05.jpg",
+            "3005033_06.jpg",
+            "3005033_07.jpg",
+            "3005033_08.jpg",
+            "3005033_09.jpg",
+            "3005033_10.jpg",
+            "3005033_11.jpg",
+            "3005033_12.jpg",
+            "3005033_13.jpg",
+            "3005033_14.jpg",
+            "3005033_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is a robot vacuum traveling through southeast asian countries  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 3\n object consistency: 2\n dynamic degree: 3\n motion smoothness: 2\n text-to-video alignment: 2\n factual consistency: 3\n overall score: 2\n"
+            }
+        ],
+        "video": "3005033.mp4",
+        "prompt": "a robot vacuum traveling through southeast asian countries"
+    },
+    {
+        "id": "lab_7004180",
+        "images": [
+            "7004180_00.jpg",
+            "7004180_01.jpg",
+            "7004180_02.jpg",
+            "7004180_03.jpg",
+            "7004180_04.jpg",
+            "7004180_05.jpg",
+            "7004180_06.jpg",
+            "7004180_07.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is jiaxuan is the most beautiful \uff0celegant\uff0ccharming girl in the world. billions of boys fell in love with her  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 3\n object consistency: 3\n dynamic degree: 3\n motion smoothness: 3\n text-to-video alignment: 3\n factual consistency: 3\n overall score: 3\n"
+            }
+        ],
+        "video": "7004180.mp4",
+        "prompt": "jiaxuan is the most beautiful \uff0celegant\uff0ccharming girl in the world. billions of boys fell in love with her"
+    },
+    {
+        "id": "lab_1006309",
+        "images": [
+            "1006309_00.jpg",
+            "1006309_01.jpg",
+            "1006309_02.jpg",
+            "1006309_03.jpg",
+            "1006309_04.jpg",
+            "1006309_05.jpg",
+            "1006309_06.jpg",
+            "1006309_07.jpg",
+            "1006309_08.jpg",
+            "1006309_09.jpg",
+            "1006309_10.jpg",
+            "1006309_11.jpg",
+            "1006309_12.jpg",
+            "1006309_13.jpg",
+            "1006309_14.jpg",
+            "1006309_15.jpg"
+        ],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \nplease watch the following frames of a given video and see the text prompt for generating the video, \nthen give scores from 7 different dimensions:\n(1) visual quality, \n(2) object consistency,\n(3) dynamic degree,\n(4) motion smoothness,\n(5) text-to-video alignment,\n(6) factual consistency, \n(7) overall score\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\nHere is an output example: \nvisual quality: 3\nobject consistency: 2 \ndynamic degree: 2\nmotion smoothness: 1\ntext-to-video alignment: 1\nfactual consistency: 2\noverall score: 1\n\nFor this item, the text prompt is scary halloween skeletons are having a conversation with each other  ,\nall the frames of video are as follows: \n\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \n"
+            },
+            {
+                "from": "gpt",
+                "value": "visual quality: 3\n object consistency: 2\n dynamic degree: 3\n motion smoothness: 2\n text-to-video alignment: 3\n factual consistency: 2\n overall score: 3\n"
+            }
+        ],
+        "video": "1006309.mp4",
+        "prompt": "scary halloween skeletons are having a conversation with each other"
+    }
+]
\ No newline at end of file
diff --git a/examples/b304986.mp4 b/examples/b304986.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4c727e456ce31e944cc860a98519d2007fa6b75c
--- /dev/null
+++ b/examples/b304986.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32b73b7521aeac3959523bb463560ee847739a4d544fe41c1627162e7aa758b7
+size 35045
diff --git a/examples/b304986/b304986_00.jpg b/examples/b304986/b304986_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..eb593a78a4596132b32fb6d0a5e61376a2f14a4e
Binary files /dev/null and b/examples/b304986/b304986_00.jpg differ
diff --git a/examples/b304986/b304986_01.jpg b/examples/b304986/b304986_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..91f63cd79d3490b9d6d44a98d24edd8511d89c72
Binary files /dev/null and b/examples/b304986/b304986_01.jpg differ
diff --git a/examples/b304986/b304986_02.jpg b/examples/b304986/b304986_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1a1934ca3633c3128683390ed8cf2abebdf14874
Binary files /dev/null and b/examples/b304986/b304986_02.jpg differ
diff --git a/examples/b304986/b304986_03.jpg b/examples/b304986/b304986_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7be80028c8d55b54f277d3aa813d0bec7a14d5ca
Binary files /dev/null and b/examples/b304986/b304986_03.jpg differ
diff --git a/examples/b304986/b304986_04.jpg b/examples/b304986/b304986_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..85f124a02e14bd976f7081bbef85f4b92ac2f2e2
Binary files /dev/null and b/examples/b304986/b304986_04.jpg differ
diff --git a/examples/b304986/b304986_05.jpg b/examples/b304986/b304986_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..85ce4c9714ff3fe26083171c5f079fdfa9f87f23
Binary files /dev/null and b/examples/b304986/b304986_05.jpg differ
diff --git a/examples/b304986/b304986_06.jpg b/examples/b304986/b304986_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..324cd0144c706ff56ee849fdb2f945869cc3da0c
Binary files /dev/null and b/examples/b304986/b304986_06.jpg differ
diff --git a/examples/b304986/b304986_07.jpg b/examples/b304986/b304986_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ac0775ae8f9c9ee20dae2415ded1c65f8adc0748
Binary files /dev/null and b/examples/b304986/b304986_07.jpg differ
diff --git a/examples/b304986/b304986_08.jpg b/examples/b304986/b304986_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..545a13bf158b63889ed569a9cb02fe9232fd38b8
Binary files /dev/null and b/examples/b304986/b304986_08.jpg differ
diff --git a/examples/b304986/b304986_09.jpg b/examples/b304986/b304986_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a883f44931e92fe955bc26ea3558715b7ac82eb4
Binary files /dev/null and b/examples/b304986/b304986_09.jpg differ
diff --git a/examples/b304986/b304986_10.jpg b/examples/b304986/b304986_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..2d7fcaa3a89b0abca2018e4c6dced94f06daf585
Binary files /dev/null and b/examples/b304986/b304986_10.jpg differ
diff --git a/examples/b304986/b304986_11.jpg b/examples/b304986/b304986_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..8068257a5b98fad9e15fb7341cb4508073f48e68
Binary files /dev/null and b/examples/b304986/b304986_11.jpg differ
diff --git a/examples/b304986/b304986_12.jpg b/examples/b304986/b304986_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..cc47c7a6c44f93688f937b1c1ac65001cc47da13
Binary files /dev/null and b/examples/b304986/b304986_12.jpg differ
diff --git a/examples/b304986/b304986_13.jpg b/examples/b304986/b304986_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0ccf9f9daffd6d084016b811f7b0564661aee80b
Binary files /dev/null and b/examples/b304986/b304986_13.jpg differ
diff --git a/examples/b304986/b304986_14.jpg b/examples/b304986/b304986_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c7c2c51b1f886e0711888790dd3f3e4c7825ee0e
Binary files /dev/null and b/examples/b304986/b304986_14.jpg differ
diff --git a/examples/b304986/b304986_15.jpg b/examples/b304986/b304986_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a77c7d05ce0838b9a66db0f1cd8fff0ae1234714
Binary files /dev/null and b/examples/b304986/b304986_15.jpg differ
diff --git a/examples/b402727.mp4 b/examples/b402727.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..001ed996d980e0de74fba7e9ecb01ea54850517e
--- /dev/null
+++ b/examples/b402727.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f3fd028a5fd58153638e23a83879fb7ec82643d7a4d422334d3f3caf21ce646
+size 26288
diff --git a/examples/b402727/b402727_00.jpg b/examples/b402727/b402727_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e04a7d43f3afd267bd8acaaa7610a2280f936f4d
Binary files /dev/null and b/examples/b402727/b402727_00.jpg differ
diff --git a/examples/b402727/b402727_01.jpg b/examples/b402727/b402727_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..51ba29906cadb1a66e63f014a94074a73a1af04e
Binary files /dev/null and b/examples/b402727/b402727_01.jpg differ
diff --git a/examples/b402727/b402727_02.jpg b/examples/b402727/b402727_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7f92bf78f40c5e5b4062f1655af47ffdc39af6cf
Binary files /dev/null and b/examples/b402727/b402727_02.jpg differ
diff --git a/examples/b402727/b402727_03.jpg b/examples/b402727/b402727_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..17b18da09d2718f0c1c169dca9697d8405a4f045
Binary files /dev/null and b/examples/b402727/b402727_03.jpg differ
diff --git a/examples/b402727/b402727_04.jpg b/examples/b402727/b402727_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..51198583d762970938b217f6f59c3882f93c1b40
Binary files /dev/null and b/examples/b402727/b402727_04.jpg differ
diff --git a/examples/b402727/b402727_05.jpg b/examples/b402727/b402727_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0a4e526783a3d1d51ad2d7217fb073acb305cc57
Binary files /dev/null and b/examples/b402727/b402727_05.jpg differ
diff --git a/examples/b402727/b402727_06.jpg b/examples/b402727/b402727_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d7039aa25cf00ed1b32a397f0c1f3709d709ed69
Binary files /dev/null and b/examples/b402727/b402727_06.jpg differ
diff --git a/examples/b402727/b402727_07.jpg b/examples/b402727/b402727_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..93b271e56239cb975e8001b02e66507eb57cebdc
Binary files /dev/null and b/examples/b402727/b402727_07.jpg differ
diff --git a/examples/b402727/b402727_08.jpg b/examples/b402727/b402727_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..320a6d79b9c81f10265daeae9806d8cea8ce05bf
Binary files /dev/null and b/examples/b402727/b402727_08.jpg differ
diff --git a/examples/b402727/b402727_09.jpg b/examples/b402727/b402727_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..4a7a0322c858642f3296598736433f90a0bc1776
Binary files /dev/null and b/examples/b402727/b402727_09.jpg differ
diff --git a/examples/b402727/b402727_10.jpg b/examples/b402727/b402727_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..869829574d03b3bc2406d8ae8444b0166972d880
Binary files /dev/null and b/examples/b402727/b402727_10.jpg differ
diff --git a/examples/b402727/b402727_11.jpg b/examples/b402727/b402727_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c85e83fbecc157a663a55662cf9620aecae7d2e3
Binary files /dev/null and b/examples/b402727/b402727_11.jpg differ
diff --git a/examples/b402727/b402727_12.jpg b/examples/b402727/b402727_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f4bd63f086af4bbcc7d19e33d356292d68727bb2
Binary files /dev/null and b/examples/b402727/b402727_12.jpg differ
diff --git a/examples/b402727/b402727_13.jpg b/examples/b402727/b402727_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a95e4974e3aac4cb5d9e46b2fa119b6e951170ea
Binary files /dev/null and b/examples/b402727/b402727_13.jpg differ
diff --git a/examples/b402727/b402727_14.jpg b/examples/b402727/b402727_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f8f57dc950c7cddd5e65972b7dab0d2c50f54e9e
Binary files /dev/null and b/examples/b402727/b402727_14.jpg differ
diff --git a/examples/b402727/b402727_15.jpg b/examples/b402727/b402727_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..af62022deca3c3801dadc2c881ed307fb11effbd
Binary files /dev/null and b/examples/b402727/b402727_15.jpg differ
diff --git a/examples/b404675.mp4 b/examples/b404675.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..54133a6d982dfc55ebbfde9f24c9898c361188a3
--- /dev/null
+++ b/examples/b404675.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfe86bcb6983f72838ba1efaf4ae74e726dd822a21e53b9f64d22b8345c497aa
+size 147785
diff --git a/examples/b404675/b404675_00.jpg b/examples/b404675/b404675_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6fca439881f21ef133cc9c2e43b510e9ce0fc883
Binary files /dev/null and b/examples/b404675/b404675_00.jpg differ
diff --git a/examples/b404675/b404675_01.jpg b/examples/b404675/b404675_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..717d648779b1e8d9deace827fc52cdf37b7396ea
Binary files /dev/null and b/examples/b404675/b404675_01.jpg differ
diff --git a/examples/b404675/b404675_02.jpg b/examples/b404675/b404675_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..60b295fd935464af592035e584411a7d7236a61f
Binary files /dev/null and b/examples/b404675/b404675_02.jpg differ
diff --git a/examples/b404675/b404675_03.jpg b/examples/b404675/b404675_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e4e0aa06e9c2f5c6e8b23016a765cf191d9776f0
Binary files /dev/null and b/examples/b404675/b404675_03.jpg differ
diff --git a/examples/b404675/b404675_04.jpg b/examples/b404675/b404675_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a305947e0754ae60ee43a4828090644ae255ff70
Binary files /dev/null and b/examples/b404675/b404675_04.jpg differ
diff --git a/examples/b404675/b404675_05.jpg b/examples/b404675/b404675_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..89e2bd2b1c6965e282dc58c4178e7b51294c8d9b
Binary files /dev/null and b/examples/b404675/b404675_05.jpg differ
diff --git a/examples/b404675/b404675_06.jpg b/examples/b404675/b404675_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a5252635f2dfc70edd8c200be891dc2c63ae900b
Binary files /dev/null and b/examples/b404675/b404675_06.jpg differ
diff --git a/examples/b404675/b404675_07.jpg b/examples/b404675/b404675_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a2a62316cd371a6a5d085dcd9233111e02bf93f7
Binary files /dev/null and b/examples/b404675/b404675_07.jpg differ
diff --git a/examples/b404675/b404675_08.jpg b/examples/b404675/b404675_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..505525d2a344f067a9c343f7954205bd9e952dbb
Binary files /dev/null and b/examples/b404675/b404675_08.jpg differ
diff --git a/examples/b404675/b404675_09.jpg b/examples/b404675/b404675_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a1ad7e1af614d46cac25f026c749b35d989c11b0
Binary files /dev/null and b/examples/b404675/b404675_09.jpg differ
diff --git a/examples/b404675/b404675_10.jpg b/examples/b404675/b404675_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..da6f4ea479f9ad27b85955ee6f3630081f1e42ed
Binary files /dev/null and b/examples/b404675/b404675_10.jpg differ
diff --git a/examples/b404675/b404675_11.jpg b/examples/b404675/b404675_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0342c750f65999f0d118b46a0aae8375c54c4a1a
Binary files /dev/null and b/examples/b404675/b404675_11.jpg differ
diff --git a/examples/b404675/b404675_12.jpg b/examples/b404675/b404675_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..71f42a8c39a7da5139700036216ba239e35d02db
Binary files /dev/null and b/examples/b404675/b404675_12.jpg differ
diff --git a/examples/b404675/b404675_13.jpg b/examples/b404675/b404675_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..971489d21233200d462bedf2b7cc069e0d0bd30b
Binary files /dev/null and b/examples/b404675/b404675_13.jpg differ
diff --git a/examples/b404675/b404675_14.jpg b/examples/b404675/b404675_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6e66c5ca0fc4c0ceb15206e7c38722507c6ae19a
Binary files /dev/null and b/examples/b404675/b404675_14.jpg differ
diff --git a/examples/b404675/b404675_15.jpg b/examples/b404675/b404675_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..458745cb662f8c6e71f817f67033356696db5ede
Binary files /dev/null and b/examples/b404675/b404675_15.jpg differ
diff --git a/examples/d401950.mp4 b/examples/d401950.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..4725d58c022e5a49157d0bf6ea666e61cbb28e36
--- /dev/null
+++ b/examples/d401950.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75066bd9680304cb1ba17be119e0e0dd12f149c9515a29a747e494a70f880163
+size 14257
diff --git a/examples/d401950/d401950_00.jpg b/examples/d401950/d401950_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0b7f1014c4b2947de7cc67f8cd5213484f4da53b
Binary files /dev/null and b/examples/d401950/d401950_00.jpg differ
diff --git a/examples/d401950/d401950_01.jpg b/examples/d401950/d401950_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7f698ea8dba1662aacc77be39f65e5abe593cf1a
Binary files /dev/null and b/examples/d401950/d401950_01.jpg differ
diff --git a/examples/d401950/d401950_02.jpg b/examples/d401950/d401950_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..baed6216de5986b6d6291c2563de40a227741b8a
Binary files /dev/null and b/examples/d401950/d401950_02.jpg differ
diff --git a/examples/d401950/d401950_03.jpg b/examples/d401950/d401950_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..baed6216de5986b6d6291c2563de40a227741b8a
Binary files /dev/null and b/examples/d401950/d401950_03.jpg differ
diff --git a/examples/d401950/d401950_04.jpg b/examples/d401950/d401950_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ab43a579e54d4389fa2f4a54cc0898cd54d3c93f
Binary files /dev/null and b/examples/d401950/d401950_04.jpg differ
diff --git a/examples/d401950/d401950_05.jpg b/examples/d401950/d401950_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d2f67c36568ae022aae739492475786bd0313bc5
Binary files /dev/null and b/examples/d401950/d401950_05.jpg differ
diff --git a/examples/d401950/d401950_06.jpg b/examples/d401950/d401950_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ed6f3993c42d4dce9a83ca84514836d3510b8eb7
Binary files /dev/null and b/examples/d401950/d401950_06.jpg differ
diff --git a/examples/d401950/d401950_07.jpg b/examples/d401950/d401950_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6c582c7fb0c76736fec425804e528ccdd45fff0a
Binary files /dev/null and b/examples/d401950/d401950_07.jpg differ
diff --git a/examples/d401950/d401950_08.jpg b/examples/d401950/d401950_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ed3b46bbbf9f2a5b9e71b3beb82f3a81abe502c3
Binary files /dev/null and b/examples/d401950/d401950_08.jpg differ
diff --git a/examples/d401950/d401950_09.jpg b/examples/d401950/d401950_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..11dc0cd6ae634b59c6bf4600c2ebf1936f6fa29d
Binary files /dev/null and b/examples/d401950/d401950_09.jpg differ
diff --git a/examples/d401950/d401950_10.jpg b/examples/d401950/d401950_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c80d41b3a078f335859a409607a0affe386e1fd5
Binary files /dev/null and b/examples/d401950/d401950_10.jpg differ
diff --git a/examples/d401950/d401950_11.jpg b/examples/d401950/d401950_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f6ac5ecb3d293c69e1b28c8975f5f842f2bd4ba2
Binary files /dev/null and b/examples/d401950/d401950_11.jpg differ
diff --git a/examples/d401950/d401950_12.jpg b/examples/d401950/d401950_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1fd5f87dc3f9945e86cd0fdb18192575120ca581
Binary files /dev/null and b/examples/d401950/d401950_12.jpg differ
diff --git a/examples/d401950/d401950_13.jpg b/examples/d401950/d401950_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1cc188150bee2d3553a3d441d59cf4345ee9a79e
Binary files /dev/null and b/examples/d401950/d401950_13.jpg differ
diff --git a/examples/d401950/d401950_14.jpg b/examples/d401950/d401950_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f48d03bc7bdcab52e14e91463cbaaaed7abbef58
Binary files /dev/null and b/examples/d401950/d401950_14.jpg differ
diff --git a/examples/d401950/d401950_15.jpg b/examples/d401950/d401950_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..83c58138f5eeca007af7358acba5b17ef2506aa5
Binary files /dev/null and b/examples/d401950/d401950_15.jpg differ
diff --git a/examples/d500506.mp4 b/examples/d500506.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..c9a05a8930d72857ebd73ef0e56d007dd52a30df
--- /dev/null
+++ b/examples/d500506.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e18b5116888477bedadc8344770303424eb3bd58404c0df389c26a484acd80b
+size 46519
diff --git a/examples/d500506/d500506_00.jpg b/examples/d500506/d500506_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c9399b4ac0072b84abbf4f40192191f76a9fcfb5
Binary files /dev/null and b/examples/d500506/d500506_00.jpg differ
diff --git a/examples/d500506/d500506_01.jpg b/examples/d500506/d500506_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ce67cec4ca4ba9cad0271eb3b665019f70b42c5a
Binary files /dev/null and b/examples/d500506/d500506_01.jpg differ
diff --git a/examples/d500506/d500506_02.jpg b/examples/d500506/d500506_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..fabbf5df07f82f8ce3e88025ca81ce0cd528cd53
Binary files /dev/null and b/examples/d500506/d500506_02.jpg differ
diff --git a/examples/d500506/d500506_03.jpg b/examples/d500506/d500506_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..082c09ab373f7e65e4fbc94827e8aa87c72b9ccf
Binary files /dev/null and b/examples/d500506/d500506_03.jpg differ
diff --git a/examples/d500506/d500506_04.jpg b/examples/d500506/d500506_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5acd35992f95595b46342f38a963d3bcb0fc3153
Binary files /dev/null and b/examples/d500506/d500506_04.jpg differ
diff --git a/examples/d500506/d500506_05.jpg b/examples/d500506/d500506_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..efbf523309534bcdb13c02708fcfa893ef22a7a1
Binary files /dev/null and b/examples/d500506/d500506_05.jpg differ
diff --git a/examples/d500506/d500506_06.jpg b/examples/d500506/d500506_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..8a7969d75d1c997948f31f0e1d48108aad08c6c2
Binary files /dev/null and b/examples/d500506/d500506_06.jpg differ
diff --git a/examples/d500506/d500506_07.jpg b/examples/d500506/d500506_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1e8f9f83dba015f4d6a2b609920bbad558a38533
Binary files /dev/null and b/examples/d500506/d500506_07.jpg differ
diff --git a/examples/d500506/d500506_08.jpg b/examples/d500506/d500506_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..96c7c27fa2174dc7fd8cdaec245e26d139bd6378
Binary files /dev/null and b/examples/d500506/d500506_08.jpg differ
diff --git a/examples/d500506/d500506_09.jpg b/examples/d500506/d500506_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a91a2194ea61c5ecb50bdf7df86221f718ae3c86
Binary files /dev/null and b/examples/d500506/d500506_09.jpg differ
diff --git a/examples/d500506/d500506_10.jpg b/examples/d500506/d500506_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a6dc11fcda8ac854bbaff116ce277fd2abf4ca6d
Binary files /dev/null and b/examples/d500506/d500506_10.jpg differ
diff --git a/examples/d500506/d500506_11.jpg b/examples/d500506/d500506_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..20db1103832fd2928cb0778e1063ddfb63c6b539
Binary files /dev/null and b/examples/d500506/d500506_11.jpg differ
diff --git a/examples/d500506/d500506_12.jpg b/examples/d500506/d500506_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..14f0dd8d993c972f712bd0a20e66fb65d04bf405
Binary files /dev/null and b/examples/d500506/d500506_12.jpg differ
diff --git a/examples/d500506/d500506_13.jpg b/examples/d500506/d500506_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e14f63bef21cb8a95260c01c36e4356bcadb97de
Binary files /dev/null and b/examples/d500506/d500506_13.jpg differ
diff --git a/examples/d500506/d500506_14.jpg b/examples/d500506/d500506_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..313425205f32be4b77073c8ef7eaa76324a2e305
Binary files /dev/null and b/examples/d500506/d500506_14.jpg differ
diff --git a/examples/d500506/d500506_15.jpg b/examples/d500506/d500506_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..067ef951086174c13e8ad82a7dad9ee443e71e96
Binary files /dev/null and b/examples/d500506/d500506_15.jpg differ
diff --git a/examples/d500937.mp4 b/examples/d500937.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..fea18e4138ece0aba62f6b7bc83b14287034c438
--- /dev/null
+++ b/examples/d500937.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0faf19200e9a00cb9b0b0d300fb567ebde34434ccb600ade4ea9812a84498ab4
+size 36967
diff --git a/examples/d500937/d500937_00.jpg b/examples/d500937/d500937_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..425a97e86de5175256d257d20ccf20dbeceda08d
Binary files /dev/null and b/examples/d500937/d500937_00.jpg differ
diff --git a/examples/d500937/d500937_01.jpg b/examples/d500937/d500937_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..99102ca55b95fed56e471b18ce04fe8609414308
Binary files /dev/null and b/examples/d500937/d500937_01.jpg differ
diff --git a/examples/d500937/d500937_02.jpg b/examples/d500937/d500937_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b964fec1a0794dcc82fd6df43e52a1e8188ecea0
Binary files /dev/null and b/examples/d500937/d500937_02.jpg differ
diff --git a/examples/d500937/d500937_03.jpg b/examples/d500937/d500937_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..3c08f192b025f224eab95bae5522f8a0ffc78270
Binary files /dev/null and b/examples/d500937/d500937_03.jpg differ
diff --git a/examples/d500937/d500937_04.jpg b/examples/d500937/d500937_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..3c88cdc0d5f1140dd5bf64f96e37edb7f2975e5b
Binary files /dev/null and b/examples/d500937/d500937_04.jpg differ
diff --git a/examples/d500937/d500937_05.jpg b/examples/d500937/d500937_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..dc317a150f7500b044a2326aee476c466ed88199
Binary files /dev/null and b/examples/d500937/d500937_05.jpg differ
diff --git a/examples/d500937/d500937_06.jpg b/examples/d500937/d500937_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..35c5a557ed43f1029694eb591a3be1372d7917ba
Binary files /dev/null and b/examples/d500937/d500937_06.jpg differ
diff --git a/examples/d500937/d500937_07.jpg b/examples/d500937/d500937_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7d7ffda064b16d52ab519554b9ed8f8d42bd076a
Binary files /dev/null and b/examples/d500937/d500937_07.jpg differ
diff --git a/examples/d500937/d500937_08.jpg b/examples/d500937/d500937_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1640b927c0e659b991905e668f114c9c36dcbf91
Binary files /dev/null and b/examples/d500937/d500937_08.jpg differ
diff --git a/examples/d500937/d500937_09.jpg b/examples/d500937/d500937_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9984230d8c60509d9874c281229e29069939b869
Binary files /dev/null and b/examples/d500937/d500937_09.jpg differ
diff --git a/examples/d500937/d500937_10.jpg b/examples/d500937/d500937_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..dd5e4d3f4c533323a6a1e556ad5c91269a94249b
Binary files /dev/null and b/examples/d500937/d500937_10.jpg differ
diff --git a/examples/d500937/d500937_11.jpg b/examples/d500937/d500937_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..eaaf3c3dc0cb444007c5fce57ae36eae8c3cb333
Binary files /dev/null and b/examples/d500937/d500937_11.jpg differ
diff --git a/examples/d500937/d500937_12.jpg b/examples/d500937/d500937_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..815b83881fa0fa925f2d660e1722a836cfce2921
Binary files /dev/null and b/examples/d500937/d500937_12.jpg differ
diff --git a/examples/d500937/d500937_13.jpg b/examples/d500937/d500937_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6f6b0c1a2498b5d356de6de5ee00a8c91b8a9236
Binary files /dev/null and b/examples/d500937/d500937_13.jpg differ
diff --git a/examples/d500937/d500937_14.jpg b/examples/d500937/d500937_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..56ebc60cf3dd3d0d6b03a497d01ff2de706b7016
Binary files /dev/null and b/examples/d500937/d500937_14.jpg differ
diff --git a/examples/d500937/d500937_15.jpg b/examples/d500937/d500937_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e298f30060a5fe0631b073c631977cc3f29833cd
Binary files /dev/null and b/examples/d500937/d500937_15.jpg differ
diff --git a/examples/hd.json b/examples/hd.json
new file mode 100644
index 0000000000000000000000000000000000000000..e31332699c2f68b8e3f38f5a5550d3a0000cae87
--- /dev/null
+++ b/examples/hd.json
@@ -0,0 +1,62 @@
+[
+    {
+        "id": "hd1",
+        "conversations": [
+            {
+                "from": "human",
+                "value": ""
+            },
+            {
+                "from": "gpt",
+                "value": ""
+            }
+        ],
+        "video": "hd1.mp4",
+        "prompt": "An indoor gym"
+    },
+    {
+        "id": "hd2",
+        "conversations": [
+            {
+                "from": "human",
+                "value": ""
+            },
+            {
+                "from": "gpt",
+                "value": ""
+            }
+        ],
+        "video": "hd2.mp4",
+        "prompt": "None"
+    },
+    {
+        "id": "hd3",
+        "conversations": [
+            {
+                "from": "human",
+                "value": ""
+            },
+            {
+                "from": "gpt",
+                "value": ""
+            }
+        ],
+        "video": "hd3.mp4",
+        "prompt": "A person barbecuing"
+    },
+    {
+        "id": "hd4",
+        "conversations": [
+            {
+                "from": "human",
+                "value": ""
+            },
+            {
+                "from": "gpt",
+                "value": ""
+            }
+        ],
+        "video": "hd4.mp4",
+        "prompt": "A child eating"
+    }
+]
\ No newline at end of file
diff --git a/examples/hd1.mp4 b/examples/hd1.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..5e0446c108356bf33f492a38b4468b7ed29e12e9
--- /dev/null
+++ b/examples/hd1.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:057409b842da7f266df0f46639a729a69eff2819e1cd0b567815bdfd93b59343
+size 97124
diff --git a/examples/hd2.mp4 b/examples/hd2.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..329f584df1937e3bb6a160cbe22c740b685386ef
--- /dev/null
+++ b/examples/hd2.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:970d1797c5e357672a3f2cfcfabcacb6eeced4032811551166cfcb7eb63a4814
+size 311262
diff --git a/examples/hd3.mp4 b/examples/hd3.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..e040a045347bc252346cb0a476dbdcc0432bb21d
--- /dev/null
+++ b/examples/hd3.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91ebff6f8e885d97bfddac3cd39c56957de269c552dc012fa711624860a7f1d6
+size 223946
diff --git a/examples/hd4.mp4 b/examples/hd4.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ff2a1fb798c552fcf318fd2c0c0141460084b1d2
--- /dev/null
+++ b/examples/hd4.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbd672b2647c473997ef2edca63607367e2768d8de8b0683d1b504977c305c49
+size 3860683
diff --git a/examples/r003679.mp4 b/examples/r003679.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..ca7a9e57eaa98b91b0a18b540831aa959f7e25ec
--- /dev/null
+++ b/examples/r003679.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:341fcb71c892ee7735111fbb216f920571f7eac4ff737f17423e32d57cd26ab1
+size 62574
diff --git a/examples/r003679/r003679_00.jpg b/examples/r003679/r003679_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e19f58471a3d6f1dcac4cb08bfd7ac9b84d83599
Binary files /dev/null and b/examples/r003679/r003679_00.jpg differ
diff --git a/examples/r003679/r003679_01.jpg b/examples/r003679/r003679_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..fde61de275a200dddcbcbd1e3ad0418eccc9b00b
Binary files /dev/null and b/examples/r003679/r003679_01.jpg differ
diff --git a/examples/r003679/r003679_02.jpg b/examples/r003679/r003679_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7f1a0a1a8c3bf1d5ddebbd3b280ca5e995517359
Binary files /dev/null and b/examples/r003679/r003679_02.jpg differ
diff --git a/examples/r003679/r003679_03.jpg b/examples/r003679/r003679_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c3c0679e60b26d0a9f8842093cefae207cf6aa6b
Binary files /dev/null and b/examples/r003679/r003679_03.jpg differ
diff --git a/examples/r003679/r003679_04.jpg b/examples/r003679/r003679_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..cfa8db7ab0425f5dc327525ccbd7394c18583018
Binary files /dev/null and b/examples/r003679/r003679_04.jpg differ
diff --git a/examples/r003679/r003679_05.jpg b/examples/r003679/r003679_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5010330c23925bfde5fb326bc3d13f958c558610
Binary files /dev/null and b/examples/r003679/r003679_05.jpg differ
diff --git a/examples/r003679/r003679_06.jpg b/examples/r003679/r003679_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..546e63ddfd4a10569011318db007394f6cd16c42
Binary files /dev/null and b/examples/r003679/r003679_06.jpg differ
diff --git a/examples/r003679/r003679_07.jpg b/examples/r003679/r003679_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..3f2b116ec3bfdfabc72f5f31f34c1b7cf6f824b1
Binary files /dev/null and b/examples/r003679/r003679_07.jpg differ
diff --git a/examples/r003679/r003679_08.jpg b/examples/r003679/r003679_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..914041c5641509674eef7d4668bc8791759d0095
Binary files /dev/null and b/examples/r003679/r003679_08.jpg differ
diff --git a/examples/r003679/r003679_09.jpg b/examples/r003679/r003679_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a987baab1a48db113c41755ace8d5e812a13f540
Binary files /dev/null and b/examples/r003679/r003679_09.jpg differ
diff --git a/examples/r003679/r003679_10.jpg b/examples/r003679/r003679_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9c8a2d1c0ba73350fd27423d49312c64285d671d
Binary files /dev/null and b/examples/r003679/r003679_10.jpg differ
diff --git a/examples/r003679/r003679_11.jpg b/examples/r003679/r003679_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9af67418c588600bb26bd05b36e3ddd8e47da2a8
Binary files /dev/null and b/examples/r003679/r003679_11.jpg differ
diff --git a/examples/r003679/r003679_12.jpg b/examples/r003679/r003679_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..503abb7ce1162cc72e10d59dcc5c8dd76056f1cf
Binary files /dev/null and b/examples/r003679/r003679_12.jpg differ
diff --git a/examples/r003679/r003679_13.jpg b/examples/r003679/r003679_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..8571c598642f1e6035a7682379f76f25d3c81bfa
Binary files /dev/null and b/examples/r003679/r003679_13.jpg differ
diff --git a/examples/r003679/r003679_14.jpg b/examples/r003679/r003679_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..662dee74fca4d43d6828ec28f3f9b37dbdddfc94
Binary files /dev/null and b/examples/r003679/r003679_14.jpg differ
diff --git a/examples/r003679/r003679_15.jpg b/examples/r003679/r003679_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..65435c4471872334e10d8b7764c764240863bf13
Binary files /dev/null and b/examples/r003679/r003679_15.jpg differ
diff --git a/examples/r003679/r003679_16.jpg b/examples/r003679/r003679_16.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5b29059476545e89423b3524c6454c7ea44bfa23
Binary files /dev/null and b/examples/r003679/r003679_16.jpg differ
diff --git a/examples/r003679/r003679_17.jpg b/examples/r003679/r003679_17.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..8262c7c68be4a53fe4c6d433d6076b41a203efc6
Binary files /dev/null and b/examples/r003679/r003679_17.jpg differ
diff --git a/examples/r003679/r003679_18.jpg b/examples/r003679/r003679_18.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c9698617eb3d7ff4b065ea42e36229e65837f593
Binary files /dev/null and b/examples/r003679/r003679_18.jpg differ
diff --git a/examples/r003679/r003679_19.jpg b/examples/r003679/r003679_19.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b802f31f91a9f040a49b765733df01259b4db0a8
Binary files /dev/null and b/examples/r003679/r003679_19.jpg differ
diff --git a/examples/r003679/r003679_20.jpg b/examples/r003679/r003679_20.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..94b41220d2ea392c4dd3a1d2c261e2515cec0e09
Binary files /dev/null and b/examples/r003679/r003679_20.jpg differ
diff --git a/examples/r003679/r003679_21.jpg b/examples/r003679/r003679_21.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f256a09b0c86e31983dfecddab28882aef3e961a
Binary files /dev/null and b/examples/r003679/r003679_21.jpg differ
diff --git a/examples/r003679/r003679_22.jpg b/examples/r003679/r003679_22.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e24dfc5c31241a58c2164ca025f308acebc03d07
Binary files /dev/null and b/examples/r003679/r003679_22.jpg differ
diff --git a/examples/r003679/r003679_23.jpg b/examples/r003679/r003679_23.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..67070d870153d322463c9448786900eca8fab953
Binary files /dev/null and b/examples/r003679/r003679_23.jpg differ
diff --git a/examples/r004061.mp4 b/examples/r004061.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..a4a1236ff6f9d1dd2fa638b05bae9cb9f92eb502
--- /dev/null
+++ b/examples/r004061.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53fe5ccaf413f29978f030e8390f854c3e0bfb6b577ea42899c523190fbffdd9
+size 222005
diff --git a/examples/r004061/r004061_00.jpg b/examples/r004061/r004061_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a47add8e1eb89d405a87c887154a87c17ef9b174
Binary files /dev/null and b/examples/r004061/r004061_00.jpg differ
diff --git a/examples/r004061/r004061_01.jpg b/examples/r004061/r004061_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7017c21652c8790a87531bb404df56ca4d2aa93e
Binary files /dev/null and b/examples/r004061/r004061_01.jpg differ
diff --git a/examples/r004061/r004061_02.jpg b/examples/r004061/r004061_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..2cfd2ef1c23223a8f4f926c21d01959ed82fbb61
Binary files /dev/null and b/examples/r004061/r004061_02.jpg differ
diff --git a/examples/r004061/r004061_03.jpg b/examples/r004061/r004061_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d4839646b3ba6ede89850de85b592e01bd7baaac
Binary files /dev/null and b/examples/r004061/r004061_03.jpg differ
diff --git a/examples/r004061/r004061_04.jpg b/examples/r004061/r004061_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9dc79988798e650f39a558a60c14b1df24d0bd40
Binary files /dev/null and b/examples/r004061/r004061_04.jpg differ
diff --git a/examples/r004061/r004061_05.jpg b/examples/r004061/r004061_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0145894dfe2dd680b8c19d4b9b1511ed25e53cb1
Binary files /dev/null and b/examples/r004061/r004061_05.jpg differ
diff --git a/examples/r004061/r004061_06.jpg b/examples/r004061/r004061_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a87b66d85b54c3feb4d3b0e2422438c8490a2c68
Binary files /dev/null and b/examples/r004061/r004061_06.jpg differ
diff --git a/examples/r004061/r004061_07.jpg b/examples/r004061/r004061_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..a619e49949140981cfc998e8377be2e2c05058d5
Binary files /dev/null and b/examples/r004061/r004061_07.jpg differ
diff --git a/examples/r004061/r004061_08.jpg b/examples/r004061/r004061_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..889edafe42f9d409b51612a6890332f8baa3a4af
Binary files /dev/null and b/examples/r004061/r004061_08.jpg differ
diff --git a/examples/r004061/r004061_09.jpg b/examples/r004061/r004061_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..704e15428b14f5bf2e36275fb5e57a4e6142de55
Binary files /dev/null and b/examples/r004061/r004061_09.jpg differ
diff --git a/examples/r004061/r004061_10.jpg b/examples/r004061/r004061_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e02c45ed008d731490df5c0de6c0a82ab8321380
Binary files /dev/null and b/examples/r004061/r004061_10.jpg differ
diff --git a/examples/r004061/r004061_11.jpg b/examples/r004061/r004061_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e417ad2692153edcd94864470a115df3a417ae0d
Binary files /dev/null and b/examples/r004061/r004061_11.jpg differ
diff --git a/examples/r004061/r004061_12.jpg b/examples/r004061/r004061_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6200f62259e9060a6da20f73446b1785512b0f58
Binary files /dev/null and b/examples/r004061/r004061_12.jpg differ
diff --git a/examples/r004061/r004061_13.jpg b/examples/r004061/r004061_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ec53a7452269cddc6e30cea95e7f8de1310b4602
Binary files /dev/null and b/examples/r004061/r004061_13.jpg differ
diff --git a/examples/r004061/r004061_14.jpg b/examples/r004061/r004061_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..85dad13e0d1823fada8e7a4cde6ce81affb52600
Binary files /dev/null and b/examples/r004061/r004061_14.jpg differ
diff --git a/examples/r004061/r004061_15.jpg b/examples/r004061/r004061_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ea413d173af01a74e0edc3b437c9e3a2fb0e0aa1
Binary files /dev/null and b/examples/r004061/r004061_15.jpg differ
diff --git a/examples/r004061/r004061_16.jpg b/examples/r004061/r004061_16.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7d54c4fef94bb1140b79b3eaf789c36f3187e11f
Binary files /dev/null and b/examples/r004061/r004061_16.jpg differ
diff --git a/examples/r004061/r004061_17.jpg b/examples/r004061/r004061_17.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..fa37e7c49ac4a0de95d833d22e10c43cf70d0f49
Binary files /dev/null and b/examples/r004061/r004061_17.jpg differ
diff --git a/examples/r004061/r004061_18.jpg b/examples/r004061/r004061_18.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..7bffdcd0aff0758514ed89cf3cd3fdaac1669aec
Binary files /dev/null and b/examples/r004061/r004061_18.jpg differ
diff --git a/examples/r004061/r004061_19.jpg b/examples/r004061/r004061_19.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..d5dccfc0093db9b438cad70a5651502daee2d0a4
Binary files /dev/null and b/examples/r004061/r004061_19.jpg differ
diff --git a/examples/r004061/r004061_20.jpg b/examples/r004061/r004061_20.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..08c2311e26b1925dd64a26dbcd415d2d03aabffe
Binary files /dev/null and b/examples/r004061/r004061_20.jpg differ
diff --git a/examples/r004061/r004061_21.jpg b/examples/r004061/r004061_21.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..758d0a2febb3e05a40ade8d2245c3fd9e877c881
Binary files /dev/null and b/examples/r004061/r004061_21.jpg differ
diff --git a/examples/r004061/r004061_22.jpg b/examples/r004061/r004061_22.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b1e28baad6329c02c668295e660d6622e5c25ea5
Binary files /dev/null and b/examples/r004061/r004061_22.jpg differ
diff --git a/examples/r004061/r004061_23.jpg b/examples/r004061/r004061_23.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b1e28baad6329c02c668295e660d6622e5c25ea5
Binary files /dev/null and b/examples/r004061/r004061_23.jpg differ
diff --git a/examples/r100916.mp4 b/examples/r100916.mp4
new file mode 100644
index 0000000000000000000000000000000000000000..27d6783e5b47c9473445318af01d9c80ac3883ae
--- /dev/null
+++ b/examples/r100916.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aae7cac3bec9fd8dd349fb2546e4c986cfc5981bff815c82f97ad8df4603af3e
+size 55529
diff --git a/examples/r100916/r100916_00.jpg b/examples/r100916/r100916_00.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e18aac9c16b4bd00296df0b2c4e0a3307e3e1b2b
Binary files /dev/null and b/examples/r100916/r100916_00.jpg differ
diff --git a/examples/r100916/r100916_01.jpg b/examples/r100916/r100916_01.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..32463ad1e0923a113c1631c5e6b740cccd28dab5
Binary files /dev/null and b/examples/r100916/r100916_01.jpg differ
diff --git a/examples/r100916/r100916_02.jpg b/examples/r100916/r100916_02.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..f35c232d3c257405e8e9d034c991a8760e94e052
Binary files /dev/null and b/examples/r100916/r100916_02.jpg differ
diff --git a/examples/r100916/r100916_03.jpg b/examples/r100916/r100916_03.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..6480b13e17936b2cb2fda2befebec08014bfff82
Binary files /dev/null and b/examples/r100916/r100916_03.jpg differ
diff --git a/examples/r100916/r100916_04.jpg b/examples/r100916/r100916_04.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..60ec43490f713f540327fa05304701cac098fa49
Binary files /dev/null and b/examples/r100916/r100916_04.jpg differ
diff --git a/examples/r100916/r100916_05.jpg b/examples/r100916/r100916_05.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b8dd7173b391364f9bfd67d270039a700913c39c
Binary files /dev/null and b/examples/r100916/r100916_05.jpg differ
diff --git a/examples/r100916/r100916_06.jpg b/examples/r100916/r100916_06.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..18519793f073b8c60b58295661b746ad40f74964
Binary files /dev/null and b/examples/r100916/r100916_06.jpg differ
diff --git a/examples/r100916/r100916_07.jpg b/examples/r100916/r100916_07.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..e9c2f297559639f8c0e1faa5bac3f2f6daa4c3f6
Binary files /dev/null and b/examples/r100916/r100916_07.jpg differ
diff --git a/examples/r100916/r100916_08.jpg b/examples/r100916/r100916_08.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..95653b979331afda94896c052e4116eff50caee3
Binary files /dev/null and b/examples/r100916/r100916_08.jpg differ
diff --git a/examples/r100916/r100916_09.jpg b/examples/r100916/r100916_09.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..1468bb9beb88cf457106c75afd4021aae7dea1a6
Binary files /dev/null and b/examples/r100916/r100916_09.jpg differ
diff --git a/examples/r100916/r100916_10.jpg b/examples/r100916/r100916_10.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..648837bf730cbf7dcc2903f43f4c9be10117a939
Binary files /dev/null and b/examples/r100916/r100916_10.jpg differ
diff --git a/examples/r100916/r100916_11.jpg b/examples/r100916/r100916_11.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..cd1dd0e4dac4d6b03e2ab89f1ceaff72937c6312
Binary files /dev/null and b/examples/r100916/r100916_11.jpg differ
diff --git a/examples/r100916/r100916_12.jpg b/examples/r100916/r100916_12.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..237fdb80fecb2a5276e2c82be1f2f2f257a1df3a
Binary files /dev/null and b/examples/r100916/r100916_12.jpg differ
diff --git a/examples/r100916/r100916_13.jpg b/examples/r100916/r100916_13.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..dfe65cb8a1f4e9ecd61fac18285f1598557ca0d6
Binary files /dev/null and b/examples/r100916/r100916_13.jpg differ
diff --git a/examples/r100916/r100916_14.jpg b/examples/r100916/r100916_14.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b6981b3de284ade8b92c8c0a1c33f751c5ee5230
Binary files /dev/null and b/examples/r100916/r100916_14.jpg differ
diff --git a/examples/r100916/r100916_15.jpg b/examples/r100916/r100916_15.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c1d5dbe98e6031f2311dd9b3fff3d9b796767b24
Binary files /dev/null and b/examples/r100916/r100916_15.jpg differ
diff --git a/examples/test.ipynb b/examples/test.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..260b6ef78bb1a130befb99cb7e644b759991b615
--- /dev/null
+++ b/examples/test.ipynb
@@ -0,0 +1,580 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "FileNotFoundError",
+     "evalue": "[Errno 2] No such file or directory: 'path/to/your/images'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 31\u001b[0m\n\u001b[1;32m     28\u001b[0m output_video \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124moutput_video.mp4\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m     29\u001b[0m frame_rate \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m30\u001b[39m  \u001b[38;5;66;03m# Frames per second\u001b[39;00m\n\u001b[0;32m---> 31\u001b[0m \u001b[43mcreate_video_from_frames\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_directory\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_video\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mframe_rate\u001b[49m\u001b[43m)\u001b[49m\n",
+      "Cell \u001b[0;32mIn[1], line 6\u001b[0m, in \u001b[0;36mcreate_video_from_frames\u001b[0;34m(input_dir, output_file, frame_rate)\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcreate_video_from_frames\u001b[39m(input_dir, output_file, frame_rate):\n\u001b[1;32m      5\u001b[0m     \u001b[38;5;66;03m# Get all image files from the input directory\u001b[39;00m\n\u001b[0;32m----> 6\u001b[0m     images \u001b[38;5;241m=\u001b[39m [img \u001b[38;5;28;01mfor\u001b[39;00m img \u001b[38;5;129;01min\u001b[39;00m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlistdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_dir\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m img\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.png\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;129;01mor\u001b[39;00m img\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m.jpg\u001b[39m\u001b[38;5;124m\"\u001b[39m)]\n\u001b[1;32m      7\u001b[0m     images\u001b[38;5;241m.\u001b[39msort()  \u001b[38;5;66;03m# Ensure the images are sorted\u001b[39;00m\n\u001b[1;32m      9\u001b[0m     \u001b[38;5;66;03m# Read the first image to get the dimensions\u001b[39;00m\n",
+      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'path/to/your/images'"
+     ]
+    }
+   ],
+   "source": [
+    "import cv2\n",
+    "import os\n",
+    "\n",
+    "def create_video_from_frames(input_dir, output_file, frame_rate):\n",
+    "    # Get all image files from the input directory\n",
+    "    images = [img for img in os.listdir(input_dir) if img.endswith(\".png\") or img.endswith(\".jpg\")]\n",
+    "    images.sort()  # Ensure the images are sorted\n",
+    "\n",
+    "    # Read the first image to get the dimensions\n",
+    "    frame = cv2.imread(os.path.join(input_dir, images[0]))\n",
+    "    height, width, layers = frame.shape\n",
+    "\n",
+    "    # Initialize the video writer\n",
+    "    fourcc = cv2.VideoWriter_fourcc(*'mp4v')  # Codec for .mp4 file\n",
+    "    video = cv2.VideoWriter(output_file, fourcc, frame_rate, (width, height))\n",
+    "\n",
+    "    # Write each image to the video\n",
+    "    for image in images:\n",
+    "        img_path = os.path.join(input_dir, image)\n",
+    "        frame = cv2.imread(img_path)\n",
+    "        video.write(frame)\n",
+    "\n",
+    "    # Release the video writer\n",
+    "    video.release()\n",
+    "\n",
+    "# Example usage\n",
+    "input_directory = 'path/to/your/images'\n",
+    "output_video = 'output_video.mp4'\n",
+    "frame_rate = 30  # Frames per second\n",
+    "\n",
+    "create_video_from_frames(input_directory, output_video, frame_rate)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "with open(\"./all_subsets.json\", 'r') as f:\n",
+    "    examples = json.load(f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "15"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(examples)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'r003679_00.jpg'"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "examples[0]['images'][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "dog rolls over and falls off couch\n",
+      "Created video for r003679\n",
+      "A young man is standing in front of a glacier with mountains in the background.\n",
+      "Created video for r100916\n",
+      "a man wearing a striped blanket over his backpack runs behind two other men.\n",
+      "Created video for r004061\n",
+      "After a few years, Ghulam Muhammad became the owner of his business  \n",
+      "Created video for b404675\n",
+      "view down to the path in the field, moving forward, night, dark.  \n",
+      "Created video for b402727\n",
+      "Old Philippino man, close up, subtle smile, teats of joy, in old wood room  \n",
+      "Created video for b304986\n",
+      "write y in the garden with flowers  \n",
+      "Created video for d500937\n",
+      "a boy paiting a beautiful landscape  \n",
+      "Created video for d500506\n",
+      "sneaker with exclusive design worth 1 billion  \n",
+      "Created video for d401950\n",
+      ":sci fi war in the moon  \n",
+      "Created video for a500251\n",
+      "for a dog accessories brand a welcome image: brand name: hot doggies  \n",
+      "Created video for a400480\n",
+      "a druid in the forest cinematic  \n",
+      "Created video for a500010\n",
+      "a robot vacuum traveling through southeast asian countries  \n",
+      "Created video for 3005033\n",
+      "jiaxuan is the most beautiful ，elegant，charming girl in the world. billions of boys fell in love with her  \n",
+      "Created video for 7004180\n",
+      "scary halloween skeletons are having a conversation with each other  \n",
+      "Created video for 1006309\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "import os\n",
+    "import cv2\n",
+    "import regex as re\n",
+    "import shutil\n",
+    "for item in examples:\n",
+    "    video_id = item['images'][0].split(\"_\")[0]\n",
+    "    images = [os.path.join(video_id, x) for x in item['images']]\n",
+    "    output_file = f\"{video_id}.mp4\"\n",
+    "    frame = cv2.imread(images[0])\n",
+    "    if frame is None:\n",
+    "        print(f\"Skipping {video_id}\")\n",
+    "        continue\n",
+    "    height, width, layers = frame.shape\n",
+    "    item['video'] = output_file\n",
+    "    \n",
+    "    pattern = r'(?<=For this item, the text prompt is ).+(?=,)'\n",
+    "    prompt = re.findall(pattern, item['conversations'][0]['value'])[0]\n",
+    "    print(prompt)\n",
+    "    item['prompt'] = prompt.strip()\n",
+    "    \n",
+    "    # Initialize the video writer\n",
+    "    fourcc = cv2.VideoWriter_fourcc(*'mp4v')   # Codec for .mp4 file|\n",
+    "    # fourcc = cv2.VideoWriter_fourcc(*'avc3')   # Codec for .mp4 file|\n",
+    "    frame_rate = 8\n",
+    "    video = cv2.VideoWriter(output_file, fourcc, frame_rate, (width, height))\n",
+    "\n",
+    "    # Write each image to the video\n",
+    "    for image in images:\n",
+    "        frame = cv2.imread(image)\n",
+    "        video.write(frame)\n",
+    "    print(f\"Created video for {video_id}\")\n",
+    "        \n",
+    "    # shutil.rmtree(video_id)\n",
+    "\n",
+    "    # Release the video writer\n",
+    "    video.release()\n",
+    "    # break\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"./all_subsets.json\", 'w') as f:\n",
+    "    json.dump(examples, f, indent=4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'id': 'real_r003679',\n",
+       "  'images': ['r003679_00.jpg',\n",
+       "   'r003679_01.jpg',\n",
+       "   'r003679_02.jpg',\n",
+       "   'r003679_03.jpg',\n",
+       "   'r003679_04.jpg',\n",
+       "   'r003679_05.jpg',\n",
+       "   'r003679_06.jpg',\n",
+       "   'r003679_07.jpg',\n",
+       "   'r003679_08.jpg',\n",
+       "   'r003679_09.jpg',\n",
+       "   'r003679_10.jpg',\n",
+       "   'r003679_11.jpg',\n",
+       "   'r003679_12.jpg',\n",
+       "   'r003679_13.jpg',\n",
+       "   'r003679_14.jpg',\n",
+       "   'r003679_15.jpg',\n",
+       "   'r003679_16.jpg',\n",
+       "   'r003679_17.jpg',\n",
+       "   'r003679_18.jpg',\n",
+       "   'r003679_19.jpg',\n",
+       "   'r003679_20.jpg',\n",
+       "   'r003679_21.jpg',\n",
+       "   'r003679_22.jpg',\n",
+       "   'r003679_23.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is dog rolls over and falls off couch,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 3\\n object consistency: 3\\n dynamic degree: 3\\n motion smoothness: 3\\n text-to-video alignment: 3\\n factual consistency: 3\\n overall score: 3\\n'}]},\n",
+       " {'id': 'real_r100916',\n",
+       "  'images': ['r100916_00.jpg',\n",
+       "   'r100916_01.jpg',\n",
+       "   'r100916_02.jpg',\n",
+       "   'r100916_03.jpg',\n",
+       "   'r100916_04.jpg',\n",
+       "   'r100916_05.jpg',\n",
+       "   'r100916_06.jpg',\n",
+       "   'r100916_07.jpg',\n",
+       "   'r100916_08.jpg',\n",
+       "   'r100916_09.jpg',\n",
+       "   'r100916_10.jpg',\n",
+       "   'r100916_11.jpg',\n",
+       "   'r100916_12.jpg',\n",
+       "   'r100916_13.jpg',\n",
+       "   'r100916_14.jpg',\n",
+       "   'r100916_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is A young man is standing in front of a glacier with mountains in the background.,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 3\\n object consistency: 3\\n dynamic degree: 3\\n motion smoothness: 3\\n text-to-video alignment: 3\\n factual consistency: 3\\n overall score: 3\\n'}]},\n",
+       " {'id': 'real_r004061',\n",
+       "  'images': ['r004061_00.jpg',\n",
+       "   'r004061_01.jpg',\n",
+       "   'r004061_02.jpg',\n",
+       "   'r004061_03.jpg',\n",
+       "   'r004061_04.jpg',\n",
+       "   'r004061_05.jpg',\n",
+       "   'r004061_06.jpg',\n",
+       "   'r004061_07.jpg',\n",
+       "   'r004061_08.jpg',\n",
+       "   'r004061_09.jpg',\n",
+       "   'r004061_10.jpg',\n",
+       "   'r004061_11.jpg',\n",
+       "   'r004061_12.jpg',\n",
+       "   'r004061_13.jpg',\n",
+       "   'r004061_14.jpg',\n",
+       "   'r004061_15.jpg',\n",
+       "   'r004061_16.jpg',\n",
+       "   'r004061_17.jpg',\n",
+       "   'r004061_18.jpg',\n",
+       "   'r004061_19.jpg',\n",
+       "   'r004061_20.jpg',\n",
+       "   'r004061_21.jpg',\n",
+       "   'r004061_22.jpg',\n",
+       "   'r004061_23.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is a man wearing a striped blanket over his backpack runs behind two other men.,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 3\\n object consistency: 3\\n dynamic degree: 3\\n motion smoothness: 3\\n text-to-video alignment: 3\\n factual consistency: 3\\n overall score: 3\\n'}]},\n",
+       " {'id': 'worsen_gen_b404675',\n",
+       "  'images': ['b404675_00.jpg',\n",
+       "   'b404675_01.jpg',\n",
+       "   'b404675_02.jpg',\n",
+       "   'b404675_03.jpg',\n",
+       "   'b404675_04.jpg',\n",
+       "   'b404675_05.jpg',\n",
+       "   'b404675_06.jpg',\n",
+       "   'b404675_07.jpg',\n",
+       "   'b404675_08.jpg',\n",
+       "   'b404675_09.jpg',\n",
+       "   'b404675_10.jpg',\n",
+       "   'b404675_11.jpg',\n",
+       "   'b404675_12.jpg',\n",
+       "   'b404675_13.jpg',\n",
+       "   'b404675_14.jpg',\n",
+       "   'b404675_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is After a few years, Ghulam Muhammad became the owner of his business  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 1\\n object consistency: 1\\n dynamic degree: 3\\n motion smoothness: 3\\n text-to-video alignment: 2\\n factual consistency: 3\\n overall score: 1\\n'}]},\n",
+       " {'id': 'worsen_gen_b402727',\n",
+       "  'images': ['b402727_00.jpg',\n",
+       "   'b402727_01.jpg',\n",
+       "   'b402727_02.jpg',\n",
+       "   'b402727_03.jpg',\n",
+       "   'b402727_04.jpg',\n",
+       "   'b402727_05.jpg',\n",
+       "   'b402727_06.jpg',\n",
+       "   'b402727_07.jpg',\n",
+       "   'b402727_08.jpg',\n",
+       "   'b402727_09.jpg',\n",
+       "   'b402727_10.jpg',\n",
+       "   'b402727_11.jpg',\n",
+       "   'b402727_12.jpg',\n",
+       "   'b402727_13.jpg',\n",
+       "   'b402727_14.jpg',\n",
+       "   'b402727_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is view down to the path in the field, moving forward, night, dark.  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 1\\n object consistency: 1\\n dynamic degree: 3\\n motion smoothness: 3\\n text-to-video alignment: 3\\n factual consistency: 3\\n overall score: 1\\n'}]},\n",
+       " {'id': 'worsen_gen_b304986',\n",
+       "  'images': ['b304986_00.jpg',\n",
+       "   'b304986_01.jpg',\n",
+       "   'b304986_02.jpg',\n",
+       "   'b304986_03.jpg',\n",
+       "   'b304986_04.jpg',\n",
+       "   'b304986_05.jpg',\n",
+       "   'b304986_06.jpg',\n",
+       "   'b304986_07.jpg',\n",
+       "   'b304986_08.jpg',\n",
+       "   'b304986_09.jpg',\n",
+       "   'b304986_10.jpg',\n",
+       "   'b304986_11.jpg',\n",
+       "   'b304986_12.jpg',\n",
+       "   'b304986_13.jpg',\n",
+       "   'b304986_14.jpg',\n",
+       "   'b304986_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is Old Philippino man, close up, subtle smile, teats of joy, in old wood room  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 3\\n object consistency: 1\\n dynamic degree: 3\\n motion smoothness: 1\\n text-to-video alignment: 2\\n factual consistency: 3\\n overall score: 1\\n'}]},\n",
+       " {'id': 'static_d500937',\n",
+       "  'images': ['d500937_00.jpg',\n",
+       "   'd500937_01.jpg',\n",
+       "   'd500937_02.jpg',\n",
+       "   'd500937_03.jpg',\n",
+       "   'd500937_04.jpg',\n",
+       "   'd500937_05.jpg',\n",
+       "   'd500937_06.jpg',\n",
+       "   'd500937_07.jpg',\n",
+       "   'd500937_08.jpg',\n",
+       "   'd500937_09.jpg',\n",
+       "   'd500937_10.jpg',\n",
+       "   'd500937_11.jpg',\n",
+       "   'd500937_12.jpg',\n",
+       "   'd500937_13.jpg',\n",
+       "   'd500937_14.jpg',\n",
+       "   'd500937_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is write y in the garden with flowers  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 1\\n object consistency: 3\\n dynamic degree: 1\\n motion smoothness: 1\\n text-to-video alignment: 1\\n factual consistency: 1\\n overall score: 1\\n'}]},\n",
+       " {'id': 'static_d500506',\n",
+       "  'images': ['d500506_00.jpg',\n",
+       "   'd500506_01.jpg',\n",
+       "   'd500506_02.jpg',\n",
+       "   'd500506_03.jpg',\n",
+       "   'd500506_04.jpg',\n",
+       "   'd500506_05.jpg',\n",
+       "   'd500506_06.jpg',\n",
+       "   'd500506_07.jpg',\n",
+       "   'd500506_08.jpg',\n",
+       "   'd500506_09.jpg',\n",
+       "   'd500506_10.jpg',\n",
+       "   'd500506_11.jpg',\n",
+       "   'd500506_12.jpg',\n",
+       "   'd500506_13.jpg',\n",
+       "   'd500506_14.jpg',\n",
+       "   'd500506_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is a boy paiting a beautiful landscape  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 1\\n object consistency: 3\\n dynamic degree: 1\\n motion smoothness: 1\\n text-to-video alignment: 1\\n factual consistency: 1\\n overall score: 1\\n'}]},\n",
+       " {'id': 'static_d401950',\n",
+       "  'images': ['d401950_00.jpg',\n",
+       "   'd401950_01.jpg',\n",
+       "   'd401950_02.jpg',\n",
+       "   'd401950_03.jpg',\n",
+       "   'd401950_04.jpg',\n",
+       "   'd401950_05.jpg',\n",
+       "   'd401950_06.jpg',\n",
+       "   'd401950_07.jpg',\n",
+       "   'd401950_08.jpg',\n",
+       "   'd401950_09.jpg',\n",
+       "   'd401950_10.jpg',\n",
+       "   'd401950_11.jpg',\n",
+       "   'd401950_12.jpg',\n",
+       "   'd401950_13.jpg',\n",
+       "   'd401950_14.jpg',\n",
+       "   'd401950_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is sneaker with exclusive design worth 1 billion  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 1\\n object consistency: 3\\n dynamic degree: 1\\n motion smoothness: 1\\n text-to-video alignment: 1\\n factual consistency: 1\\n overall score: 1\\n'}]},\n",
+       " {'id': 'insf_a500251',\n",
+       "  'images': ['a500251_00.jpg',\n",
+       "   'a500251_01.jpg',\n",
+       "   'a500251_02.jpg',\n",
+       "   'a500251_03.jpg',\n",
+       "   'a500251_04.jpg',\n",
+       "   'a500251_05.jpg',\n",
+       "   'a500251_06.jpg',\n",
+       "   'a500251_07.jpg',\n",
+       "   'a500251_08.jpg',\n",
+       "   'a500251_09.jpg',\n",
+       "   'a500251_10.jpg',\n",
+       "   'a500251_11.jpg',\n",
+       "   'a500251_12.jpg',\n",
+       "   'a500251_13.jpg',\n",
+       "   'a500251_14.jpg',\n",
+       "   'a500251_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is :sci fi war in the moon  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 1\\n object consistency: 1\\n dynamic degree: 3\\n motion smoothness: 1\\n text-to-video alignment: 1\\n factual consistency: 1\\n overall score: 1\\n'}]},\n",
+       " {'id': 'insf_a400480',\n",
+       "  'images': ['a400480_00.jpg',\n",
+       "   'a400480_01.jpg',\n",
+       "   'a400480_02.jpg',\n",
+       "   'a400480_03.jpg',\n",
+       "   'a400480_04.jpg',\n",
+       "   'a400480_05.jpg',\n",
+       "   'a400480_06.jpg',\n",
+       "   'a400480_07.jpg',\n",
+       "   'a400480_08.jpg',\n",
+       "   'a400480_09.jpg',\n",
+       "   'a400480_10.jpg',\n",
+       "   'a400480_11.jpg',\n",
+       "   'a400480_12.jpg',\n",
+       "   'a400480_13.jpg',\n",
+       "   'a400480_14.jpg',\n",
+       "   'a400480_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is for a dog accessories brand a welcome image: brand name: hot doggies  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 1\\n object consistency: 1\\n dynamic degree: 3\\n motion smoothness: 1\\n text-to-video alignment: 1\\n factual consistency: 1\\n overall score: 1\\n'}]},\n",
+       " {'id': 'insf_a500010',\n",
+       "  'images': ['a500010_00.jpg',\n",
+       "   'a500010_01.jpg',\n",
+       "   'a500010_02.jpg',\n",
+       "   'a500010_03.jpg',\n",
+       "   'a500010_04.jpg',\n",
+       "   'a500010_05.jpg',\n",
+       "   'a500010_06.jpg',\n",
+       "   'a500010_07.jpg',\n",
+       "   'a500010_08.jpg',\n",
+       "   'a500010_09.jpg',\n",
+       "   'a500010_10.jpg',\n",
+       "   'a500010_11.jpg',\n",
+       "   'a500010_12.jpg',\n",
+       "   'a500010_13.jpg',\n",
+       "   'a500010_14.jpg',\n",
+       "   'a500010_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is a druid in the forest cinematic  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 1\\n object consistency: 1\\n dynamic degree: 3\\n motion smoothness: 1\\n text-to-video alignment: 1\\n factual consistency: 1\\n overall score: 1\\n'}]},\n",
+       " {'id': 'lab_3005033',\n",
+       "  'images': ['3005033_00.jpg',\n",
+       "   '3005033_01.jpg',\n",
+       "   '3005033_02.jpg',\n",
+       "   '3005033_03.jpg',\n",
+       "   '3005033_04.jpg',\n",
+       "   '3005033_05.jpg',\n",
+       "   '3005033_06.jpg',\n",
+       "   '3005033_07.jpg',\n",
+       "   '3005033_08.jpg',\n",
+       "   '3005033_09.jpg',\n",
+       "   '3005033_10.jpg',\n",
+       "   '3005033_11.jpg',\n",
+       "   '3005033_12.jpg',\n",
+       "   '3005033_13.jpg',\n",
+       "   '3005033_14.jpg',\n",
+       "   '3005033_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is a robot vacuum traveling through southeast asian countries  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 3\\n object consistency: 2\\n dynamic degree: 3\\n motion smoothness: 2\\n text-to-video alignment: 2\\n factual consistency: 3\\n overall score: 2\\n'}]},\n",
+       " {'id': 'lab_7004180',\n",
+       "  'images': ['7004180_00.jpg',\n",
+       "   '7004180_01.jpg',\n",
+       "   '7004180_02.jpg',\n",
+       "   '7004180_03.jpg',\n",
+       "   '7004180_04.jpg',\n",
+       "   '7004180_05.jpg',\n",
+       "   '7004180_06.jpg',\n",
+       "   '7004180_07.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is jiaxuan is the most beautiful ，elegant，charming girl in the world. billions of boys fell in love with her  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 3\\n object consistency: 3\\n dynamic degree: 3\\n motion smoothness: 3\\n text-to-video alignment: 3\\n factual consistency: 3\\n overall score: 3\\n'}]},\n",
+       " {'id': 'lab_1006309',\n",
+       "  'images': ['1006309_00.jpg',\n",
+       "   '1006309_01.jpg',\n",
+       "   '1006309_02.jpg',\n",
+       "   '1006309_03.jpg',\n",
+       "   '1006309_04.jpg',\n",
+       "   '1006309_05.jpg',\n",
+       "   '1006309_06.jpg',\n",
+       "   '1006309_07.jpg',\n",
+       "   '1006309_08.jpg',\n",
+       "   '1006309_09.jpg',\n",
+       "   '1006309_10.jpg',\n",
+       "   '1006309_11.jpg',\n",
+       "   '1006309_12.jpg',\n",
+       "   '1006309_13.jpg',\n",
+       "   '1006309_14.jpg',\n",
+       "   '1006309_15.jpg'],\n",
+       "  'conversations': [{'from': 'human',\n",
+       "    'value': \"\\nSuppose you are an expert in judging and evaluating the quality of AI-generated videos, \\nplease watch the following frames of a given video and see the text prompt for generating the video, \\nthen give scores from 7 different dimensions:\\n(1) visual quality, \\n(2) object consistency,\\n(3) dynamic degree,\\n(4) motion smoothness,\\n(5) text-to-video alignment,\\n(6) factual consistency, \\n(7) overall score\\nfor each dimension, output a number from [1,2,3], in which '1' stands for 'Bad', '2' stands for 'Average', '3' stands for 'Good'.\\nHere is an output example: \\nvisual quality: 3\\nobject consistency: 2 \\ndynamic degree: 2\\nmotion smoothness: 1\\ntext-to-video alignment: 1\\nfactual consistency: 2\\noverall score: 1\\n\\nFor this item, the text prompt is scary halloween skeletons are having a conversation with each other  ,\\nall the frames of video are as follows: \\n\\n<image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> <image> \\n\"},\n",
+       "   {'from': 'gpt',\n",
+       "    'value': 'visual quality: 3\\n object consistency: 2\\n dynamic degree: 3\\n motion smoothness: 2\\n text-to-video alignment: 3\\n factual consistency: 2\\n overall score: 3\\n'}]}]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "examples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "e"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "mantis",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.19"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/models/__init__.py b/models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/models/conversation.py b/models/conversation.py
new file mode 100644
index 0000000000000000000000000000000000000000..6a84cc61e56d21c81d53de10223b64fde3708c13
--- /dev/null
+++ b/models/conversation.py
@@ -0,0 +1,473 @@
+import dataclasses
+from enum import auto, Enum
+from typing import List, Tuple
+
+
+class SeparatorStyle(Enum):
+    """Different separator style."""
+    SINGLE = auto()
+    TWO = auto()
+    MPT = auto()
+    PLAIN = auto()
+    LLAMA_2 = auto()
+    LLAMA_3 = auto()
+    IDEFICS_2 = auto()
+    MFUYU = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that keeps all conversation history."""
+    system: str
+    roles: List[str]
+    messages: List[List[str]]
+    offset: int
+    sep_style: SeparatorStyle = SeparatorStyle.SINGLE
+    sep: str = "###"
+    sep2: str = None
+    version: str = "Unknown"
+
+    skip_next: bool = False
+
+    def get_prompt(self):
+        messages = self.messages
+        if len(messages) > 0 and type(messages[0][1]) is tuple:
+            
+            messages = self.messages.copy()
+            init_role, init_msg = messages[0].copy()
+            init_msg = init_msg[0].replace("<image>", "").strip()
+            if 'mmtag' in self.version:
+                messages[0] = (init_role, init_msg)
+                messages.insert(0, (self.roles[0], "<Image><image></Image>"))
+                messages.insert(1, (self.roles[1], "Received."))
+            else:
+                messages[0] = (init_role, "<image>" + init_msg)
+        if self.sep_style == SeparatorStyle.SINGLE:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + self.sep
+                else:
+                    ret += role + ":"
+        elif self.sep_style == SeparatorStyle.TWO:
+            seps = [self.sep, self.sep2]
+            ret = self.system + seps[0]
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+        elif self.sep_style == SeparatorStyle.MPT:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+        elif self.sep_style == SeparatorStyle.LLAMA_2:
+            wrap_sys = lambda msg: f"<<SYS>>\n{msg}\n<</SYS>>\n\n"
+            wrap_inst = lambda msg: f"[INST] {msg} [/INST]"
+            ret = ""
+
+            for i, (role, message) in enumerate(messages):
+                if i == 0:
+                    assert message, "first message should not be none"
+                    assert role == self.roles[0], "first message should come from user"
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    if i == 0: message = wrap_sys(self.system) + message
+                    if i % 2 == 0:
+                        message = wrap_inst(message)
+                        ret += self.sep + message
+                    else:
+                        ret += " " + message + " " + self.sep2
+                else:
+                    ret += ""
+            ret = ret.lstrip(self.sep)
+        elif self.sep_style == SeparatorStyle.LLAMA_3:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += f"<|start_header_id|>{role}<|end_header_id|>\n\n" + message + self.sep
+                else:
+                    ret += f"<|start_header_id|>{role}<|end_header_id|>\n\n"
+        elif self.sep_style == SeparatorStyle.MFUYU:
+            seps = [self.sep, self.sep2]
+            ret = self.system + "\n"
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+        elif self.sep_style == SeparatorStyle.PLAIN:
+            seps = [self.sep, self.sep2]
+            ret = self.system
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += message + seps[i % 2]
+                else:
+                    ret += ""
+        elif self.sep_style == SeparatorStyle.IDEFICS_2:
+            if self.system:
+                ret = self.system + self.sep
+            else:
+                ret = ""
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ":" + message + self.sep + "\n"
+                else:
+                    ret += role + ":"
+        else:
+            raise ValueError(f"Invalid style: {self.sep_style}")
+
+        return ret
+
+    def append_message(self, role, message):
+        self.messages.append([role, message])
+
+    def get_images(self, return_pil=False):
+        images = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    import base64
+                    from io import BytesIO
+                    from PIL import Image
+                    msg, image, image_process_mode = msg
+                    if image_process_mode == "Pad":
+                        def expand2square(pil_img, background_color=(122, 116, 104)):
+                            width, height = pil_img.size
+                            if width == height:
+                                return pil_img
+                            elif width > height:
+                                result = Image.new(pil_img.mode, (width, width), background_color)
+                                result.paste(pil_img, (0, (width - height) // 2))
+                                return result
+                            else:
+                                result = Image.new(pil_img.mode, (height, height), background_color)
+                                result.paste(pil_img, ((height - width) // 2, 0))
+                                return result
+                        image = expand2square(image)
+                    elif image_process_mode in ["Default", "Crop"]:
+                        pass
+                    elif image_process_mode == "Resize":
+                        image = image.resize((336, 336))
+                    else:
+                        raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
+                    max_hw, min_hw = max(image.size), min(image.size)
+                    aspect_ratio = max_hw / min_hw
+                    max_len, min_len = 800, 400
+                    shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+                    longest_edge = int(shortest_edge * aspect_ratio)
+                    W, H = image.size
+                    if longest_edge != max(image.size):
+                        if H > W:
+                            H, W = longest_edge, shortest_edge
+                        else:
+                            H, W = shortest_edge, longest_edge
+                        image = image.resize((W, H))
+                    if return_pil:
+                        images.append(image)
+                    else:
+                        buffered = BytesIO()
+                        image.save(buffered, format="PNG")
+                        img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+                        images.append(img_b64_str)
+        return images
+
+    def to_gradio_chatbot(self):
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    import base64
+                    from io import BytesIO
+                    msg, image, image_process_mode = msg
+                    max_hw, min_hw = max(image.size), min(image.size)
+                    aspect_ratio = max_hw / min_hw
+                    max_len, min_len = 800, 400
+                    shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+                    longest_edge = int(shortest_edge * aspect_ratio)
+                    W, H = image.size
+                    if H > W:
+                        H, W = longest_edge, shortest_edge
+                    else:
+                        H, W = shortest_edge, longest_edge
+                    image = image.resize((W, H))
+                    buffered = BytesIO()
+                    image.save(buffered, format="JPEG")
+                    img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+                    img_str = f'<img src="data:image/png;base64,{img_b64_str}" alt="user upload image" />'
+                    msg = img_str + msg.replace('<image>', '').strip()
+                    ret.append([msg, None])
+                else:
+                    ret.append([msg, None])
+            else:
+                ret[-1][-1] = msg
+        return ret
+
+    def copy(self):
+        return Conversation(
+            system=self.system,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            version=self.version)
+
+    def dict(self):
+        if len(self.get_images()) > 0:
+            return {
+                "system": self.system,
+                "roles": self.roles,
+                "messages": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages],
+                "offset": self.offset,
+                "sep": self.sep,
+                "sep2": self.sep2,
+            }
+        return {
+            "system": self.system,
+            "roles": self.roles,
+            "messages": self.messages,
+            "offset": self.offset,
+            "sep": self.sep,
+            "sep2": self.sep2,
+        }
+
+
+conv_vicuna_v0 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("Human", "Assistant"),
+    messages=(
+        ("Human", "What are the key differences between renewable and non-renewable energy sources?"),
+        ("Assistant",
+            "Renewable energy sources are those that can be replenished naturally in a relatively "
+            "short amount of time, such as solar, wind, hydro, geothermal, and biomass. "
+            "Non-renewable energy sources, on the other hand, are finite and will eventually be "
+            "depleted, such as coal, oil, and natural gas. Here are some key differences between "
+            "renewable and non-renewable energy sources:\n"
+            "1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable "
+            "energy sources are finite and will eventually run out.\n"
+            "2. Environmental impact: Renewable energy sources have a much lower environmental impact "
+            "than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, "
+            "and other negative effects.\n"
+            "3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically "
+            "have lower operational costs than non-renewable sources.\n"
+            "4. Reliability: Renewable energy sources are often more reliable and can be used in more remote "
+            "locations than non-renewable sources.\n"
+            "5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different "
+            "situations and needs, while non-renewable sources are more rigid and inflexible.\n"
+            "6. Sustainability: Renewable energy sources are more sustainable over the long term, while "
+            "non-renewable sources are not, and their depletion can lead to economic and social instability.\n")
+    ),
+    offset=2,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+)
+
+conv_vicuna_v1 = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+)
+
+conv_llama_2 = Conversation(
+    system="""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_2,
+    sep="<s>",
+    sep2="</s>",
+)
+
+conv_llava_llama_2 = Conversation(
+    system="You are a helpful language and vision assistant. "
+           "You are able to understand the visual content that the user provides, "
+           "and assist the user with a variety of tasks using natural language.",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_2,
+    sep="<s>",
+    sep2="</s>",
+)
+
+conv_mpt = Conversation(
+    system="""<|im_start|>system
+A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""",
+    roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
+    version="mpt",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.MPT,
+    sep="<|im_end|>",
+)
+
+conv_llava_plain = Conversation(
+    system="",
+    roles=("", ""),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.PLAIN,
+    sep="\n",
+)
+
+conv_llava_v0 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("Human", "Assistant"),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+)
+
+conv_llava_v0_mmtag = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+           "The visual content will be provided with the following format: <Image>visual content</Image>.",
+    roles=("Human", "Assistant"),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+    version="v0_mmtag",
+)
+
+conv_llava_v1 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+)
+
+conv_llava_v1_mmtag = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+           "The visual content will be provided with the following format: <Image>visual content</Image>.",
+    roles=("USER", "ASSISTANT"),
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+    version="v1_mmtag",
+)
+
+conv_mfuyu_v1 = Conversation(
+    system="You are a helpful language and vision assistant. "
+           "You are able to understand the visual content that the user provides, "
+           "and assist the user with a variety of tasks using natural language.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.MFUYU,
+    sep="<0x04>", # begin of answer token
+    sep2="|ENDOFTEXT|",
+) # copied from conv_vicuna_v1
+    
+conv_mllava_v1_mmtag = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant is able to understand the multiple visual contents that the user provides, and assist the user with a variety of tasks using natural language."
+           "Each visual content will be provided with the following format: <Image>visual content</Image>.",
+    roles=("USER", "ASSISTANT"),
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="</s>",
+    version="v1_mmtag",
+)
+
+conv_mllava_v1 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="</s>",
+)
+
+conv_llama_3 = Conversation(
+    system="<|start_header_id|>system<|end_header_id|>\n\nYou are a pirate chatbot who always responds in pirate speak!",
+    roles=("user", "assistant"),
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_3,
+    sep="<|eot_id|>",
+)
+
+conv_idefics_2 = Conversation(
+    system="",
+    roles=("User", "Assistant"),
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.IDEFICS_2,
+    sep="<end_of_utterance>",
+)
+
+default_conversation = conv_mfuyu_v1
+conv_templates = {
+    "default": conv_vicuna_v0,
+    "v0": conv_vicuna_v0,
+    "v1": conv_vicuna_v1,
+    "vicuna_v1": conv_vicuna_v1,
+    "llama_2": conv_llama_2,
+
+    "plain": conv_llava_plain,
+    "v0_plain": conv_llava_plain,
+    "llava_v0": conv_llava_v0,
+    "v0_mmtag": conv_llava_v0_mmtag,
+    "llava_v1": conv_llava_v1,
+    "v1_mmtag": conv_llava_v1_mmtag,
+    "llava_llama_2": conv_llava_llama_2,
+    "llama_3": conv_llama_3,
+    "mllava_v1": conv_mllava_v1,
+    "mllava_v1_mmtag": conv_mllava_v1_mmtag,
+    "idefics_2": conv_idefics_2,
+
+    "mpt": conv_mpt,
+}
+
+
+if __name__ == "__main__":
+    print(default_conversation.get_prompt())
\ No newline at end of file
diff --git a/models/idefics2/__init__.py b/models/idefics2/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb140fe1b130a2cd5f927b81d5d5e9350934b862
--- /dev/null
+++ b/models/idefics2/__init__.py
@@ -0,0 +1 @@
+from .modeling_idefics2 import Idefics2ForConditionalGeneration, Idefics2ForSequenceClassification
\ No newline at end of file
diff --git a/models/idefics2/modeling_idefics2.py b/models/idefics2/modeling_idefics2.py
new file mode 100644
index 0000000000000000000000000000000000000000..b84a2ba53f2a88ebae3a8ca44d0c8efb213332fa
--- /dev/null
+++ b/models/idefics2/modeling_idefics2.py
@@ -0,0 +1,2309 @@
+# coding=utf-8
+# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch Idefics2 model."""
+
+import inspect
+import math
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+
+# from ... import PreTrainedModel
+# from ...activations import ACT2FN
+# from ...cache_utils import Cache, DynamicCache
+# from ...modeling_attn_mask_utils import _prepare_4d_attention_mask
+# from ...modeling_outputs import BaseModelOutput, ModelOutput
+# from ...utils import (
+#     add_start_docstrings,
+#     add_start_docstrings_to_model_forward,
+#     is_flash_attn_2_available,
+#     is_flash_attn_greater_or_equal_2_10,
+#     logging,
+#     replace_return_docstrings,
+# )
+# from ..auto import AutoModel
+# from .configuration_idefics2 import Idefics2Config, Idefics2VisionConfig
+
+from transformers import PreTrainedModel
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
+from transformers.modeling_outputs import BaseModelOutput, ModelOutput, SequenceClassifierOutputWithPast
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    is_flash_attn_2_available,
+    is_flash_attn_greater_or_equal_2_10,
+    logging,
+    replace_return_docstrings,
+)
+from transformers.models.auto import AutoModel
+from transformers.models.idefics2.configuration_idefics2 import Idefics2Config, Idefics2VisionConfig
+
+
+if is_flash_attn_2_available():
+    from flash_attn import flash_attn_func, flash_attn_varlen_func
+    from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
+
+    _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters)
+
+
+logger = logging.get_logger(__name__)
+
+_CONFIG_FOR_DOC = "Idefics2Config"
+
+IDEFICS2_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "HuggingFaceM4/idefics2-8b",
+    # See all IDEFICS2 models at https://huggingface.co/models?filter=idefics2
+]
+
+
+@dataclass
+class Idefics2BaseModelOutputWithPast(ModelOutput):
+    """
+    Base class for Idefics2 model's outputs that may also contain a past key/values (to speed up sequential decoding).
+    Args:
+        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+            If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+            hidden_size)` is output.
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
+            `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads,
+            encoder_sequence_length, embed_size_per_head)`.
+            Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
+            `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
+            input) to speed up sequential decoding.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
+            Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
+            sequence_length, hidden_size)`.
+            image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
+    """
+
+    last_hidden_state: torch.FloatTensor = None
+    past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+    image_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+
+
+@dataclass
+# Copied from transformers.models.idefics.modeling_idefics.IdeficsCausalLMOutputWithPast with Idefics->Idefics2
+class Idefics2CausalLMOutputWithPast(ModelOutput):
+    """
+    Base class for Idefics2 causal language model (or autoregressive) outputs.
+    Args:
+        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+            Language modeling loss (for next-token prediction).
+        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
+            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
+            `past_key_values` input) to speed up sequential decoding.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
+            Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
+            sequence_length, hidden_size)`.
+            image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
+    """
+
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    past_key_values: Optional[List[torch.FloatTensor]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+    image_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+
+
+class Idefics2VisionEmbeddings(nn.Module):
+    """
+    This is a modified version of `siglip.modelign_siglip.SiglipVisionEmbeddings` to enable images of variable
+    resolution.
+
+    The modifications are adapted from [Patch n' Pack: NaViT, a Vision Transformer for any Aspect Ratio and Resolution](https://arxiv.org/abs/2307.06304)
+    which allows treating images in their native aspect ratio and without the need to resize them to the same
+    fixed size. In particular, we start from the original pre-trained SigLIP model
+    (which uses images of fixed-size square images) and adapt it by training on images of variable resolutions.
+    """
+
+    def __init__(self, config: Idefics2VisionConfig):
+        super().__init__()
+        self.embed_dim = config.hidden_size
+        self.image_size = config.image_size
+        self.patch_size = config.patch_size
+
+        self.patch_embedding = nn.Conv2d(
+            in_channels=config.num_channels,
+            out_channels=self.embed_dim,
+            kernel_size=self.patch_size,
+            stride=self.patch_size,
+            padding="valid",
+        )
+
+        self.num_patches_per_side = self.image_size // self.patch_size
+        self.num_patches = self.num_patches_per_side**2
+        self.num_positions = self.num_patches
+        self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
+
+    def forward(self, pixel_values: torch.FloatTensor, patch_attention_mask: torch.BoolTensor) -> torch.Tensor:
+        batch_size, _, max_im_h, max_im_w = pixel_values.shape
+
+        patch_embeds = self.patch_embedding(pixel_values)
+        embeddings = patch_embeds.flatten(2).transpose(1, 2)
+
+        max_nb_patches_h, max_nb_patches_w = max_im_h // self.patch_size, max_im_w // self.patch_size
+        boundaries = torch.arange(1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side)
+        position_ids = torch.full(size=(batch_size, max_nb_patches_h * max_nb_patches_w), fill_value=0)
+
+        for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
+            nb_patches_h = p_attn_mask[:, 0].sum()
+            nb_patches_w = p_attn_mask[0].sum()
+
+            fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
+            fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
+
+            bucket_coords_h = torch.bucketize(fractional_coords_h, boundaries, right=True)
+            bucket_coords_w = torch.bucketize(fractional_coords_w, boundaries, right=True)
+
+            pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w).flatten()
+            position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
+
+        position_ids = position_ids.to(self.position_embedding.weight.device)
+        embeddings = embeddings + self.position_embedding(position_ids)
+        return embeddings
+
+
+# Copied from transformers.models.siglip.modeling_siglip.SiglipAttention with Siglip->Idefics2Vision
+class Idefics2VisionAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+        self.scale = self.head_dim**-0.5
+        self.dropout = config.attention_dropout
+
+        self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
+        self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
+
+        # Ignore copy
+        self.is_causal = False
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        """Input shape: Batch x Time x Channel"""
+
+        batch_size, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(batch_size, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        k_v_seq_len = key_states.shape[-2]
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) * self.scale
+
+        if attn_weights.size() != (batch_size, self.num_heads, q_len, k_v_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(batch_size, 1, q_len, k_v_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (batch_size, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
+
+        attn_output = self.out_proj(attn_output)
+
+        return attn_output, attn_weights
+
+
+class Idefics2VisionFlashAttention2(Idefics2VisionAttention):
+    """
+    Idefics2Vision flash attention module. This module inherits from `Idefics2VisionAttention` as the weights of the module stays
+    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
+    flash attention and deal with padding tokens in case the input contains any of them.
+    """
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
+        # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
+        # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
+        self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        output_attentions = False
+
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        # Flash attention requires the input to have the shape
+        # batch_size x seq_length x head_dim x hidden_dim
+        # therefore we just need to keep the original shape
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+
+        # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
+        # to be able to avoid many of these transpose/reshape/view.
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        dropout_rate = self.dropout if self.training else 0.0
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in the correct dtype just to be sure everything works as expected.
+        # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+        # in fp32. (Idefics2VisionRMSNorm handles it correctly)
+
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+
+            logger.warning_once(
+                f"The input hidden states seems to be silently casted in float32, this might be related to"
+                f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+
+        attn_output = self._flash_attention_forward(
+            query_states, key_states, value_states, attention_mask, q_len, dropout=dropout_rate
+        )
+
+        attn_output = attn_output.reshape(bsz, q_len, self.embed_dim).contiguous()
+        attn_output = self.out_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._flash_attention_forward
+    def _flash_attention_forward(
+        self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`float`):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+        """
+        if not self._flash_attn_uses_top_left_mask:
+            causal = self.is_causal
+        else:
+            # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
+            causal = self.is_causal and query_length != 1
+
+        # Contains at least one padding token in the sequence
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            attn_output_unpad = flash_attn_varlen_func(
+                query_states,
+                key_states,
+                value_states,
+                cu_seqlens_q=cu_seqlens_q,
+                cu_seqlens_k=cu_seqlens_k,
+                max_seqlen_q=max_seqlen_in_batch_q,
+                max_seqlen_k=max_seqlen_in_batch_k,
+                dropout_p=dropout,
+                softmax_scale=softmax_scale,
+                causal=causal,
+            )
+
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            attn_output = flash_attn_func(
+                query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
+            )
+
+        return attn_output
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._upad_input
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+
+        key_layer = index_first_axis(
+            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        value_layer = index_first_axis(
+            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+
+IDEFICS_VISION_ATTENTION_CLASSES = {
+    "eager": Idefics2VisionAttention,
+    "flash_attention_2": Idefics2VisionFlashAttention2,
+}
+
+
+# Copied from transformers.models.siglip.modeling_siglip.SiglipMLP with Siglip->Idefics2Vision
+class Idefics2VisionMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+class Idefics2MLP(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        intermediate_size: int,
+        output_size: int,
+        hidden_act: str,
+    ):
+        super().__init__()
+        self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
+        self.down_proj = nn.Linear(intermediate_size, output_size, bias=False)
+        self.act_fn = ACT2FN[hidden_act]
+
+    def forward(self, x):
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+
+# Copied from transformers.models.siglip.modeling_siglip.SiglipMultiheadAttentionPoolingHead with Siglip->Idefics2
+class Idefics2MultiheadAttentionPoolingHead(nn.Module):
+    """Multihead Attention Pooling."""
+
+    def __init__(self, config: Idefics2VisionConfig):
+        super().__init__()
+
+        self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size))
+        self.attention = torch.nn.MultiheadAttention(config.hidden_size, config.num_attention_heads, batch_first=True)
+        self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        # Ignore copy
+        self.mlp = Idefics2MLP(
+            hidden_size=config.hidden_size,
+            intermediate_size=config.intermediate_size,
+            hidden_act=config.hidden_act,
+            output_size=config.hidden_size,
+        )
+
+    def forward(self, hidden_state):
+        batch_size = hidden_state.shape[0]
+        probe = self.probe.repeat(batch_size, 1, 1)
+
+        hidden_state = self.attention(probe, hidden_state, hidden_state)[0]
+
+        residual = hidden_state
+        hidden_state = self.layernorm(hidden_state)
+        hidden_state = residual + self.mlp(hidden_state)
+
+        return hidden_state[:, 0]
+
+
+class Idefics2EncoderLayer(nn.Module):
+    def __init__(self, config: Idefics2Config):
+        super().__init__()
+        self.embed_dim = config.hidden_size
+        self.self_attn = IDEFICS_VISION_ATTENTION_CLASSES[config._attn_implementation](config)
+        self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+        self.mlp = Idefics2VisionMLP(config)
+        self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
+
+    # Copied from transformers.models.siglip.modeling_siglip.SiglipEncoderLayer.forward
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: torch.Tensor,
+        output_attentions: Optional[bool] = False,
+    ) -> Tuple[torch.FloatTensor]:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`):
+                Input to the layer of shape `(batch, seq_len, embed_dim)`.
+            attention_mask (`torch.FloatTensor`):
+                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
+            output_attentions (`bool`, *optional*, defaults to `False`):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+        """
+        residual = hidden_states
+
+        hidden_states = self.layer_norm1(hidden_states)
+        hidden_states, attn_weights = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+        )
+        hidden_states = residual + hidden_states
+
+        residual = hidden_states
+        hidden_states = self.layer_norm2(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (attn_weights,)
+
+        return outputs
+
+
+# Copied from transformers.models.siglip.modeling_siglip.SiglipEncoder with Siglip->Idefics2
+class Idefics2Encoder(nn.Module):
+    """
+    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
+    [`Idefics2EncoderLayer`].
+
+    Args:
+        config: Idefics2Config
+    """
+
+    def __init__(self, config: Idefics2Config):
+        super().__init__()
+        self.config = config
+        self.layers = nn.ModuleList([Idefics2EncoderLayer(config) for _ in range(config.num_hidden_layers)])
+        self.gradient_checkpointing = False
+
+    # Ignore copy
+    def forward(
+        self,
+        inputs_embeds,
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutput]:
+        r"""
+        Args:
+            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+                Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+                This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+                than the model's internal embedding lookup matrix.
+            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+                - 1 for tokens that are **not masked**,
+                - 0 for tokens that are **masked**.
+
+                [What are attention masks?](../glossary#attention-mask)
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        hidden_states = inputs_embeds
+        for encoder_layer in self.layers:
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    encoder_layer.__call__,
+                    hidden_states,
+                    attention_mask,
+                    output_attentions,
+                )
+            else:
+                layer_outputs = encoder_layer(
+                    hidden_states,
+                    attention_mask,
+                    output_attentions=output_attentions,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+        )
+
+
+class Idefics2VisionTransformer(nn.Module):
+    def __init__(self, config: Idefics2VisionConfig):
+        super().__init__()
+        embed_dim = config.hidden_size
+
+        self.config = config
+        self.embeddings = Idefics2VisionEmbeddings(config)
+        self.encoder = Idefics2Encoder(config)
+        self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+
+    def get_input_embeddings(self):
+        return self.embeddings
+
+    def set_input_embeddings(self, value):
+        self.embeddings = value
+
+    def forward(
+        self,
+        pixel_values,
+        patch_attention_mask: Optional[torch.BoolTensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutput]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        batch_size = pixel_values.size(0)
+        if patch_attention_mask is None:
+            patch_size = self.config.patch_size
+            patch_attention_mask = torch.ones(
+                (
+                    batch_size,
+                    pixel_values.size(2) // patch_size,
+                    pixel_values.size(3) // patch_size,
+                )
+            )
+            patch_attention_mask = patch_attention_mask.to(dtype=torch.bool, device=pixel_values.device)
+
+        hidden_states = self.embeddings(pixel_values=pixel_values, patch_attention_mask=patch_attention_mask)
+
+        patch_attention_mask = patch_attention_mask.view(batch_size, -1)
+        # The call to `_upad_input` in `_flash_attention_forward` is expensive
+        # So when the `patch_attention_mask` is full of 1s (i.e. attending to the whole sequence),
+        # avoiding passing the attention_mask, which is equivalent to attending to the full sequence
+        if not torch.any(~patch_attention_mask):
+            patch_attention_mask = None
+        elif not self._use_flash_attention_2:
+            patch_attention_mask = _prepare_4d_attention_mask(patch_attention_mask, hidden_states.dtype)
+
+        encoder_outputs = self.encoder(
+            inputs_embeds=hidden_states,
+            attention_mask=patch_attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        last_hidden_state = encoder_outputs[0]
+        last_hidden_state = self.post_layernorm(last_hidden_state)
+
+        if not return_dict:
+            return (last_hidden_state,) + encoder_outputs[1:]
+
+        return BaseModelOutput(
+            last_hidden_state=last_hidden_state,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )
+
+
+# Copied from transformers.models.llama.modeling_llama.repeat_kv
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Idefics2
+class Idefics2RMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        """
+        Idefics2RMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+
+class Idefics2PerceiverAttention(nn.Module):
+    def __init__(self, config, layer_idx: Optional[int] = None) -> None:
+        """Perceiver Cross-Attention Module --> let long-form inputs be `context`, resampled embeddings be `latents`"""
+        super().__init__()
+
+        self.layer_idx = None
+        self.hidden_size = config.text_config.hidden_size
+        self.num_heads = config.perceiver_config.resampler_n_heads
+        self.head_dim = config.perceiver_config.resampler_head_dim
+        self.num_key_value_heads = config.perceiver_config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.attention_dropout = config.perceiver_config.attention_dropout
+
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+
+        self.is_causal = False
+
+    def forward(
+        self,
+        latents: torch.Tensor,
+        context: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        """
+        Runs Perceiver Self-Attention, with special (context, latents) appended along the `seq` dimension!
+
+        Args:
+            latents (`torch.Tensor`): Tensor of shape [bsz, n_latents, embed_dim] representing fixed length latents to compress to.
+            context (`torch.Tensor`): Tensor of shape [bsz, seq, embed_dim] representing long-form context to resample.
+            attention_mask (`torch.Tensor`, *optional*): Tensor of shape [bsz, 1, seq, n_latents] representing attention mask.
+            position_ids (`torch.LongTensor`, *optional*): Tensor of shape [bsz, seq] representing position indices of each input token.
+            past_key_value (`Tuple[torch.Tensor]`, *optional*): Tuple of tensors containing cached key and value states.
+            output_attentions (`bool`, *optional*, defaults to `False`): Whether to return attention weights.
+            use_cache (`bool`, *optional*, defaults to `False`): Whether to use past_key_value for caching.
+        """
+        bsz, q_len, _ = latents.size()
+        kv_seq_len = q_len + context.size()[1]
+
+        hidden_states = torch.concat([context, latents], dim=-2)
+
+        query_states = self.q_proj(latents)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, kv_seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, kv_seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        past_key_value = getattr(self, "past_key_value", past_key_value)
+
+        if past_key_value is not None:
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx)
+
+        # repeat k/v heads if n_kv_heads < n_heads
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+
+            attn_weights = attn_weights + attention_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim)
+
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+# Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2 with MistralAttention->Idefics2PerceiverAttention,MistralFlashAttention->Idefics2PerceiverFlashAttention,Mistral->Idefics2
+class Idefics2PerceiverFlashAttention2(Idefics2PerceiverAttention):
+    """
+    Idefics2 flash attention module. This module inherits from `Idefics2PerceiverAttention` as the weights of the module stays
+    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
+    flash attention and deal with padding tokens in case the input contains any of them.
+    """
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
+        # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
+        # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
+        self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
+
+    # Ignore copy
+    def forward(
+        self,
+        latents: torch.Tensor,
+        context: torch.Tensor,
+        attention_mask: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: bool = False,
+        use_cache: bool = False,
+        **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        bsz, q_len, _ = latents.size()
+        kv_seq_len = q_len + context.size()[1]
+
+        # Query, Key, Value Projections --> Note that in Flamingo, latents are *concatenated* with context prior to attn!
+        #   Note: This results in queries w/ `seq = n_latents`, and keys, values with `seq = len(context) + n_latents`
+        query_states = self.q_proj(latents)
+        key_states = self.k_proj(torch.cat([context, latents], dim=-2))
+        value_states = self.v_proj(torch.cat([context, latents], dim=-2))
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, kv_seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, kv_seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value[0].shape[-2]
+
+        if past_key_value is not None:
+            # Activate slicing cache only if the config has a value `sliding_windows` attribute
+            if hasattr(self.config, "sliding_window") and kv_seq_len > self.config.sliding_window:
+                slicing_tokens = kv_seq_len - self.config.sliding_window
+
+                past_key = past_key_value[0]
+                past_value = past_key_value[1]
+
+                past_key = past_key[:, :, slicing_tokens:, :].contiguous()
+                past_value = past_value[:, :, slicing_tokens:, :].contiguous()
+
+                if past_key.shape[-2] != self.config.sliding_window - 1:
+                    raise ValueError(
+                        "past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1,"
+                        f" head_dim`), got {past_key.shape}"
+                    )
+
+                past_key_value = (past_key, past_value)
+
+                if attention_mask is not None:
+                    attention_mask = attention_mask[:, slicing_tokens:]
+                    attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1)
+
+            key_states = torch.cat([past_key_value[0], key_states], dim=2)
+            value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+        past_key_value = (key_states, value_states) if use_cache else None
+
+        # repeat k/v heads if n_kv_heads < n_heads
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+        dropout_rate = 0.0 if not self.training else self.attention_dropout
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in float16 just to be sure everything works as expected.
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+
+            logger.warning_once(
+                f"The input hidden states seems to be silently casted in float32, this might be related to"
+                f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+
+        # Reashape to the expected shape for Flash Attention
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        attn_output = self._flash_attention_forward(
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            q_len,
+            dropout=dropout_rate,
+            use_sliding_windows=False,
+        )
+
+        attn_output = attn_output.reshape(bsz, q_len, self.num_heads * self.head_dim).contiguous()
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+    def _flash_attention_forward(
+        self,
+        query_states,
+        key_states,
+        value_states,
+        attention_mask,
+        query_length,
+        dropout=0.0,
+        softmax_scale=None,
+        use_sliding_windows=False,
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`float`):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+            use_sliding_windows (`bool`, *optional*):
+                Whether to activate sliding window attention.
+        """
+        if not self._flash_attn_uses_top_left_mask:
+            causal = self.is_causal
+        else:
+            # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
+            causal = self.is_causal and query_length != 1
+
+        # Contains at least one padding token in the sequence
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            if not use_sliding_windows:
+                attn_output_unpad = flash_attn_varlen_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    cu_seqlens_q=cu_seqlens_q,
+                    cu_seqlens_k=cu_seqlens_k,
+                    max_seqlen_q=max_seqlen_in_batch_q,
+                    max_seqlen_k=max_seqlen_in_batch_k,
+                    dropout_p=dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                )
+            else:
+                attn_output_unpad = flash_attn_varlen_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    cu_seqlens_q=cu_seqlens_q,
+                    cu_seqlens_k=cu_seqlens_k,
+                    max_seqlen_q=max_seqlen_in_batch_q,
+                    max_seqlen_k=max_seqlen_in_batch_k,
+                    dropout_p=dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                    window_size=(self.config.sliding_window, self.config.sliding_window),
+                )
+
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            if not use_sliding_windows:
+                attn_output = flash_attn_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                )
+            else:
+                attn_output = flash_attn_func(
+                    query_states,
+                    key_states,
+                    value_states,
+                    dropout,
+                    softmax_scale=softmax_scale,
+                    causal=causal,
+                    window_size=(self.config.sliding_window, self.config.sliding_window),
+                )
+
+        return attn_output
+
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape
+
+        # On the first iteration we need to properly re-create the padding mask
+        # by slicing it on the proper place
+        if kv_seq_len != attention_mask.shape[-1]:
+            attention_mask_num_tokens = attention_mask.shape[-1]
+            attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :]
+
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+
+        key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
+        value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
+
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+
+IDEFICS2_PERCEIVER_ATTENTION_CLASSES = {
+    "eager": Idefics2PerceiverAttention,
+    "flash_attention_2": Idefics2PerceiverFlashAttention2,
+}
+
+
+class Idefics2PerceiverLayer(nn.Module):
+    def __init__(self, config, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.text_config.hidden_size
+        self.n_latents = config.perceiver_config.resampler_n_latents
+        self.depth = config.perceiver_config.resampler_depth
+        self.rms_norm_eps = config.text_config.rms_norm_eps
+
+        self.input_latents_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps)
+        self.input_context_norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps)
+        self.self_attn = IDEFICS2_PERCEIVER_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx)
+        self.post_attention_layernorm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps)
+        self.mlp = Idefics2MLP(
+            hidden_size=config.text_config.hidden_size,
+            intermediate_size=config.text_config.hidden_size * 4,
+            output_size=config.text_config.hidden_size,
+            hidden_act=config.perceiver_config.hidden_act,
+        )
+
+    def forward(
+        self,
+        latents: torch.Tensor,
+        context: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Tuple[torch.Tensor]] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        **kwargs,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        """
+        Args:
+            latents (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            context (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
+                `(batch, sequence_length)` where padding elements are indicated by 0.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            use_cache (`bool`, *optional*):
+                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+                (see `past_key_values`).
+            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+        """
+        residual = latents
+
+        latents = self.input_latents_norm(latents)
+        context = self.input_context_norm(context)
+
+        latents, self_attn_weights, present_key_value = self.self_attn(
+            latents=latents,
+            context=context,
+            attention_mask=attention_mask,
+        )
+        latents = residual + latents
+        residual = latents
+
+        latents = self.post_attention_layernorm(latents)
+        latents = self.mlp(latents)
+        latents = residual + latents
+
+        outputs = (latents,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        return outputs
+
+
+class Idefics2PerceiverResampler(nn.Module):
+    def __init__(self, config) -> None:
+        """
+        Instantiates a Perceiver Resampler that operates over a sequence of embeddings (say from a ResNet or ViT or
+        MAE) of a given dimension, performs `depth` blocks of cross-attention with a fixed `n_latents` inputs, then
+        returns a Tensor of shape [bsz, n_latents, embed_dim]. The Resampler acts as a form of learned pooling and
+        is derived from [Perceiver: General Perception with Iterative Attention](https://arxiv.org/abs/2103.03206).
+        """
+        super().__init__()
+        self.hidden_size = config.text_config.hidden_size
+        self.hidden_act = config.perceiver_config.hidden_act
+        self.n_latents = config.perceiver_config.resampler_n_latents
+        self.depth = config.perceiver_config.resampler_depth
+        self.rms_norm_eps = config.text_config.rms_norm_eps
+
+        # Create Latents for Perceiver
+        self.latents = nn.Parameter(torch.ones(self.n_latents, self.hidden_size))
+
+        # Create Transformer Blocks
+        self.layers = nn.ModuleList([Idefics2PerceiverLayer(config, idx) for idx in range(self.depth)])
+        self.norm = Idefics2RMSNorm(self.hidden_size, eps=self.rms_norm_eps)
+
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+
+    def forward(
+        self,
+        context: torch.Tensor,
+        attention_mask,
+    ) -> torch.Tensor:
+        # seq embed -> bsz seq embed
+        latents = self.latents.unsqueeze(0).expand((context.shape[0], *self.latents.size()))
+
+        latent_attention_mask = torch.ones(
+            (attention_mask.size(0), latents.size(1)), dtype=attention_mask.dtype, device=attention_mask.device
+        )
+        attention_mask = torch.cat([attention_mask, latent_attention_mask], dim=-1)
+        attention_mask = (
+            _prepare_4d_attention_mask(attention_mask, latents.dtype, tgt_len=self.n_latents)
+            if not self._use_flash_attention_2
+            else attention_mask
+        )
+
+        compressed_context = latents
+        for perceiver_layer in self.layers:
+            layer_outputs = perceiver_layer(
+                compressed_context,
+                context,
+                attention_mask=attention_mask,
+                position_ids=None,
+                past_key_value=None,
+                output_attentions=False,
+                use_cache=False,
+            )
+
+            compressed_context = layer_outputs[0]
+
+        compressed_context = self.norm(compressed_context)
+
+        return compressed_context
+
+
+class Idefics2Connector(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.modality_projection = Idefics2MLP(
+            hidden_size=config.vision_config.hidden_size,
+            intermediate_size=config.text_config.intermediate_size,
+            output_size=config.text_config.hidden_size,
+            hidden_act=config.text_config.hidden_act,
+        )
+        self.perceiver_resampler = Idefics2PerceiverResampler(config)
+
+    def forward(self, image_hidden_states, attention_mask):
+        image_hidden_states = self.modality_projection(image_hidden_states)
+        image_hidden_states = self.perceiver_resampler(context=image_hidden_states, attention_mask=attention_mask)
+        return image_hidden_states
+
+
+IDEFICS2_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+
+    Parameters:
+        config ([`Idefics2Config`] or [`Idefics2VisionConfig`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+    "The bare Idefics2 Model outputting raw hidden-states without any specific head on top.",
+    IDEFICS2_START_DOCSTRING,
+)
+class Idefics2PreTrainedModel(PreTrainedModel):
+    config_class = Idefics2Config
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Idefics2VisionAttention", "Idefics2MLP", "Idefics2PerceiverLayer", "Idefics2DecoderLayer"]
+    _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+
+    def _init_weights(self, module):
+        # important: this ported version of Idefics2 isn't meant for training from scratch - only
+        # inference and fine-tuning - so the proper init weights code has been removed - the original codebase
+        # https://github.com/haotian-liu/LLaVA/tree/main/idefics2 should serve for that purpose
+        std = (
+            self.config.text_config.initializer_range
+            if hasattr(self.config, "initializer_range")
+            else self.config.text_config.initializer_range
+        )
+
+        if hasattr(module, "class_embedding"):
+            module.class_embedding.data.normal_(mean=0.0, std=std)
+
+        if isinstance(module, (nn.Linear, nn.Conv2d)):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+    @classmethod
+    def _autoset_attn_implementation(
+        cls,
+        config,
+        use_flash_attention_2: bool = False,
+        torch_dtype: Optional[torch.dtype] = None,
+        device_map: Optional[Union[str, Dict[str, int]]] = None,
+        check_device_map: bool = True,
+        **kwargs,
+    ):
+        """
+        Overrides the method in `PreTrainedModel` to update the vision config with the correct attention implementation
+        """
+        config = super()._autoset_attn_implementation(
+            config=config,
+            use_flash_attention_2=use_flash_attention_2,
+            torch_dtype=torch_dtype,
+            device_map=device_map,
+            check_device_map=check_device_map,
+            **kwargs,
+        )
+        config.vision_config._attn_implementation = config._attn_implementation
+        return config
+
+
+IDEFICS2_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+            it.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+
+            [What are attention masks?](../glossary#attention-mask)
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
+            `past_key_values`).
+
+            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+            information on the default strategy.
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
+
+            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
+            The tensors corresponding to the input images. Pixel values can be obtained using
+            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses
+            [`CLIPImageProcessor`] for processing images).
+        pixel_attention_mask (`torch.Tensor` of shape `(batch_size, image_size, image_size)`, *optional*):
+            Mask to avoid performing attention on padding pixel indices.
+        image_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)`):
+            The hidden states of the image encoder after modality projection and perceiver resampling.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+    """Idefics2 model consisting of a SIGLIP vision encoder and Mistral language decoder""",
+    IDEFICS2_START_DOCSTRING,
+)
+class Idefics2Model(Idefics2PreTrainedModel):
+    def __init__(self, config: Idefics2Config):
+        super().__init__(config)
+        self.padding_idx = self.config.text_config.pad_token_id
+        self.vocab_size = self.config.text_config.vocab_size
+
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+        
+        config.vision_config._attn_implementation = config._attn_implementation
+        self.vision_model = Idefics2VisionTransformer(config.vision_config)
+        self.connector = Idefics2Connector(config)
+        self.text_model = AutoModel.from_config(config.text_config, attn_implementation=config._attn_implementation) # Dongfu: add attn_implementation for text_model
+
+        self.image_seq_len = config.perceiver_config.resampler_n_latents
+        self.image_token_id = self.config.image_token_id
+
+
+        self.post_init()
+
+    def enable_input_require_grads(self):
+        """
+        Enables the gradients for the input embeddings.
+
+        This is useful for lora when using gradient checkpointing.
+        c.f. https://github.com/huggingface/peft/issues/1402#issuecomment-1913675032
+
+        Override to set output.requires_grad = True for both the decoder's and vision model's embeddings.
+        """
+
+        def get_lowest_module(module):
+            if len(list(module.children())) == 0:
+                # If the module has no children, it is a leaf module (e.g., Linear, Conv2d, etc.)
+                return module
+            else:
+                # Recursively call the function on each child module
+                return get_lowest_module(list(module.children())[0])
+
+        def make_inputs_require_grads(module, input, output):
+            output.requires_grad_(True)
+
+        self._text_require_grads_hook = self.get_input_embeddings().register_forward_hook(make_inputs_require_grads)
+        self._vision_require_grads_hook = get_lowest_module(self.vision_model).register_forward_hook(
+            make_inputs_require_grads
+        )
+
+    def get_input_embeddings(self):
+        return self.text_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.text_model.set_input_embeddings(value)
+
+    def resize_token_embeddings(self, new_num_tokens: Optional[int] = None, pad_to_multiple_of=None) -> nn.Embedding:
+        model_embeds = self.text_model.resize_token_embeddings(
+            new_num_tokens=new_num_tokens, pad_to_multiple_of=pad_to_multiple_of
+        )
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        return model_embeds
+
+    def inputs_merger(
+        self,
+        input_ids: torch.LongTensor,
+        inputs_embeds: Optional[torch.Tensor],
+        image_hidden_states: Optional[torch.Tensor],
+    ):
+        """
+        This method aims at merging the token embeddings with the image hidden states into one single sequence of vectors that are fed to the transformer LM.
+        The merging happens as follows:
+        - The text token sequence is: `tok_1 tok_2 tok_3 <fake_token_around_image> <image> <image> ... <image> <fake_token_around_image> tok_4`.
+        - We get the image hidden states for the image through the vision encoder (and potentially the perceiver), and that hidden state is then projected into the text embedding space.
+        We thus have a sequence of image hidden states of size (1, image_seq_len, hidden_dim), where 1 is for batch_size of 1 image and hidden_dim is the hidden_dim of the LM transformer.
+        - The merging happens so that we obtain the following sequence: `vector_tok_1 vector_tok_2 vector_tok_3 vector_fake_tok_around_image {sequence of image_seq_len image hidden states} vector_fake_toke_around_image vector_tok_4`. That sequence is fed to the LM.
+        - To fit the format of that sequence, `input_ids`, `input_embeds`, `attention_mask` are all 3 adapted to insert the image hidden states.
+        """
+        num_images, _, vision_hidden_size = image_hidden_states.shape
+        special_image_token_mask = input_ids == self.image_token_id
+        new_inputs_embeds = inputs_embeds.clone()
+        reshaped_image_hidden_states = image_hidden_states.view(-1, vision_hidden_size)
+        new_inputs_embeds[special_image_token_mask] = reshaped_image_hidden_states
+        return new_inputs_embeds
+
+    @add_start_docstrings_to_model_forward(
+        """
+        Inputs fed to the model can have an arbitrary number of images. To account for this, pixel_values fed to
+        the model have image padding -> (batch_size, max_num_images, 3, max_heights, max_widths) where
+        max_num_images is the maximum number of images among the batch_size samples in the batch.
+
+        Padding images are not needed beyond padding the pixel_values at the entrance of the model.
+        For efficiency, we only pass through the vision_model's forward the real images by
+        discarding the padding images i.e. pixel_values of size (image_batch_size, 3, height, width) where
+        image_batch_size would be 7 when num_images_per_sample=[1, 3, 1, 2] and max_num_images would be 3.
+        """,
+        IDEFICS2_INPUTS_DOCSTRING,
+    )
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        pixel_attention_mask: Optional[torch.BoolTensor] = None,
+        image_hidden_states: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, Idefics2BaseModelOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if self.training and self.text_model.gradient_checkpointing and use_cache:
+            logger.warning_once(
+                "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+            )
+            use_cache = False
+            
+        # retrieve input_ids and inputs_embeds
+        if input_ids is not None:
+            batch_size, seq_length = input_ids.shape
+        elif inputs_embeds is not None:
+            batch_size, seq_length, _ = inputs_embeds.shape
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        past_seen_tokens = 0
+        if use_cache:
+            if not isinstance(past_key_values, Cache):
+                past_key_values = DynamicCache.from_legacy_cache(past_key_values)
+            past_seen_tokens = past_key_values.get_usable_length(seq_length)
+
+        if inputs_embeds is not None and input_ids is None and past_seen_tokens == 0:
+            raise ValueError("When first calling the model, if input_embeds are passed, input_ids should not be None.")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.text_model.get_input_embeddings()(input_ids)
+
+        # START VISUAL INPUTS INTEGRATION
+        if pixel_values is not None and image_hidden_states is not None:
+            raise ValueError("You cannot specify both pixel_values and image_hidden_states at the same time")
+        elif pixel_values is not None:
+            batch_size, num_images, num_channels, height, width = pixel_values.shape
+            pixel_values = pixel_values.to(dtype=self.dtype)  # fp16 compatibility
+            pixel_values = pixel_values.view(batch_size * num_images, *pixel_values.shape[2:])
+
+            # Remove padding images - padding images are full 0.
+            nb_values_per_image = pixel_values.shape[1:].numel()
+            real_images_inds = (pixel_values == 0.0).sum(dim=(-1, -2, -3)) != nb_values_per_image
+            pixel_values = pixel_values[real_images_inds].contiguous()
+
+            # Handle the vision attention mask
+            if pixel_attention_mask is None:
+                pixel_attention_mask = torch.ones(
+                    size=(pixel_values.size(0), pixel_values.size(2), pixel_values.size(3)),
+                    dtype=torch.bool,
+                    device=pixel_values.device,
+                )
+            else:
+                # Remove padding images from the mask/pP p
+                pixel_attention_mask = pixel_attention_mask.view(
+                    batch_size * num_images, *pixel_attention_mask.shape[2:]
+                )
+                pixel_attention_mask = pixel_attention_mask[real_images_inds].contiguous()
+
+            patch_size = self.config.vision_config.patch_size
+            patches_subgrid = pixel_attention_mask.unfold(dimension=1, size=patch_size, step=patch_size)
+            patches_subgrid = patches_subgrid.unfold(dimension=2, size=patch_size, step=patch_size)
+            patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
+
+            # Get sequence from the vision encoder
+            pixel_batch_size = 4
+            all_image_hidden_states = []
+            for i in range(0, pixel_values.size(0), pixel_batch_size):
+                batch_pixel_values = pixel_values[i : i + pixel_batch_size]
+                batch_patch_attention_mask = patch_attention_mask[i : i + pixel_batch_size]
+                
+                batch_image_hidden_states = self.vision_model(
+                    pixel_values=batch_pixel_values,
+                    patch_attention_mask=batch_patch_attention_mask,
+                ).last_hidden_state
+                
+                batch_image_hidden_states = self.connector(
+                    batch_image_hidden_states, attention_mask=batch_patch_attention_mask.view(batch_pixel_values.size(0), -1)
+                )
+                all_image_hidden_states.append(batch_image_hidden_states)
+                
+            image_hidden_states = torch.cat(all_image_hidden_states, dim=0)
+            
+            # image_hidden_states = self.vision_model(
+            #     pixel_values=pixel_values,
+            #     patch_attention_mask=patch_attention_mask,
+            # ).last_hidden_state
+
+            # # Modality projection & resampling
+            # image_hidden_states = self.connector(
+            #     image_hidden_states, attention_mask=patch_attention_mask.view(pixel_values.size(0), -1)
+            # )
+
+        elif image_hidden_states is not None:
+            image_hidden_states = image_hidden_states.to(dtype=self.dtype, device=input_ids.device)
+
+        if past_seen_tokens == 0 and inputs_embeds is not None and image_hidden_states is not None:
+            # When we generate, we don't want to replace the potential image_token_id that we generated by images
+            # that simply don't exist
+            inputs_embeds = self.inputs_merger(
+                input_ids=input_ids,
+                inputs_embeds=inputs_embeds,
+                image_hidden_states=image_hidden_states,
+            )
+
+        outputs = self.text_model(
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        if not return_dict:
+            return tuple(v for v in [*outputs, image_hidden_states] if v is not None)
+
+        return Idefics2BaseModelOutputWithPast(
+            last_hidden_state=outputs.last_hidden_state,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            image_hidden_states=image_hidden_states,
+        )
+
+
+@add_start_docstrings(
+    """The Idefics2 Model with a language modeling head. It is made up a SigLIP vision encoder, with a language modeling head on top. """,
+    IDEFICS2_START_DOCSTRING,
+)
+class Idefics2ForConditionalGeneration(Idefics2PreTrainedModel):
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = Idefics2Model(config)
+        self.image_token_id = self.config.image_token_id
+
+        self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
+        self.vocab_size = config.text_config.vocab_size
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def enable_input_require_grads(self):
+        """
+        Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping
+        the model weights fixed.
+        """
+
+        def make_inputs_require_grads(module, input, output):
+            output.requires_grad_(True)
+
+        self._text_require_grads_hook = self.get_input_embeddings().register_forward_hook(make_inputs_require_grads)
+        self._vision_require_grads_hook = self.model.vision_model.get_input_embeddings().register_forward_hook(
+            make_inputs_require_grads
+        )
+
+    def get_input_embeddings(self):
+        return self.model.text_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.text_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def resize_token_embeddings(self, new_num_tokens: Optional[int] = None, pad_to_multiple_of=None) -> nn.Embedding:
+        # model_embeds = self.model.resize_token_embeddings(new_num_tokens=new_num_tokens, pad_to_multiple_of=pad_to_multiple_of)
+        model_embeds = self._resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
+        if new_num_tokens is None and pad_to_multiple_of is None:
+            return model_embeds
+
+        # Update base model and current model config
+        # Ignore copy
+        self.config.text_config.vocab_size = model_embeds.weight.shape[0]
+        self.vocab_size = self.config.text_config.vocab_size
+
+        # Tie weights again if needed
+        self.tie_weights()
+
+        return model_embeds
+
+    def tie_weights(self):
+        """
+        Overwrite `transformers.modeling_utils.PreTrainedModel.tie_weights` to handle the case of DecoupledLinear and DecoupledEmbedding.
+        """
+        output_embeddings = self.get_output_embeddings()
+        input_embeddings = self.get_input_embeddings()
+
+        if getattr(self.config, "tie_word_embeddings", True):
+            output_embeddings.weight = input_embeddings.weight
+
+    @add_start_docstrings_to_model_forward(IDEFICS2_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=Idefics2CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        pixel_attention_mask: Optional[torch.BoolTensor] = None,
+        image_hidden_states: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, Idefics2CausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Returns:
+
+        Example:
+
+        ```python
+        >>> import requests
+        >>> import torch
+        >>> from PIL import Image
+        >>> from io import BytesIO
+
+        >>> from transformers import AutoProcessor, AutoModelForVision2Seq
+        >>> from transformers.image_utils import load_image
+
+        >>> # Note that passing the image urls (instead of the actual pil images) to the processor is also possible
+        >>> image1 = load_image("https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg")
+        >>> image2 = load_image("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg")
+        >>> image3 = load_image("https://cdn.britannica.com/68/170868-050-8DDE8263/Golden-Gate-Bridge-San-Francisco.jpg")
+
+        >>> processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics2-8b-base")
+        >>> model = AutoModelForVision2Seq.from_pretrained("HuggingFaceM4/idefics2-8b-base", device_map="auto")
+
+        >>> BAD_WORDS_IDS = processor.tokenizer(["<image>", "<fake_token_around_image>"], add_special_tokens=False).input_ids
+        >>> EOS_WORDS_IDS = [processor.tokenizer.eos_token_id]
+
+        >>> # Create inputs
+        >>> prompts = [
+        ...   "<image>In this image, we can see the city of New York, and more specifically the Statue of Liberty.<image>In this image,",
+        ...   "In which city is that bridge located?<image>",
+        ... ]
+        >>> images = [[image1, image2], [image3]]
+        >>> inputs = processor(text=prompts, padding=True, return_tensors="pt").to("cuda")
+
+        >>> # Generate
+        >>> generated_ids = model.generate(**inputs, bad_words_ids=BAD_WORDS_IDS, max_new_tokens=20)
+        >>> generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=True)
+
+        >>> print(generated_texts)
+        ['In this image, we can see the city of New York, and more specifically the Statue of Liberty. In this image, we can see the city of New York, and more specifically the Statue of Liberty.\n\n', 'In which city is that bridge located?\n\nThe bridge is located in the city of Pittsburgh, Pennsylvania.\n\n\nThe bridge is']
+        ```"""
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            pixel_values=pixel_values,
+            pixel_attention_mask=pixel_attention_mask,
+            image_hidden_states=image_hidden_states,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+            # Shift so that tokens < n predict n
+            if attention_mask is not None:
+                shift_attention_mask = attention_mask[..., 1:].to(logits.device)
+                shift_logits = logits[..., :-1, :][shift_attention_mask != 0].contiguous()
+                shift_labels = labels[..., 1:][shift_attention_mask != 0].contiguous()
+            else:
+                shift_logits = logits[..., :-1, :].contiguous()
+                shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss(ignore_index=self.image_token_id)
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return Idefics2CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            image_hidden_states=outputs.image_hidden_states,
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        # Omit tokens covered by past_key_values
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.seen_tokens
+                max_cache_length = past_key_values.get_max_length()
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+                max_cache_length = None
+
+            # Keep only the unprocessed tokens:
+            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+            # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+            # input)
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+            # input_ids based on the past_length.
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+
+            # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
+            if (
+                max_cache_length is not None
+                and attention_mask is not None
+                and cache_length + input_ids.shape[1] > max_cache_length
+            ):
+                attention_mask = attention_mask[:, -max_cache_length:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        image_hidden_states = kwargs.get("image_hidden_states", None)
+        if image_hidden_states is not None:
+            pixel_values = None
+            pixel_attention_mask = None
+        else:
+            pixel_values = kwargs.get("pixel_values", None)
+            pixel_attention_mask = kwargs.get("pixel_attention_mask", None)
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "pixel_values": pixel_values,
+                "pixel_attention_mask": pixel_attention_mask,
+                "image_hidden_states": image_hidden_states,
+            }
+        )
+        return model_inputs
+
+    def _update_model_kwargs_for_generation(self, outputs, model_kwargs, is_encoder_decoder, **kwargs):
+        model_kwargs = super()._update_model_kwargs_for_generation(
+            outputs=outputs,
+            model_kwargs=model_kwargs,
+            is_encoder_decoder=is_encoder_decoder,
+            **kwargs,
+        )
+        # Get the precomputed image_hidden_states
+        model_kwargs["image_hidden_states"] = outputs.image_hidden_states
+        return model_kwargs
+
+    @staticmethod
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM._reorder_cache
+    def _reorder_cache(past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (
+                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
+            )
+        return reordered_past
+
+@add_start_docstrings(
+    """The Idefics2 Model transformer with a sequence classification head on top (linear layer). 
+    It is made up a SigLIP vision encoder, with a language modeling head on top. 
+    
+    [`Idefics2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+    (e.g. GPT-2) do.
+
+    Since it does classification on the last token, it requires to know the position of the last token. If a
+    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+    each row of the batch).
+    """,
+    IDEFICS2_START_DOCSTRING,
+)
+class Idefics2ForSequenceClassification(Idefics2PreTrainedModel):
+    _tied_weights_keys = ["score.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.model = Idefics2Model(config)
+        self.image_token_id = self.config.image_token_id
+
+        self.score = nn.Linear(config.text_config.hidden_size, self.num_labels)
+        self.vocab_size = config.text_config.vocab_size
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def enable_input_require_grads(self):
+        """
+        Enables the gradients for the input embeddings. This is useful for fine-tuning adapter weights while keeping
+        the model weights fixed.
+        """
+
+        def make_inputs_require_grads(module, input, output):
+            output.requires_grad_(True)
+
+        self._text_require_grads_hook = self.get_input_embeddings().register_forward_hook(make_inputs_require_grads)
+        self._vision_require_grads_hook = self.model.vision_model.get_input_embeddings().register_forward_hook(
+            make_inputs_require_grads
+        )
+
+    def get_input_embeddings(self):
+        return self.model.text_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.text_model.set_input_embeddings(value)
+
+    def resize_token_embeddings(self, new_num_tokens: Optional[int] = None, pad_to_multiple_of=None) -> nn.Embedding:
+        # model_embeds = self.model.resize_token_embeddings(new_num_tokens=new_num_tokens, pad_to_multiple_of=pad_to_multiple_of)
+        model_embeds = self._resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
+        if new_num_tokens is None and pad_to_multiple_of is None:
+            return model_embeds
+
+        # Update base model and current model config
+        # Ignore copy
+        self.config.text_config.vocab_size = model_embeds.weight.shape[0]
+        self.vocab_size = self.config.text_config.vocab_size
+
+        # Tie weights again if needed
+        self.tie_weights()
+
+        return model_embeds
+
+    def tie_weights(self):
+        """
+        Overwrite `transformers.modeling_utils.PreTrainedModel.tie_weights` to handle the case of DecoupledLinear and DecoupledEmbedding.
+        """
+        output_embeddings = self.get_output_embeddings()
+        input_embeddings = self.get_input_embeddings()
+
+        if getattr(self.config, "tie_word_embeddings", True):
+            output_embeddings.weight = input_embeddings.weight
+
+    @add_start_docstrings_to_model_forward(IDEFICS2_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=Idefics2CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        pixel_attention_mask: Optional[torch.BoolTensor] = None,
+        image_hidden_states: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, Idefics2CausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Returns:
+
+        Example:
+
+        ```python
+        >>> import requests
+        >>> import torch
+        >>> from PIL import Image
+        >>> from io import BytesIO
+
+        >>> from transformers import AutoProcessor, AutoModelForVision2Seq
+        >>> from transformers.image_utils import load_image
+
+        >>> # Note that passing the image urls (instead of the actual pil images) to the processor is also possible
+        >>> image1 = load_image("https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg")
+        >>> image2 = load_image("https://cdn.britannica.com/59/94459-050-DBA42467/Skyline-Chicago.jpg")
+        >>> image3 = load_image("https://cdn.britannica.com/68/170868-050-8DDE8263/Golden-Gate-Bridge-San-Francisco.jpg")
+
+        >>> processor = AutoProcessor.from_pretrained("HuggingFaceM4/idefics2-8b-base")
+        >>> model = AutoModelForVision2Seq.from_pretrained("HuggingFaceM4/idefics2-8b-base", device_map="auto")
+
+        >>> BAD_WORDS_IDS = processor.tokenizer(["<image>", "<fake_token_around_image>"], add_special_tokens=False).input_ids
+        >>> EOS_WORDS_IDS = [processor.tokenizer.eos_token_id]
+
+        >>> # Create inputs
+        >>> prompts = [
+        ...   "<image>In this image, we can see the city of New York, and more specifically the Statue of Liberty.<image>In this image,",
+        ...   "In which city is that bridge located?<image>",
+        ... ]
+        >>> images = [[image1, image2], [image3]]
+        >>> inputs = processor(text=prompts, padding=True, return_tensors="pt").to("cuda")
+
+        >>> # Generate
+        >>> generated_ids = model.generate(**inputs, bad_words_ids=BAD_WORDS_IDS, max_new_tokens=20)
+        >>> generated_texts = processor.batch_decode(generated_ids, skip_special_tokens=True)
+
+        >>> print(generated_texts)
+        ['In this image, we can see the city of New York, and more specifically the Statue of Liberty. In this image, we can see the city of New York, and more specifically the Statue of Liberty.\n\n', 'In which city is that bridge located?\n\nThe bridge is located in the city of Pittsburgh, Pennsylvania.\n\n\nThe bridge is']
+        ```"""
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            pixel_values=pixel_values,
+            pixel_attention_mask=pixel_attention_mask,
+            image_hidden_states=image_hidden_states,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.score(hidden_states)
+        
+        
+        if input_ids is not None:
+            batch_size = input_ids.shape[0]
+        else:
+            batch_size = inputs_embeds.shape[0]
+
+        if self.config.pad_token_id is None and batch_size != 1:
+            raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
+        if self.config.pad_token_id is None:
+            sequence_lengths = -1
+        else:
+            if input_ids is not None:
+                # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
+                sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
+                sequence_lengths = sequence_lengths % input_ids.shape[-1]
+                sequence_lengths = sequence_lengths.to(logits.device)
+            else:
+                sequence_lengths = -1
+                
+        pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+                
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(pooled_logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(pooled_logits, labels)
+        if not return_dict:
+            output = (pooled_logits,) + outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=pooled_logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+        
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        # Omit tokens covered by past_key_values
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.seen_tokens
+                max_cache_length = past_key_values.get_max_length()
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+                max_cache_length = None
+
+            # Keep only the unprocessed tokens:
+            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+            # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+            # input)
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+            # input_ids based on the past_length.
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+
+            # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
+            if (
+                max_cache_length is not None
+                and attention_mask is not None
+                and cache_length + input_ids.shape[1] > max_cache_length
+            ):
+                attention_mask = attention_mask[:, -max_cache_length:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        image_hidden_states = kwargs.get("image_hidden_states", None)
+        if image_hidden_states is not None:
+            pixel_values = None
+            pixel_attention_mask = None
+        else:
+            pixel_values = kwargs.get("pixel_values", None)
+            pixel_attention_mask = kwargs.get("pixel_attention_mask", None)
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "pixel_values": pixel_values,
+                "pixel_attention_mask": pixel_attention_mask,
+                "image_hidden_states": image_hidden_states,
+            }
+        )
+        return model_inputs
+
+    def _update_model_kwargs_for_generation(self, outputs, model_kwargs, is_encoder_decoder, **kwargs):
+        model_kwargs = super()._update_model_kwargs_for_generation(
+            outputs=outputs,
+            model_kwargs=model_kwargs,
+            is_encoder_decoder=is_encoder_decoder,
+            **kwargs,
+        )
+        # Get the precomputed image_hidden_states
+        model_kwargs["image_hidden_states"] = outputs.image_hidden_states
+        return model_kwargs
+
+    @staticmethod
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM._reorder_cache
+    def _reorder_cache(past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (
+                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
+            )
+        return reordered_past
\ No newline at end of file
diff --git a/models/mllava/__init__.py b/models/mllava/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..30c3bb5efe2250ae8ce4546b4c485608ad58f914
--- /dev/null
+++ b/models/mllava/__init__.py
@@ -0,0 +1,4 @@
+from .modeling_llava import LlavaForConditionalGeneration, MLlavaForConditionalGeneration
+from .processing_llava import MLlavaProcessor
+from .configuration_llava import LlavaConfig
+from .utils import chat_mllava, chat_mllava_stream
\ No newline at end of file
diff --git a/models/mllava/configuration_llava.py b/models/mllava/configuration_llava.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d556b00f7cf32eea1e804add6e7841c26859962
--- /dev/null
+++ b/models/mllava/configuration_llava.py
@@ -0,0 +1,134 @@
+# coding=utf-8
+# Copyright 2023 Microsoft Research & University of Wisconsin-Madison and the HuggingFace Inc. team. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Llava model configuration"""
+
+
+# from ...configuration_utils import PretrainedConfig
+# from ...utils import logging
+# from ..auto import CONFIG_MAPPING
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+from transformers.models.auto import CONFIG_MAPPING
+
+
+logger = logging.get_logger(__name__)
+
+LLAVA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "llava-hf/llava-v1.5-7b": "https://huggingface.co/llava-hf/llava-v1.5-7b/resolve/main/config.json",
+}
+
+
+class LlavaConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`LlavaForConditionalGeneration`]. It is used to instantiate an
+    Llava model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of the Llava-9B.
+
+    e.g. [llava-hf/llava-9b](https://huggingface.co/llava-hf/llava-9b)
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        vision_config (`LlavaVisionConfig`,  *optional*):
+            Custom vision config or dict
+        text_config (`Union[AutoConfig, dict]`, *optional*):
+            The config object of the text backbone. Can be any of `LlamaConfig` or `MistralConfig`.
+        ignore_index (`int`, *optional*, defaults to -100):
+            The ignore index for the loss function.
+        image_token_index (`int`, *optional*, defaults to 32000):
+            The image token index to encode the image prompt.
+        projector_hidden_act (`str`, *optional*, defaults to `"gelu"`):
+            The activation function used by the multimodal projector.
+        vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`):
+            The feature selection strategy used to select the vision feature from the CLIP backbone.
+        vision_feature_layer (`int`, *optional*, defaults to -2):
+            The index of the layer to select the vision feature.
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the Llava model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`~LlavaForConditionalGeneration`]
+
+    Example:
+
+    ```python
+    >>> from transformers import LlavaForConditionalGeneration, LlavaConfig, CLIPVisionConfig, LlamaConfig
+
+    >>> # Initializing a CLIP-vision config
+    >>> vision_config = CLIPVisionConfig()
+
+    >>> # Initializing a Llama config
+    >>> text_config = LlamaConfig()
+
+    >>> # Initializing a Llava llava-1.5-7b style configuration
+    >>> configuration = LlavaConfig(vision_config, text_config)
+
+    >>> # Initializing a model from the llava-1.5-7b style configuration
+    >>> model = LlavaForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "llava"
+    is_composition = False
+
+    def __init__(
+        self,
+        vision_config=None,
+        text_config=None,
+        ignore_index=-100,
+        image_token_index=32000,
+        projector_hidden_act="gelu",
+        vision_feature_select_strategy="default",
+        vision_feature_layer=-2,
+        vocab_size=32000,
+        **kwargs,
+    ):
+        self.ignore_index = ignore_index
+        self.image_token_index = image_token_index
+        self.projector_hidden_act = projector_hidden_act
+        self.vision_feature_select_strategy = vision_feature_select_strategy
+        self.vision_feature_layer = vision_feature_layer
+        self.vocab_size = vocab_size
+
+        self.vision_config = vision_config
+
+        if isinstance(self.vision_config, dict):
+            vision_config["model_type"] = (
+                vision_config["model_type"] if "model_type" in vision_config else "clip_vision_model"
+            )
+            self.vision_config = CONFIG_MAPPING[vision_config["model_type"]](**vision_config)
+        elif vision_config is None:
+            self.vision_config = CONFIG_MAPPING["clip_vision_model"](
+                intermediate_size=4096,
+                hidden_size=1024,
+                patch_size=14,
+                image_size=336,
+                num_hidden_layers=24,
+                num_attention_heads=16,
+                vocab_size=32000,
+                projection_dim=768,
+            )
+        self.vocab_size = self.vocab_size
+
+        self.text_config = text_config
+
+        if isinstance(self.text_config, dict):
+            text_config["model_type"] = text_config["model_type"] if "model_type" in text_config else "llama"
+            self.text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
+            self.vocab_size = self.text_config.vocab_size
+        elif text_config is None:
+            self.text_config = CONFIG_MAPPING["llama"]()
+
+        super().__init__(**kwargs)
diff --git a/models/mllava/modeling_llava.py b/models/mllava/modeling_llava.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb5207e1649c75fcac6d3ad0f9f6a57c95c27118
--- /dev/null
+++ b/models/mllava/modeling_llava.py
@@ -0,0 +1,770 @@
+# coding=utf-8
+# Copyright 2023 the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" PyTorch Llava model."""
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.utils.checkpoint
+from torch import nn
+
+# from ... import PreTrainedModel
+# from ...activations import ACT2FN
+# from ...cache_utils import Cache
+# from ...modeling_outputs import ModelOutput
+# from ...utils import (
+#     add_start_docstrings,
+#     add_start_docstrings_to_model_forward,
+#     logging,
+#     replace_return_docstrings,
+# )
+# from ..auto import AutoModel, AutoModelForCausalLM
+
+from .configuration_llava import LlavaConfig
+
+from transformers import PreTrainedModel
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache
+from transformers.modeling_outputs import ModelOutput
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    logging,
+    replace_return_docstrings,
+)
+from transformers.models.auto import AutoModel, AutoModelForCausalLM
+from .configuration_llava import LlavaConfig
+
+
+logger = logging.get_logger(__name__)
+
+_CONFIG_FOR_DOC = "LlavaConfig"
+
+LLAVA_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "llava-hf/llava-1.5-7b-hf",
+    "llava-hf/llava-1.5-13b-hf",
+    "llava-hf/bakLlava-v1-hf",
+    # See all Llava models at https://huggingface.co/models?filter=llava
+]
+
+
+@dataclass
+# Copied from transformers.models.idefics.modeling_idefics.IdeficsCausalLMOutputWithPast with Idefics->Llava
+class LlavaCausalLMOutputWithPast(ModelOutput):
+    """
+    Base class for Llava causal language model (or autoregressive) outputs.
+
+    Args:
+        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+            Language modeling loss (for next-token prediction).
+        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
+            `past_key_values` input) to speed up sequential decoding.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        image_hidden_states (`tuple(torch.FloatTensor)`, *optional*):
+            Tuple of `torch.FloatTensor` (one for the output of the image embeddings, `(batch_size, num_images,
+            sequence_length, hidden_size)`.
+
+            image_hidden_states of the model produced by the vision encoder, and optionally by the perceiver
+    """
+
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    past_key_values: Optional[List[torch.FloatTensor]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+    image_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+
+
+class LlavaMultiModalProjector(nn.Module):
+    def __init__(self, config: LlavaConfig):
+        super().__init__()
+
+        self.linear_1 = nn.Linear(config.vision_config.hidden_size, config.text_config.hidden_size, bias=True)
+        self.act = ACT2FN[config.projector_hidden_act]
+        self.linear_2 = nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size, bias=True)
+
+    def forward(self, image_features):
+        hidden_states = self.linear_1(image_features)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        return hidden_states
+
+
+LLAVA_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+
+    Parameters:
+        config ([`LlavaConfig`] or [`LlavaVisionConfig`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+    "The bare LLaMA Model outputting raw hidden-states without any specific head on top.",
+    LLAVA_START_DOCSTRING,
+)
+class LlavaPreTrainedModel(PreTrainedModel):
+    config_class = LlavaConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["LlavaVisionAttention"]
+    _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+
+    def _init_weights(self, module):
+        # important: this ported version of Llava isn't meant for training from scratch - only
+        # inference and fine-tuning - so the proper init weights code has been removed - the original codebase
+        # https://github.com/haotian-liu/LLaVA/tree/main/llava should serve for that purpose
+        std = (
+            self.config.initializer_range
+            if hasattr(self.config, "initializer_range")
+            else self.config.text_config.initializer_range
+        )
+
+        if hasattr(module, "class_embedding"):
+            module.class_embedding.data.normal_(mean=0.0, std=std)
+
+        if isinstance(module, (nn.Linear, nn.Conv2d)):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+    @property
+    def _supports_sdpa(self):
+        """
+        Retrieve language_model's attribute to check whether the model supports
+        SDPA or not.
+        """
+        return self.language_model._supports_sdpa
+
+
+LLAVA_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+            it.
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
+            The tensors corresponding to the input images. Pixel values can be obtained using
+            [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`] for details ([]`LlavaProcessor`] uses
+            [`CLIPImageProcessor`] for processing images).
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+
+            [What are attention masks?](../glossary#attention-mask)
+
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+
+            If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
+            `past_key_values`).
+
+            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+            information on the default strategy.
+
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+            `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
+
+            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+    """The LLAVA model which consists of a vision backbone and a language model.""",
+    LLAVA_START_DOCSTRING,
+)
+class LlavaForConditionalGeneration(LlavaPreTrainedModel):
+    def __init__(self, config: LlavaConfig, vision_tower=None, language_model=None):
+        super().__init__(config)
+        self.vision_tower = AutoModel.from_config(config.vision_config) if vision_tower is None else vision_tower
+
+        self.multi_modal_projector = LlavaMultiModalProjector(config)
+        self.vocab_size = config.vocab_size
+        self.language_model = AutoModelForCausalLM.from_config(
+            config.text_config, attn_implementation=config._attn_implementation
+        ) if language_model is None else language_model
+        self.pad_token_id = self.config.pad_token_id if self.config.pad_token_id is not None else -1
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
+    def tie_weights(self):
+        return self.language_model.tie_weights()
+
+    def resize_token_embeddings(self, new_num_tokens: Optional[int] = None, pad_to_multiple_of=None) -> nn.Embedding:
+        model_embeds = self.language_model.resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
+        # update vocab size
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.config.vocab_size = model_embeds.num_embeddings
+        self.vocab_size = model_embeds.num_embeddings
+        return model_embeds
+
+    def _merge_input_ids_with_image_features(self, image_features, inputs_embeds, input_ids, attention_mask, labels):
+        num_images, num_image_patches, embed_dim = image_features.shape
+        batch_size, sequence_length = input_ids.shape
+        left_padding = not torch.sum(input_ids[:, -1] == torch.tensor(self.pad_token_id))
+        # 1. Create a mask to know where special image tokens are
+        special_image_token_mask = input_ids == self.config.image_token_index
+        num_special_image_tokens = torch.sum(special_image_token_mask, dim=-1)
+        # Compute the maximum embed dimension
+        max_embed_dim = (num_special_image_tokens.max() * (num_image_patches - 1)) + sequence_length
+        batch_indices, non_image_indices = torch.where(input_ids != self.config.image_token_index)
+
+        # 2. Compute the positions where text should be written
+        # Calculate new positions for text tokens in merged image-text sequence.
+        # `special_image_token_mask` identifies image tokens. Each image token will be replaced by `nb_text_tokens_per_images - 1` text tokens.
+        # `torch.cumsum` computes how each image token shifts subsequent text token positions.
+        # - 1 to adjust for zero-based indexing, as `cumsum` inherently increases indices by one.
+        new_token_positions = torch.cumsum((special_image_token_mask * (num_image_patches - 1) + 1), -1) - 1
+        nb_image_pad = max_embed_dim - 1 - new_token_positions[:, -1]
+        if left_padding:
+            new_token_positions += nb_image_pad[:, None]  # offset for left padding
+        text_to_overwrite = new_token_positions[batch_indices, non_image_indices]
+
+        # 3. Create the full embedding, already padded to the maximum position
+        final_embedding = torch.zeros(
+            batch_size, max_embed_dim, embed_dim, dtype=inputs_embeds.dtype, device=inputs_embeds.device
+        )
+        final_attention_mask = torch.zeros(
+            batch_size, max_embed_dim, dtype=attention_mask.dtype, device=inputs_embeds.device
+        )
+        if labels is not None:
+            final_labels = torch.full(
+                (batch_size, max_embed_dim), self.config.ignore_index, dtype=input_ids.dtype, device=input_ids.device
+            )
+        # In case the Vision model or the Language model has been offloaded to CPU, we need to manually
+        # set the corresponding tensors into their correct target device.
+        target_device = inputs_embeds.device
+        batch_indices, non_image_indices, text_to_overwrite = (
+            batch_indices.to(target_device),
+            non_image_indices.to(target_device),
+            text_to_overwrite.to(target_device),
+        )
+        attention_mask = attention_mask.to(target_device)
+
+        # 4. Fill the embeddings based on the mask. If we have ["hey" "<image>", "how", "are"]
+        # we need to index copy on [0, 577, 578, 579] for the text and [1:576] for the image features
+        final_embedding[batch_indices, text_to_overwrite] = inputs_embeds[batch_indices, non_image_indices]
+        final_attention_mask[batch_indices, text_to_overwrite] = attention_mask[batch_indices, non_image_indices]
+        if labels is not None:
+            final_labels[batch_indices, text_to_overwrite] = labels[batch_indices, non_image_indices]
+
+        # 5. Fill the embeddings corresponding to the images. Anything that is still zeros needs filling
+        image_to_overwrite = torch.all(final_embedding == 0, dim=-1)
+        image_to_overwrite &= image_to_overwrite.cumsum(-1) - 1 >= nb_image_pad[:, None].to(target_device)
+
+        if image_to_overwrite.sum() != image_features.shape[:-1].numel():
+            raise ValueError(
+                f"The input provided to the model are wrong. The number of image tokens is {torch.sum(special_image_token_mask)} while"
+                f" the number of image given to the model is {num_images}. This prevents correct indexing and breaks batch generation."
+            )
+
+        final_embedding[image_to_overwrite] = image_features.contiguous().reshape(-1, embed_dim).to(target_device)
+        final_attention_mask |= image_to_overwrite
+        position_ids = (final_attention_mask.cumsum(-1) - 1).masked_fill_((final_attention_mask == 0), 1)
+
+        if labels is None:
+            final_labels = None
+
+        return final_embedding, final_attention_mask, final_labels, position_ids
+
+    @add_start_docstrings_to_model_forward(LLAVA_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=LlavaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        pixel_values: torch.FloatTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        vision_feature_layer: Optional[int] = None,
+        vision_feature_select_strategy: Optional[str] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, LlavaCausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Returns:
+
+        Example:
+
+        ```python
+        >>> from PIL import Image
+        >>> import requests
+        >>> from transformers import AutoProcessor, LlavaForConditionalGeneration
+
+        >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
+        >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
+
+        >>> prompt = "<image>\nUSER: What's the content of the image?\nASSISTANT:"
+        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
+        >>> image = Image.open(requests.get(url, stream=True).raw)
+
+        >>> inputs = processor(text=prompt, images=image, return_tensors="pt")
+
+        >>> # Generate
+        >>> generate_ids = model.generate(**inputs, max_length=30)
+        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+        "\nUSER: What's the content of the image?\nASSISTANT: The image features a stop sign on a street corner"
+        ```"""
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        vision_feature_layer = (
+            vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer
+        )
+        vision_feature_select_strategy = (
+            vision_feature_select_strategy
+            if vision_feature_select_strategy is not None
+            else self.config.vision_feature_select_strategy
+        )
+
+        if inputs_embeds is None:
+            # 1. Extra the input embeddings
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+            # 2. Merge text and images
+            if pixel_values is not None and input_ids.shape[1] != 1:
+                if isinstance(pixel_values, list):
+                    pixel_values = torch.cat([x for x in pixel_values if x is not None], dim=0)
+                # for siglip, need to transform the pixel_values to the right data type
+                if pixel_values.dtype != self.vision_tower.dtype:
+                    pixel_values = pixel_values.type(self.vision_tower.dtype)
+                image_outputs = self.vision_tower(pixel_values, output_hidden_states=True)
+                # this is not memory efficient at all (output_hidden_states=True) will save all the hidden stated.
+                selected_image_feature = image_outputs.hidden_states[vision_feature_layer]
+
+                if vision_feature_select_strategy == "default":
+                    selected_image_feature = selected_image_feature[:, 1:]
+                elif vision_feature_select_strategy == "full":
+                    selected_image_feature = selected_image_feature
+                else:
+                    raise ValueError(
+                        f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}"
+                    )
+
+                image_features = self.multi_modal_projector(selected_image_feature)
+                inputs_embeds, attention_mask, labels, position_ids = self._merge_input_ids_with_image_features(
+                    image_features, inputs_embeds, input_ids, attention_mask, labels
+                )
+                if labels is None:
+                    labels = torch.full_like(attention_mask, self.config.ignore_index).to(torch.long)
+            else:
+                # In case input_ids.shape[1] == 1 & pixel_values==None & past_key_values != None, we are in the case of
+                # generation with cache
+                if past_key_values is not None and pixel_values is not None and input_ids.shape[1] == 1:
+                    # Retrieve the first layer to inspect the logits and mask out the hidden states
+                    # that are set to 0
+                    first_layer_past_key_value = past_key_values[0][0][:, :, :, 0]
+
+                    # Sum all dimensions of head_dim (-2) to avoid random errors such as: https://github.com/huggingface/transformers/pull/28032#issuecomment-1863691941
+                    batch_index, non_attended_tokens = torch.where(first_layer_past_key_value.float().sum(-2) == 0)
+
+                    # Get the target length
+                    target_seqlen = first_layer_past_key_value.shape[-1] + 1
+
+                    extended_attention_mask = torch.ones(
+                        (attention_mask.shape[0], target_seqlen - attention_mask.shape[1]),
+                        dtype=attention_mask.dtype,
+                        device=attention_mask.device,
+                    )
+
+                    # Filter out only the tokens that can be un-attended, this can happen
+                    # if one uses Llava + Fused modules where the cache on the
+                    # first iteration is already big enough, or if one passes custom cache
+                    valid_indices = non_attended_tokens < extended_attention_mask.size(-1)
+                    new_batch_index = batch_index[valid_indices]
+                    new_non_attended_tokens = non_attended_tokens[valid_indices]
+
+                    # Zero-out the places where we don't need to attend
+                    extended_attention_mask[new_batch_index, new_non_attended_tokens] = 0
+
+                    attention_mask = torch.cat((attention_mask, extended_attention_mask), dim=1)
+                    position_ids = torch.sum(attention_mask, dim=1).unsqueeze(-1) - 1
+        
+        outputs = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        logits = outputs[0]
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            if attention_mask is not None:
+                shift_attention_mask = attention_mask[..., 1:]
+                shift_logits = logits[..., :-1, :][shift_attention_mask.to(logits.device) != 0].contiguous()
+                shift_labels = labels[..., 1:][shift_attention_mask.to(labels.device) != 0].contiguous()
+            else:
+                shift_logits = logits[..., :-1, :].contiguous()
+                shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1).to(shift_logits.device)
+            )
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return LlavaCausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, inputs_embeds=None, pixel_values=None, attention_mask=None, **kwargs
+    ):
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.seen_tokens
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+
+            # Keep only the unprocessed tokens:
+            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+            # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+            # input)
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+            # input_ids based on the past_length.
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+            elif self.config.image_token_index in input_ids:
+                input_ids = input_ids[:, input_ids.shape[1] - 1 :]
+            # If the cache has seen more tokens than it can hold, then the cache has a size limit. Let's discard the
+            # older attention values, as their corresponding values are not part of the input.
+            if cache_length < past_length and attention_mask is not None:
+                attention_mask = attention_mask[:, -(cache_length + input_ids.shape[1]) :]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "pixel_values": pixel_values,
+            }
+        )
+        return model_inputs
+
+    def _reorder_cache(self, *args, **kwargs):
+        return self.language_model._reorder_cache(*args, **kwargs)
+
+
+
+
+from transformers.models.clip.modeling_clip import CLIPEncoderLayer, CLIPEncoder
+@add_start_docstrings(
+    """The MLLAVA model which consists of a vision backbone and a language model.""",
+    LLAVA_START_DOCSTRING,
+)
+class MLlavaForConditionalGeneration(LlavaForConditionalGeneration):
+    def __init__(self, config: LlavaConfig):
+        super().__init__(config)
+        config.vision_config.type_vocab_size = 144
+        self.image_type_embeddings = nn.Embedding(config.vision_config.type_vocab_size, config.vision_config.hidden_size)
+        # self.vision_xatten_layers = nn.ModuleList([CLIPEncoderLayer(config.vision_config) for _ in range(config.vision_config.num_hidden_layers)])
+        self.vision_xatten_layers = CLIPEncoder(config.vision_config)
+        
+        
+    @add_start_docstrings_to_model_forward(LLAVA_INPUTS_DOCSTRING)
+    @replace_return_docstrings(output_type=LlavaCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        pixel_values: torch.FloatTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        vision_feature_layer: Optional[int] = None,
+        vision_feature_select_strategy: Optional[str] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, LlavaCausalLMOutputWithPast]:
+        r"""
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+        Returns:
+
+        Example:
+
+        ```python
+        >>> from PIL import Image
+        >>> import requests
+        >>> from transformers import AutoProcessor, LlavaForConditionalGeneration
+
+        >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
+        >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
+
+        >>> prompt = "<image>\nUSER: What's the content of the image?\nASSISTANT:"
+        >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
+        >>> image = Image.open(requests.get(url, stream=True).raw)
+
+        >>> inputs = processor(text=prompt, images=image, return_tensors="pt")
+
+        >>> # Generate
+        >>> generate_ids = model.generate(**inputs, max_length=30)
+        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+        "\nUSER: What's the content of the image?\nASSISTANT: The image features a stop sign on a street corner"
+        ```"""
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        vision_feature_layer = (
+            vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer
+        )
+        vision_feature_select_strategy = (
+            vision_feature_select_strategy
+            if vision_feature_select_strategy is not None
+            else self.config.vision_feature_select_strategy
+        )
+
+        if inputs_embeds is None:
+            # 1. Extra the input embeddings
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+            # 2. Merge text and images
+            if pixel_values is not None and input_ids.shape[1] != 1:
+                image_outputs = self.vision_tower(pixel_values, output_hidden_states=True)
+                # this is not memory efficient at all (output_hidden_states=True) will save all the hidden stated.
+                selected_image_feature = image_outputs.hidden_states[vision_feature_layer]
+
+                if vision_feature_select_strategy == "default":
+                    selected_image_feature = selected_image_feature[:, 1:]
+                elif vision_feature_select_strategy == "full":
+                    selected_image_feature = selected_image_feature
+                else:
+                    raise ValueError(
+                        f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}"
+                    )
+                    
+                # added by Dongfu
+                num_images, num_image_patches, embed_dim = selected_image_feature.shape
+                image_type_embeddings = self.image_type_embeddings(torch.arange(num_images, device=selected_image_feature.device))
+                selected_image_feature += image_type_embeddings.unsqueeze(1)
+                xatten_output = self.vision_xatten_layers(selected_image_feature, attention_mask=None, causal_attention_mask=None)
+                selected_image_feature = xatten_output[0]
+                # end of added by Dongfu
+                
+                image_features = self.multi_modal_projector(selected_image_feature)
+                inputs_embeds, attention_mask, labels, position_ids = self._merge_input_ids_with_image_features(
+                    image_features, inputs_embeds, input_ids, attention_mask, labels
+                )
+                if labels is None:
+                    labels = torch.full_like(attention_mask, self.config.ignore_index).to(torch.long)
+            else:
+                # In case input_ids.shape[1] == 1 & pixel_values==None & past_key_values != None, we are in the case of
+                # generation with cache
+                if past_key_values is not None and pixel_values is not None and input_ids.shape[1] == 1:
+                    # Retrieve the first layer to inspect the logits and mask out the hidden states
+                    # that are set to 0
+                    first_layer_past_key_value = past_key_values[0][0][:, :, :, 0]
+
+                    # Sum all dimensions of head_dim (-2) to avoid random errors such as: https://github.com/huggingface/transformers/pull/28032#issuecomment-1863691941
+                    batch_index, non_attended_tokens = torch.where(first_layer_past_key_value.float().sum(-2) == 0)
+
+                    # Get the target length
+                    target_seqlen = first_layer_past_key_value.shape[-1] + 1
+
+                    extended_attention_mask = torch.ones(
+                        (attention_mask.shape[0], target_seqlen - attention_mask.shape[1]),
+                        dtype=attention_mask.dtype,
+                        device=attention_mask.device,
+                    )
+
+                    # Filter out only the tokens that can be un-attended, this can happen
+                    # if one uses Llava + Fused modules where the cache on the
+                    # first iteration is already big enough, or if one passes custom cache
+                    valid_indices = non_attended_tokens < extended_attention_mask.size(-1)
+                    new_batch_index = batch_index[valid_indices]
+                    new_non_attended_tokens = non_attended_tokens[valid_indices]
+
+                    # Zero-out the places where we don't need to attend
+                    extended_attention_mask[new_batch_index, new_non_attended_tokens] = 0
+
+                    attention_mask = torch.cat((attention_mask, extended_attention_mask), dim=1)
+                    position_ids = torch.sum(attention_mask, dim=1).unsqueeze(-1) - 1
+        
+        outputs = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        logits = outputs[0]
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            if attention_mask is not None:
+                shift_attention_mask = attention_mask[..., 1:]
+                shift_logits = logits[..., :-1, :][shift_attention_mask.to(logits.device) != 0].contiguous()
+                shift_labels = labels[..., 1:][shift_attention_mask.to(labels.device) != 0].contiguous()
+            else:
+                shift_logits = logits[..., :-1, :].contiguous()
+                shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1).to(shift_logits.device)
+            )
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return LlavaCausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
\ No newline at end of file
diff --git a/models/mllava/processing_llava.py b/models/mllava/processing_llava.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4cd6c55df133ba15912aa9c6ebf4d12d6976260
--- /dev/null
+++ b/models/mllava/processing_llava.py
@@ -0,0 +1,381 @@
+# coding=utf-8
+# Copyright 2023 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Processor class for Llava.
+"""
+
+import os
+import json
+from typing import List, Optional, Union, Dict
+
+# from ...feature_extraction_utils import BatchFeature
+# from ...image_utils import ImageInput
+# from ...processing_utils import ProcessorMixin
+# from ...tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
+# from ...utils import TensorType
+
+from transformers.feature_extraction_sequence_utils import BatchFeature
+from transformers.image_utils import ImageInput
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
+from transformers.utils import TensorType
+from transformers.processing_utils import transformers_module
+from transformers.utils.hub import is_remote_url, download_url, cached_file, is_offline_mode
+from transformers.utils import IMAGE_PROCESSOR_NAME
+
+from PIL import Image
+import logging
+import torch
+import numpy as np
+logger = logging.getLogger(__name__)
+
+class MLlavaProcessor(ProcessorMixin):
+    r"""
+    Constructs a Llava processor which wraps a Llava image processor and a Llava tokenizer into a single processor.
+
+    [`LlavaProcessor`] offers all the functionalities of [`CLIPImageProcessor`] and [`LlamaTokenizerFast`]. See the
+    [`~LlavaProcessor.__call__`] and [`~LlavaProcessor.decode`] for more information.
+
+    Args:
+        image_processor ([`CLIPImageProcessor`], *optional*):
+            The image processor is a required input.
+        tokenizer ([`LlamaTokenizerFast`], *optional*):
+            The tokenizer is a required input.
+    """
+
+    attributes = ["image_processor", "tokenizer"]
+    image_processor_class = ("CLIPImageProcessor", "SiglipImageProcessor")
+    tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast", "PreTrainedTokenizerFast")
+
+    def __init__(self, image_processor=None, tokenizer=None):
+        super().__init__(image_processor, tokenizer)
+        
+    def preprocess_interleaved_images_and_text(
+        self,
+        text,
+        images=None,
+    ):
+        """
+        Args:
+            text (`str`, `List[str]`):
+                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
+                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
+                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
+                text can contain <image> tokens as the placeholder for the image(s) to be inserted.
+            images (`PIL.Image.Image`, `List[PIL.Image.Image]`, `List[List[PIL.Image.Image]]`):
+                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
+                tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a
+                number of channels, H and W are image height and width.
+                the number of the images should match the number of <image> tokens in the text.
+        
+        """
+        assert text is not None, "text cannot be None."
+            
+        if images is not None:
+            if isinstance(images, Image.Image):
+                images = [images]
+            if isinstance(images, list) and isinstance(images[0], Image.Image):
+                if isinstance(text, str):
+                    images = [images]
+                elif isinstance(text, list):
+                    if len(text) != len(images):
+                        raise ValueError("Invalid input text. Number of texts does not match number of images.")
+                    images = [[image] for image in images]
+            if isinstance(text, str):
+                num_images = len(images[0])    
+                num_image_tokens = text.count("<image>")
+                if num_image_tokens < num_images:
+                    # prepend empty image tokens to text
+                    if "USER:" in text:
+                        text = text.replace("USER:", "USER:" + "<image>" * (num_images - num_image_tokens), 1)
+                    elif "Human:" in text:
+                        text = text.replace("Human:", "Human:" + "<image>" * (num_images - num_image_tokens), 1)
+                    elif "HUMAN:" in text:
+                        text = text.replace("HUMAN:", "HUMAN:" + "<image>" * (num_images - num_image_tokens), 1)
+                    else:
+                        text = "<image>" * (num_images - num_image_tokens) + text
+                    # logger.warning("Image Tokens <image> are not provided in the text. Automatically prepending them before the text. This might cause model to behave unexpectedly.")
+                elif num_image_tokens > num_images:
+                    text = text.split("<image>")
+                    for i, t in enumerate(text):
+                        if i < num_images:
+                            text[i] = t + "<image>"
+                    text = "".join(text)
+                    logger.warning(f"Number of <image> tokens: {num_image_tokens} exceeds number of images: {num_images}. Automatically removing extra tokens at the end of the text.")
+                    # raise ValueError("Invalid input text. Number of <image> tokens exceeds number of images.")
+                texts = [text]
+            elif isinstance(text, list):
+                if not isinstance(text[0], str):
+                    raise ValueError("Invalid input text. Each element of text must be a string.")
+                for i, t in enumerate(text):
+                    num_image_tokens = t.count("<image>")
+                    num_images = len(images[i])
+                    if num_image_tokens < num_images:
+                        # prepend empty image tokens to text
+                        if "USER:" in t:
+                            t = t.replace("USER:", "USER:" + "<image>" * (num_images - num_image_tokens), 1)
+                        elif "Human:" in t:
+                            t = t.replace("Human:", "Human:" + "<image>" * (num_images - num_image_tokens), 1)
+                        elif "HUMAN:" in t:
+                            t = t.replace("HUMAN:", "HUMAN:" + "<image>" * (num_images - num_image_tokens), 1)
+                        else:
+                            t = "<image>" * (num_images - num_image_tokens) + t
+                        # logger.warning("Image Tokens <image> are not provided in the text. Automatically prepending them before the text. This might cause model to behave unexpectedly.")
+                    elif num_image_tokens > num_images:
+                        t = t.split("<image>")
+                        for j, s in enumerate(t):
+                            if j < num_images:
+                                t[j] = s + "<image>"
+                        t = "".join(t)
+                        logger.warning(f"Number of <image> tokens: {num_image_tokens} exceeds number of images: {num_images}. Automatically removing extra tokens at the end of the text.")
+                        # raise ValueError("Invalid input text. Number of <image> tokens exceeds number of images.")
+                    text[i] = t
+                texts = text
+            else:
+                raise ValueError("Invalid input text. text must be a string or a list of strings.")
+            assert all([t.count("<image>") == len(images_per_text) for t, images_per_text in zip(texts, images)]), "Number of <image> tokens in text does not match number of images."
+            # add image denotation in text before each <image> as "(image {i}: <image>)"
+            for i, t in enumerate(texts):
+                for j in range(len(images[i])):
+                    t = t.replace("<image>", f"(image {j+1}: <Image><IMAGE></Image>)", 1)
+                t = t.replace("<IMAGE>", "<image>")
+                texts[i] = t
+            
+            # flatten images
+            images = [image for images_per_text in images for image in images_per_text]
+        else:
+            if isinstance(text, str):
+                texts = [text]
+            elif isinstance(text, list):
+                if not isinstance(text[0], str):
+                    raise ValueError("Invalid input text. Each element of text must be a string.")
+                texts = text
+            else:
+                raise ValueError("Invalid input text. text must be a string or a list of strings.")
+        
+        return texts, images
+
+    def __call__(
+        self,
+        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
+        images: ImageInput = None,
+        padding: Union[bool, str, PaddingStrategy] = False,
+        truncation: Union[bool, str, TruncationStrategy] = None,
+        max_length=None,
+        return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
+        add_image_ids: bool = True,
+    ) -> BatchFeature:
+        """
+        Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
+        and `kwargs` arguments to LlamaTokenizerFast's [`~LlamaTokenizerFast.__call__`] if `text` is not `None` to encode
+        the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to
+        CLIPImageProcessor's [`~CLIPImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring
+        of the above two methods for more information.
+
+        Args:
+            text (`str`, `List[str]`, `List[List[str]]`):
+                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
+                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
+                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
+            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
+                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
+                tensor. In case of a NumPy array/PyTorch tensor, each image should be of shape (C, H, W), where C is a
+                number of channels, H and W are image height and width.
+            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
+                Select a strategy to pad the returned sequences (according to the model's padding side and padding
+                index) among:
+                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+                  sequence if provided).
+                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+                  acceptable input length for the model if that argument is not provided.
+                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+                  lengths).
+            max_length (`int`, *optional*):
+                Maximum length of the returned list and optionally padding length (see above).
+            truncation (`bool`, *optional*):
+                Activates truncation to cut input sequences longer than `max_length` to `max_length`.
+            return_tensors (`str` or [`~utils.TensorType`], *optional*):
+                If set, will return tensors of a particular framework. Acceptable values are:
+
+                - `'tf'`: Return TensorFlow `tf.constant` objects.
+                - `'pt'`: Return PyTorch `torch.Tensor` objects.
+                - `'np'`: Return NumPy `np.ndarray` objects.
+                - `'jax'`: Return JAX `jnp.ndarray` objects.
+
+        Returns:
+            [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+              `None`).
+            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
+        """
+        if add_image_ids:
+            text, images = self.preprocess_interleaved_images_and_text(text, images)
+        if images is not None:
+            pixel_values = self.image_processor(images, return_tensors=return_tensors)["pixel_values"] # [batch_size, num_channels, height, width], e.g. [1, 3, 336, 336]
+        else:
+            pixel_values = None
+        text_inputs = self.tokenizer(
+            text, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length
+        )
+        # text_inputs: 
+        # 1. input_ids: [batch_size, sequence_length], e.g. [1, 6]
+        # 2. attention_mask: [batch_size, sequence_length], e.g. [1, 6]
+
+        return BatchFeature(data={**text_inputs, "pixel_values": pixel_values})
+
+    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    @property
+    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        image_processor_input_names = self.image_processor.model_input_names
+        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
+
+    def _right_pad_inputs_with_attention_mask(self, model_inputs: List[Dict]):
+        results = {}
+        assert len(model_inputs) == 1, "This method only supports a single input, but get {} inputs".format(len(model_inputs))
+        for k in model_inputs[0].keys():
+            if k == "pixel_values":
+                results[k] = [inputs[k] if inputs[k] is not None else None for inputs in model_inputs]
+            else:
+                results[k] = torch.cat([inputs[k] for inputs in model_inputs], dim=0)
+        return results
+
+    @classmethod
+    def _get_arguments_from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+        args = []
+        
+        cache_dir = kwargs.pop("cache_dir", None)
+        force_download = kwargs.pop("force_download", False)
+        resume_download = kwargs.pop("resume_download", False)
+        proxies = kwargs.pop("proxies", None)
+        token = kwargs.pop("token", None)
+        local_files_only = kwargs.pop("local_files_only", False)
+        revision = kwargs.pop("revision", None)
+        subfolder = kwargs.pop("subfolder", "")
+
+        from_pipeline = kwargs.pop("_from_pipeline", None)
+        from_auto_class = kwargs.pop("_from_auto", False)
+
+        user_agent = {"file_type": "processor", "from_auto_class": from_auto_class}
+        if from_pipeline is not None:
+            user_agent["using_pipeline"] = from_pipeline
+
+        if is_offline_mode() and not local_files_only:
+            logger.info("Offline mode: forcing local_files_only=True")
+            local_files_only = True
+            
+        pretrained_model_name_or_path = str(pretrained_model_name_or_path)
+        is_local = os.path.isdir(pretrained_model_name_or_path)
+        if os.path.isdir(pretrained_model_name_or_path):
+            processor_file = os.path.join(pretrained_model_name_or_path, IMAGE_PROCESSOR_NAME)
+        if os.path.isfile(pretrained_model_name_or_path):
+            resolved_processor_file = pretrained_model_name_or_path
+            is_local = True
+        elif is_remote_url(pretrained_model_name_or_path):
+            processor_file = pretrained_model_name_or_path
+            resolved_processor_file = download_url(pretrained_model_name_or_path)
+        else:
+            processor_file = IMAGE_PROCESSOR_NAME
+            try:
+                # Load from local folder or from cache or download from model Hub and cache
+                resolved_processor_file = cached_file(
+                    pretrained_model_name_or_path,
+                    processor_file,
+                    cache_dir=cache_dir,
+                    force_download=force_download,
+                    proxies=proxies,
+                    resume_download=resume_download,
+                    local_files_only=local_files_only,
+                    token=token,
+                    user_agent=user_agent,
+                    revision=revision,
+                    subfolder=subfolder,
+                    _raise_exceptions_for_missing_entries=True,
+                )
+            except EnvironmentError:
+                # Raise any environment error raise by `cached_file`. It will have a helpful error message adapted to
+                # the original exception.
+                raise
+            except Exception:
+                # For any other exception, we throw a generic error.
+                raise EnvironmentError(
+                    f"Can't load processor for '{pretrained_model_name_or_path}'. If you were trying to load"
+                    " it from 'https://huggingface.co/models', make sure you don't have a local directory with the"
+                    f" same name. Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a"
+                    f" directory containing a {IMAGE_PROCESSOR_NAME} file"
+                )
+        
+        # Existing processors on the Hub created before #27761 being merged don't have `processor_config.json` (if not
+        # updated afterward), and we need to keep `from_pretrained` work. So here it fallbacks to the empty dict.
+        # (`cached_file` called using `_raise_exceptions_for_missing_entries=False` to avoid exception)
+        # However, for models added in the future, we won't get the expected error if this file is missing.
+        if resolved_processor_file is None:
+            image_processor_dict = {}
+
+        try:
+            # Load processor dict
+            with open(resolved_processor_file, "r", encoding="utf-8") as reader:
+                text = reader.read()
+            image_processor_dict = json.loads(text)
+
+        except json.JSONDecodeError:
+            raise EnvironmentError(
+                f"It looks like the config file at '{resolved_processor_file}' is not a valid JSON file."
+            )
+            
+        for attribute_name in cls.attributes:
+            class_name = getattr(cls, f"{attribute_name}_class")
+            if isinstance(class_name, tuple):
+                if attribute_name == "tokenizer":
+                    classes = tuple(getattr(transformers_module, n) if n is not None else None for n in class_name)
+                    use_fast = kwargs.get("use_fast", True)
+                    if use_fast and classes[1] is not None:
+                        attribute_class = classes[1]
+                    else:
+                        attribute_class = classes[0]
+                elif attribute_name == "image_processor":
+                    image_processor_type = image_processor_dict.get("image_processor_type", None)
+                    if image_processor_type is not None:
+                        assert image_processor_type in class_name, f"Invalid image processor type: {image_processor_type}"
+                        attribute_class = getattr(transformers_module, image_processor_type)
+                    else:
+                        attribute_class = getattr(transformers_module, class_name[0])
+                else:
+                    raise ValueError(f"Invalid attribute name: {attribute_name}")
+            else:
+                attribute_class = getattr(transformers_module, class_name)
+
+            args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs))
+        return args
+        
\ No newline at end of file
diff --git a/models/mllava/utils.py b/models/mllava/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..aac29f2e1865588d434b2dfdb4f14e59d014a681
--- /dev/null
+++ b/models/mllava/utils.py
@@ -0,0 +1,188 @@
+import PIL
+import torch
+from .modeling_llava import LlavaForConditionalGeneration
+from .processing_llava import MLlavaProcessor
+# from ..conversation import conv_mllava_v1_mmtag as default_conv
+from ..conversation import conv_mllava_v1 as default_conv, conv_templates
+
+from typing import List, Tuple, Union, Tuple
+
+def chat_mllava(
+    text:str, 
+    images: List[Union[PIL.Image.Image, str]], 
+    model:LlavaForConditionalGeneration, 
+    processor:MLlavaProcessor, 
+    max_input_length:int=None, 
+    history:List[dict]=None, 
+    **kwargs) -> Tuple[str, List[dict]]:
+    """
+    Chat with the Mllava model
+    Args:
+        text: str, the text to be sent to the model, where <image> will be the placeholder for the image
+        images: List[PIL.Image.Image], the images to be sent to the model, or None  
+        model: LlavaForConditionalGeneration, the model to be used
+        processor: MLlavaProcessor, the processor to be used
+        max_input_length: int, the maximum input length
+        history: List[dict], list of messages in the conversation as history. Each message is a dictionary {"role": "ASSISTANT/USER", "text": "the message"}. If None, the conversation will start from scratch
+        kwargs: dict, the generation kwargs
+    Returns:
+        Tuple[str, List[dict]], the generated text and the history of the conversation
+        
+
+    """
+    if "llama-3" in model.language_model.name_or_path.lower():
+        conv = conv_templates['llama_3']
+        terminators = [
+            processor.tokenizer.eos_token_id,
+            processor.tokenizer.convert_tokens_to_ids("<|eot_id|>")
+        ]
+    else:
+        conv = default_conv
+        terminators = None
+    kwargs["eos_token_id"] = terminators
+    conv = conv.copy()
+    conv.messages = []
+    if history is not None:
+        for message in history:
+            assert message["role"] in conv.roles
+            conv.append_message(message["role"], message["text"])
+        if text:
+            assert conv.messages[-1][0] == conv.roles[1], "The last message in the history should be the assistant, if the given text is not empty"
+            conv.append_message(conv.roles[0], text)
+            conv.append_message(conv.roles[1], "")
+            history.append({"role": conv.roles[0], "text": text})
+            history.append({"role": conv.roles[1], "text": ""})
+        else:
+            if conv.messages[-1][0] == conv.roles[1]:
+                assert conv.messages[-1][1] == "", "No user message should be provided"
+            else:
+                assert conv.messages[-1][0] == conv.roles[0], "The last message in the history should be the user, if the given text is empty"
+                conv.append_message(conv.roles[0], "")
+                history.append({"role": conv.roles[0], "text": ""})
+    else:
+        history = []
+        history.append({"role": conv.roles[0], "text": text})
+        history.append({"role": conv.roles[1], "text": ""})
+        conv.append_message(conv.roles[0], text)
+        conv.append_message(conv.roles[1], "")
+    assert conv.messages[-1][0] == conv.roles[1] and conv.messages[-1][1] == "", "Format check"
+    assert history[-1]["role"] == conv.roles[1] and history[-1]["text"] == "", "Format check"
+    
+    prompt = conv.get_prompt()
+    if images:
+        for i in range(len(images)):
+            if isinstance(images[i], str):
+                images[i] = PIL.Image.open(images[i]).convert("RGB")
+    
+    inputs = processor(images=images, text=prompt, return_tensors="pt", truncation=True, max_length=max_input_length)
+    for k, v in inputs.items():
+        if v is not None:
+            if isinstance(v, torch.Tensor):
+                inputs[k] = v.to(model.device)
+            elif isinstance(v, list):
+                inputs[k] = [x.to(model.device) for x in v]
+            else:
+                raise ValueError(f"Invalid input type: {type(v)}")
+    
+
+    output_ids = model.generate(**inputs, **kwargs)
+    output_ids = output_ids[0]
+    
+    # remove the input tokens
+    generated_ids = output_ids[inputs["input_ids"].shape[-1]:]
+    generated_text = processor.decode(generated_ids, skip_special_tokens=True)
+
+    history[-1]["text"] = generated_text
+    
+    return generated_text, history
+
+
+def chat_mllava_stream(
+    text:str, 
+    images: List[Union[PIL.Image.Image, str]], 
+    model:LlavaForConditionalGeneration, 
+    processor:MLlavaProcessor, 
+    max_input_length:int=None, 
+    history:List[dict]=None, 
+    **kwargs) -> Tuple[str, List[dict]]:
+    """
+    Chat with the Mllava model
+    Args:
+        text: str, the text to be sent to the model, where <image> will be the placeholder for the image
+        images: List[PIL.Image.Image], the images to be sent to the model, or None  
+        model: LlavaForConditionalGeneration, the model to be used
+        processor: MLlavaProcessor, the processor to be used
+        max_input_length: int, the maximum input length
+        history: List[dict], list of messages in the conversation as history. Each message is a dictionary {"role": "ASSISTANT/USER", "text": "the message"}. If None, the conversation will start from scratch
+        kwargs: dict, the generation kwargs
+    Returns:
+        Tuple[str, List[dict]], the generated text and the history of the conversation
+        
+
+    """
+    if "llama-3" in model.language_model.name_or_path.lower():
+        conv = conv_templates['llama_3']
+        terminators = [
+            processor.tokenizer.eos_token_id,
+            processor.tokenizer.convert_tokens_to_ids("<|eot_id|>")
+        ]
+    else:
+        conv = default_conv
+        terminators = None
+    kwargs["eos_token_id"] = terminators
+    conv = conv.copy()
+    conv.messages = []
+    if history is not None:
+        for message in history:
+            assert message["role"] in conv.roles
+            conv.append_message(message["role"], message["text"])
+        if text:
+            assert conv.messages[-1][0] == conv.roles[1], "The last message in the history should be the assistant, if the given text is not empty"
+            conv.append_message(conv.roles[0], text)
+            conv.append_message(conv.roles[1], "")
+            history.append({"role": conv.roles[0], "text": text})
+            history.append({"role": conv.roles[1], "text": ""})
+        else:
+            if conv.messages[-1][0] == conv.roles[1]:
+                assert conv.messages[-1][1] == "", "No user message should be provided"
+            else:
+                assert conv.messages[-1][0] == conv.roles[0], "The last message in the history should be the user, if the given text is empty"
+                conv.append_message(conv.roles[0], "")
+                history.append({"role": conv.roles[0], "text": ""})
+    else:
+        history = []
+        history.append({"role": conv.roles[0], "text": text})
+        history.append({"role": conv.roles[1], "text": ""})
+        conv.append_message(conv.roles[0], text)
+        conv.append_message(conv.roles[1], "")
+    assert conv.messages[-1][0] == conv.roles[1] and conv.messages[-1][1] == "", "Format check"
+    assert history[-1]["role"] == conv.roles[1] and history[-1]["text"] == "", "Format check"
+    
+    prompt = conv.get_prompt()
+    if images:
+        for i in range(len(images)):
+            if isinstance(images[i], str):
+                images[i] = PIL.Image.open(images[i])
+            images[i] = images[i].convert("RGB")
+    
+    inputs = processor(images=images, text=prompt, return_tensors="pt", truncation=True, max_length=max_input_length)
+    print(processor.tokenizer.decode(inputs["input_ids"][0]))
+    for k, v in inputs.items():
+        if v is not None:
+            if isinstance(v, torch.Tensor):
+                inputs[k] = v.to(model.device)
+            elif isinstance(v, list):
+                inputs[k] = [x.to(model.device) for x in v]
+            else:
+                raise ValueError(f"Invalid input type: {type(v)}")
+    
+    from transformers import TextIteratorStreamer
+    from threading import Thread
+    streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)
+    kwargs["streamer"] = streamer
+    inputs.update(kwargs)
+    thread = Thread(target=model.generate, kwargs=inputs)
+    thread.start()
+    for _output in streamer:
+        history[-1]["text"] += _output
+        yield history[-1]["text"], history
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 939711ab6737637e37a3eb0f5929cb40a2a597f7..c32a0e8efb4345138890ec52e5dd0004206c561f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,73 +1,8 @@
-flask
-flask_restful
-flask_cors
-faiss-cpu
-fire
-h5py
--e git+https://github.com/facebookresearch/xformers.git@main#egg=xformers
-numpy>=1.23.5
-pandas<2.0.0
-peft>=0.12
 torch
-torchvision 
-torchaudio
-jupyterlab>=4.0.2
-notebook>=6.5.4
-albumentations>=1.1.0
-opencv-python>=4.2.0
-pudb~=2019.2
-imageio>=2.14.1
-imageio-ffmpeg>=0.4.7
-pytorch-lightning>=1.5.9
-omegaconf~=2.1.1
-gradio==4.21.0
-pillow~=9.5.0
-einops>=0.4.1
-torch-fidelity>=0.3.0
-setuptools>=59.5.0
-transformers
-torchmetrics>=0.6.0
-lpips
-image-reward
-kornia>=0.6
-diffusers>=0.18.0
-accelerate>=0.20.3
-safetensors
-datasets
-tqdm>=4.64.1
-matplotlib>=3.7.1
-taming-transformers-rom1504~=0.0.6
-madgrad>=1.1
--e git+https://github.com/openai/CLIP.git@main#egg=clip
-dominate>=2.8.0
--e git+https://github.com/CompVis/latent-diffusion.git#egg=latent-diffusion #ldm
-openai
-nltk~=3.8.1
-krippendorff
-statsmodels
-plotly
-git+https://github.com/TIGER-AI-Lab/ImagenHub.git#egg=imagen-hub
-git+https://github.com/TIGER-AI-Lab/VideoGenHub.git@arena#egg=videogen-hub
-fal_client
-open_clip_torch~=2.24.0
-decord
-huggingface_hub
-open-clip-torch-any-py3
-modelscope
-protobuf==3.20.*
-rotary_embedding_torch
-av
-oss2
-simplejson
-sortedcontainers
-natsort
-colossalai
-mmengine
-pre-commit
-pyav
-pyarrow
-tensorboard
-timm
-wandb
-pandarallel
-kaleido
+transformers==4.41.0
+Pillow
+gradio
+spaces
+multiprocess
+peft
+av
\ No newline at end of file