first commit

README.md

@@ -1,3 +1,76 @@
 ---
 license: mit
 ---
+
+## Model Summary
+
+Video-CCAM-4B is a lightweight Video-MLLM built on [Phi-3-Mini-4K-Instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct) and [SigLIP SO400M](https://huggingface.co/google/siglip-so400m-patch14-384).
+
+## Usage
+
+Inference using Huggingface transformers on NVIDIA GPUs. Requirements tested on python 3.10：
+```
+torch==2.1.0
+torchvision==0.16.0
+transformers==4.40.2
+peft==0.10.0
+pyarrow==13.0.0     # load parquet
+decord==0.6.0       # load video
+pysubs2==1.7.2      # load subtitle
+```
+
+### Sample Inference Code
+
+```
+import torch
+
+from eval import load_video
+from videoccam import VideoCCAM
+
+video_path = 'assets/example.mp4'
+question = 'Can you please describe what happens in the video in detail?'
+
+sample_config = dict(
+    sample_type='uniform',
+    num_frames=32
+)
+
+mllm = VideoCCAM(
+    model_path='.',
+    chat_template='<|user|>\n{input}<|end|>\n<|assistant|>\n',
+    generation_args=dict(
+        stop_tokens=['<|end|>', '<|endoftext|>'],
+        max_new_tokens=512,
+        do_sample=False,
+        num_beams=5,
+    ),
+    llm_name_or_path='microsoft/Phi-3-mini-4k-instruct',    # you can replace this with local directory if the model has been downloaded before
+    visual_encoder_name_or_path='google/siglip-so400m-patch14-384',     # you can replace this with local directory if the model has been downloaded before
+    special_tokens=['<time>', '</time>'],
+    visual_select_layer=-2,
+    torch_dtype=torch.bfloat16,
+    device_map='cuda:0'
+)
+
+frames, = load_video(video_path, **sample_config)
+response = mllm.generate(texts=[question], videos=[frames])[0]
+
+print(response)
+```
+
+### Video-MME Evaluation
+
+You are expected to reproduce the results of 48.2 (without subtitle) and 52.9 (with subtitle) by running the following command. By default, the results are saved as `output_w_sub.json` and `output_wo_sub.json` in local directory. We provide our results in `ref_results` directory.
+
+```
+python eval.py
+```
+
+## Acknowledgement
+
+* [xtuner](https://github.com/InternLM/xtuner): Video-CCAM-4B is trained using the xtuner framework. Thanks for their excellent works!
+* [Phi-3-Mini-4K-Instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct): Great small language models developed by Microsoft.
+* [SigLIP SO400M](https://huggingface.co/google/siglip-so400m-patch14-384): Outstanding vision encoder developed by Google.
+
+## License
+The model is licensed under the MIT license.

eval.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+================================================
+@author: Jaron
+@time: 2024/06/23 12:59:38
+@email: fjjth98@163.com
+@description: Evaluate MLLM on Video-MME Benchmark
+================================================
+"""
+
+import json
+import torch
+import pysubs2
+import os.path as osp
+
+from PIL import Image
+from tqdm import tqdm
+from typing import Any
+from copy import deepcopy
+from pandas import read_parquet
+from decord import VideoReader, cpu
+from torch.utils.data import Dataset, DataLoader, default_collate
+
+
+def video_collate_fn(batch: Any) -> Any:
+    """this collate function address dict video inputs, support to process variable number of frames for different inputs
+
+    Args:
+        batch (_type_): _description_
+
+    Returns:
+        _type_: _description_
+    """
+    if isinstance(batch[0], dict) and 'video' in batch[0]:
+        video = [b.pop('video') for b in batch]
+        batch = default_collate(batch)
+        batch['video'] = video
+    else:
+        batch = default_collate(batch)
+    return batch
+
+
+def uniform_indices(num_frames: int, total_frames: int) -> list[int]:
+    """Get uniform indices 
+
+    Args:
+        num_frames (int): number of frames
+        total_frames (int): total number of frames
+
+    Returns:
+        list[int]: Output frame indices
+    """
+    if num_frames < total_frames:
+        splits = torch.linspace(0, total_frames, num_frames+1, dtype=int)
+        indices = ((splits[:-1] + splits[1:]) // 2).tolist()
+    else:
+        indices = list(range(total_frames))
+
+    return indices
+
+
+def fps_indices(input_fps: float, total_frames: int, output_fps: float = None, max_num_frames: int = -1) -> list[int]:
+    """Get indices according to the output_fps
+
+    Args:
+        input_fps (float): input fps
+        total_frames (int): total number of frames
+        output_fps (float, optional): output fps. Defaults to None, means output_fps==input_fps.
+        max_num_frames (int, optional): max number of frames. Defaults to -1, means no limitation.
+
+    Returns:
+        list[int]: Output frame indices
+    """
+    delta = 1 if output_fps is None else input_fps / output_fps
+    indices = torch.arange(0, total_frames, delta).round().to(int)
+    indices = [e for e in indices if e < total_frames]
+    if 0 < max_num_frames < len(indices):
+        indices = indices[:max_num_frames]
+
+    return indices
+
+
+def load_video(src_path: str, sample_type: str, sub_path: str = None, **kwargs) -> list[Image.Image] | tuple[list[Image.Image], str]:
+    """Load video using decord, optionally load subtitles
+
+    Args:
+        src_path (str): video path
+        sample_type (str): 'uniform' or 'fps'
+        sub_path (str): subtitle path, .srt
+        kwargs: for 'uniform', require 'num_frames'; for 'fps', optionally require 'output_fps' and 'max_num_frames'
+
+    Returns:
+        list[Image.Image]: frame list
+    """
+    vr = VideoReader(src_path, ctx=cpu(0), num_threads=1)
+    total_frames = len(vr)
+    if sample_type == 'uniform':
+        num_frames = kwargs.pop('num_frames')
+        indices = uniform_indices(num_frames, total_frames)
+    elif sample_type == 'fps':
+        input_fps = float(vr.get_avg_fps())
+        output_fps = kwargs.pop('output_fps', None)
+        max_num_frames = kwargs.pop('max_num_frames', -1)
+        indices = fps_indices(input_fps, total_frames, output_fps, max_num_frames)
+    else:
+        raise ValueError(f'Do not support {sample_type} sample type')
+    frames = vr.get_batch(indices).asnumpy()        # (T, H, W, C), np.uint8
+    frames = [Image.fromarray(frame) for frame in frames]
+
+    if sub_path is None:
+        return frames
+    elif osp.exists(sub_path):
+        subs = pysubs2.load(sub_path, encoding='utf-8')
+        subtitles = []
+        for idx in indices:
+            sub_text = []
+            cur_time = pysubs2.make_time(fps=float(vr.get_avg_fps()), frames=idx)
+            for sub in subs:
+                if sub.end < cur_time:
+                    continue
+                elif sub.start < cur_time:
+                    sub_text.append(sub.text)
+                else:
+                    break
+            sub_text = ' '.join(sub_text)
+            if sub_text.strip():
+                subtitles.append(sub_text)
+        subtitles = '\n'.join(subtitles)
+        return frames, subtitles
+    else:
+        return frames, ''
+
+
+class VideoMMEDataset(Dataset):
+
+    def __init__(self, dataset_path: str, sample_config: dict, use_subtitle: bool = False):
+        super().__init__()
+        self.dataset_path = dataset_path
+        self.sample_config = sample_config
+        self.use_subtitle = use_subtitle
+
+        data_dict = {}
+        index_keys = ['video_id', 'duration', 'domain', 'sub_category', 'videoID']
+        value_keys = ['question_id', 'task_type', 'question', 'options', 'answer']
+        df = read_parquet(osp.join(dataset_path, 'videomme', 'test-00000-of-00001.parquet'))
+        df['options'] = df['options'].apply(list)
+        for _, data in df.iterrows():
+            key = tuple(data[k] for k in index_keys)
+            value = data[value_keys].to_dict()
+            if key in data_dict:
+                data_dict[key].append(value)
+            else:
+                data_dict[key] = [value]
+        self.data_list = [dict(zip(index_keys + ['questions'], list(k) + [v])) for k, v in data_dict.items()]
+
+    def __len__(self):
+        return len(self.data_list)
+
+    def __getitem__(self, idx) -> dict:
+        if self.use_subtitle:
+            frames, subtitles = load_video(
+                src_path=osp.join(self.dataset_path, 'video', self.data_list[idx]['videoID'] + '.mp4'),
+                sub_path=osp.join(self.dataset_path, 'subtitle', self.data_list[idx]['videoID'] + '.srt'),
+                **self.sample_config
+            )
+            text = ['\n'.join([
+                "This video's subtitles are listed below:",
+                subtitles,
+                'Select the best answer to the following multiple-choice question based on the video. Respond with only the letter (A, B, C, or D) of the correct option.',
+                i['question']
+            ] + i['options']) for i in self.data_list[idx]['questions']]
+        else:
+            frames = load_video(
+                src_path=osp.join(self.dataset_path, 'video', self.data_list[idx]['videoID'] + '.mp4'),
+                **self.sample_config
+            )
+            text = ['\n'.join([
+                'Select the best answer to the following multiple-choice question based on the video. Respond with only the letter (A, B, C, or D) of the correct option.',
+                i['question']
+            ] + i['options']) for i in self.data_list[idx]['questions']]
+            subtitles = ''
+
+        return dict(
+            video=frames,
+            text=text
+        )
+
+
+if __name__ == '__main__':
+
+    from videoccam import VideoCCAM, DEFAULT_VIDEO_TOKEN
+
+    mllm = VideoCCAM(
+        model_path='.',
+        chat_template='<|user|>\n{input}<|end|>\n<|assistant|>\n',
+        generation_args=dict(
+            stop_tokens=['<|end|>', '<|endoftext|>'],
+            max_new_tokens=512,
+            do_sample=False
+        ),
+        llm_name_or_path='microsoft/Phi-3-mini-4k-instruct',
+        visual_encoder_name_or_path='google/siglip-so400m-patch14-384',
+        special_tokens=['<time>', '</time>'],
+        visual_select_layer=-2,
+        torch_dtype=torch.bfloat16,
+        device_map='cuda:0'
+    )
+    mllm.eval()
+    
+    dataset = VideoMMEDataset(
+        dataset_path='your/dataset/path',
+        sample_config=dict(
+            sample_type='uniform',
+            num_frames=32
+        )
+    )
+
+    with torch.inference_mode():
+        for use_subtitle in (True,):
+            dataset.use_subtitle = use_subtitle
+            dataloader = DataLoader(
+                dataset,
+                batch_size=4,
+                num_workers=8,
+                shuffle=False,
+                pin_memory=True,
+                collate_fn=video_collate_fn
+            )
+            results = []
+            for data in tqdm(dataloader):
+                print(data['text'][0])
+                response, pixel_values = mllm.generate(
+                    texts=['\n'.join([DEFAULT_VIDEO_TOKEN, t]) for t in data['text'][0]],
+                    videos=data['video'],
+                    return_pixel_values=True
+                )
+                response = [response]
+                for i in range(1, len(data['text'])):
+                    response.append(mllm.generate(
+                        texts=['\n'.join([DEFAULT_VIDEO_TOKEN, t]) for t in data['text'][i]],
+                        pixel_values=pixel_values
+                    ))
+                response = [[response[i][j] for i in range(len(response))] for j in range(len(response[0]))]
+                results.extend(response)
+            
+            outputs = []
+            for data, responses in zip(dataset.data_list, results):
+                data = deepcopy(data)
+                data.pop('videoID')
+                for question, response in zip(data['questions'], responses):
+                    question['response'] = response
+                outputs.append(data)
+
+            suffix = 'w_sub' if use_subtitle else 'wo_sub'
+            with open(f'output_{suffix}.json', 'w') as f:
+                json.dump(outputs, f, indent=4, ensure_ascii=False)

llm_adapter/README.md

@@ -0,0 +1,202 @@
+---
+base_model: /group/40006/jaronfei/models/Phi-3-mini-4k-instruct
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

llm_adapter/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "/group/40006/jaronfei/models/Phi-3-mini-4k-instruct",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 512,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_up_proj",
+    "down_proj",
+    "o_proj",
+    "qkv_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

llm_adapter/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e10c0045b6d740a7b065e483419c6d468bb55e87d8369e1e6e2272d716481c9
+size 1610648152

projector/config.json

@@ -0,0 +1,38 @@
+{
+  "architectures": [
+    "CCAMModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoConfig": "configuration_ccam_projector.CCAMConfig",
+    "AutoModel": "modeling_ccam_projector.CCAMModel"
+  },
+  "cross_attention_frequency": 1,
+  "encoder_hidden_size": 1152,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1152,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "ccam_projector",
+  "num_attention_heads": 18,
+  "num_hidden_layers": 1,
+  "num_query_tokens": 1024,
+  "output_size": 3072,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "query_attn_mask_type": "full",
+  "spatial_pos_embed_type": "none",
+  "spatial_resolution": [
+    1,
+    1
+  ],
+  "temporal_pos_embed_type": "none",
+  "temporal_resolution": 16,
+  "torch_dtype": "float16",
+  "transformers_version": "4.41.2",
+  "visual_attn_mask_type": "ccam",
+  "vocab_size": 30522
+}

projector/configuration_ccam_projector.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+================================================
+@author: Jaron
+@time: 2024/02/20 16:37:16
+@email: fjjth98@163.com
+@description: different projector in Video-LLM
+================================================
+"""
+
+from transformers.models.blip_2.configuration_blip_2 import Blip2QFormerConfig
+
+
+class CCAMConfig(Blip2QFormerConfig):
+    model_type = 'ccam_projector'
+    _auto_class = 'AutoConfig'
+
+    def __init__(
+        self, 
+        spatial_pos_embed_type: str = 'learnable',      # ['none', 'learnable', 'cosine']
+        spatial_resolution: tuple[int, int] = (1, 1),   # (H, W)
+        temporal_pos_embed_type: str = 'learnable',
+        temporal_resolution: int = 0,                   # T
+        num_query_tokens: int = 512,
+        visual_attn_mask_type: str = 'ccam',        # ['ccam', 'full']
+        query_attn_mask_type: str = 'full',       # ['causal', 'full']
+        num_hidden_layers=1,
+        cross_attention_frequency=1,
+        output_size=4096,   # llm dimension
+        encoder_hidden_size=1024, # visual dimension
+        hidden_size=1024,
+        vocab_size=30522, num_attention_heads=16, intermediate_size=4096, hidden_act="gelu", hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, max_position_embeddings=512, initializer_range=0.02, layer_norm_eps=1e-12, pad_token_id=0, position_embedding_type="absolute", **kwargs):
+        super().__init__(vocab_size, hidden_size, num_hidden_layers, num_attention_heads, intermediate_size, hidden_act, hidden_dropout_prob, attention_probs_dropout_prob, max_position_embeddings, initializer_range, layer_norm_eps, pad_token_id, position_embedding_type, cross_attention_frequency, encoder_hidden_size, **kwargs)
+        self.spatial_pos_embed_type = spatial_pos_embed_type
+        self.spatial_resolution = spatial_resolution
+        self.temporal_pos_embed_type = temporal_pos_embed_type
+        self.temporal_resolution = temporal_resolution
+        self.num_query_tokens = num_query_tokens
+        self.visual_attn_mask_type = visual_attn_mask_type
+        self.query_attn_mask_type = query_attn_mask_type
+        self.output_size = output_size

projector/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86d539c564feda72963eb5a2f2d712112a29afa8cc42f3738cffbe38c637c18a
+size 47613856

projector/modeling_ccam_projector.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+================================================
+@author: Jaron
+@time: 2024/02/20 16:21:56
+@email: fjjth98@163.com
+@description: QFormer projector, convert image and video into fixed-length tokens
+================================================
+"""
+
+import math
+import torch
+import torch.nn as nn
+from torch.nn.functional import interpolate
+from transformers.models.blip_2.modeling_blip_2 import Blip2QFormerModel, Blip2QFormerEncoder
+
+from .configuration_ccam_projector import CCAMConfig
+
+
+class SimpleQFormerOutput(nn.Module):
+    # replace last residual MLP with normal MLP
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.intermediate_size, config.output_size)
+
+    def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor = None) -> torch.Tensor:
+        return self.dense(hidden_states)
+
+
+class SimpleQFormerIdentity(nn.Module):
+    # just to replace the first attention module with identity, since it is useless
+ 
+    def forward(self, hidden_states: torch.Tensor, *args, **kwargs) -> torch.Tensor:
+        return hidden_states,
+
+
+class CCAMModel(Blip2QFormerModel):
+    _auto_class = 'AutoModel'
+    config_class = CCAMConfig
+    base_model_prefix = 'model'
+    supports_gradient_checkpointing = True
+ 
+    def __init__(self, config: CCAMConfig):
+        super(Blip2QFormerModel, self).__init__(config)
+        self.gradient_checkpointing = False
+        self.config = config
+        self.num_query_tokens = config.num_query_tokens
+        self.visual_attn_mask_type = config.visual_attn_mask_type
+
+        self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+        self.encoder = Blip2QFormerEncoder(config)
+        self.encoder.layer[0].attention = SimpleQFormerIdentity()   # replace the 1st attention module with identity
+        self.encoder.layer[-1].output_query = SimpleQFormerOutput(config)
+
+        # initialize query tokens
+        self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.hidden_size))
+
+        # initialize pos embed
+        self.spatial_pos_embed = self._create_pos_embed(*config.spatial_resolution, type=config.spatial_pos_embed_type)     # (H, W, C)
+        self.temporal_pos_embed = self._create_pos_embed(config.temporal_resolution, type=config.temporal_pos_embed_type)   # (T, C)
+
+        # initialize query attn mask
+        if config.query_attn_mask_type == 'full':
+            self.query_attn_mask = None
+        elif config.query_attn_mask_type == 'causal':
+            query_attn_mask = torch.ones(self.num_query_tokens, self.num_query_tokens)
+            q = torch.arange(self.num_query_tokens)
+            query_attn_mask.masked_fill_(q > q[:, None], 0)
+            self.query_attn_mask = query_attn_mask[None]
+        else:
+            raise NotImplementedError(f'Do not support {self.query_attn_mask} query_attn_mask')
+
+        self.post_init()
+
+    def _create_pos_embed(self, *size: int, type: str = 'none') -> torch.Tensor:
+        C = self.config.encoder_hidden_size
+        if type == 'none':
+            pos_embed = None
+        elif type == 'learnable':
+            pos_embed = nn.Parameter(.02 * torch.randn(*size, C))
+        elif type == 'cosine':
+            total_len = 1
+            for i in size:
+                total_len *= i
+            raw = torch.outer(torch.arange(total_len), torch.exp(torch.arange(0, C, 2) * (-math.log(10000.) / C)))
+            pos_embed = nn.Parameter(torch.stack((raw.sin(), raw.cos()), dim=-1).view(*size, C), requires_grad=False)
+        else:
+            raise NotImplementedError(f'Do not support {type} position embeddings')
+        return pos_embed
+
+    def get_attn_mask(self, embeddings: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+        """Get visual_attn_mask and query_attn_mask if needed 
+            embeddings (torch.Tensor): (B, T, L, C)
+        """
+        B, T, L, _ = embeddings.size()
+        device = embeddings.device
+
+        # visual attn mask only work for videos
+        if T > 1:
+            if self.visual_attn_mask_type == 'ccam':
+                base_attn_mask = torch.ones(T, T, device=device)
+                t = torch.arange(T, device=device)
+                base_attn_mask.masked_fill_(t > t[:, None], 0)
+                visual_attn_mask = torch.cat((
+                    torch.kron(
+                        base_attn_mask,
+                        torch.ones(self.num_query_tokens // T, L, device=device)
+                    ),
+                    torch.ones(self.num_query_tokens % T, T * L, device=device)
+                ), dim=0)[None].expand(B, -1, -1)
+            elif self.attn_mask_type == 'full':
+                visual_attn_mask = None
+            else:
+                raise NotImplementedError(f'Do not support {self.visual_attn_mask_type} attn_mask')
+        else:
+            visual_attn_mask = None
+
+        if self.query_attn_mask is None:
+            query_attn_mask = None
+        else:
+            query_attn_mask = self.query_attn_mask.expand(B, -1, -1)
+
+        return visual_attn_mask, query_attn_mask
+
+    def batch_forward_no_spatial(self, visual_embeds: torch.Tensor) -> torch.Tensor:
+        """Batch forward without spatial mask position embeddings
+
+        Args:
+            visual_embeds (torch.Tensor): (B, T, L, C)
+
+        Returns:
+            torch.Tensor: (B, Q, C)
+        """
+        B, T, _, C = visual_embeds.size()
+        query_embeds = self.query_tokens.expand(B, -1, -1)
+        visual_attn_mask, query_attn_mask = self.get_attn_mask(visual_embeds)
+
+        # add temporal position embeddings
+        if self.temporal_pos_embed is not None:
+            if T == self.temporal_pos_embed.size(0):
+                pos_embed = self.temporal_pos_embed
+            elif T == 1:
+                pos_embed = 0. * self.temporal_pos_embed[:1]     # for deepspeed 
+            else:
+                pos_embed = interpolate(
+                    self.temporal_pos_embed.T[None],    # (1, C, t)
+                    size=(T,),
+                    mode='linear',
+                    align_corners=False
+                )[0].T      # (T, C)
+            visual_embeds = visual_embeds + pos_embed.view(1, T, 1, C)
+        visual_embeds = visual_embeds.flatten(1, 2)
+
+        return super().forward(
+            query_embeds=query_embeds,
+            attention_mask=query_attn_mask,
+            encoder_hidden_states=visual_embeds,
+            encoder_attention_mask=visual_attn_mask
+        )[0]
+
+    def forward(self, visual_embeds: torch.Tensor, split_sizes: list[int], unmasked_ids: torch.LongTensor = None):
+        """
+            visual_embeds (torch.Tensor): (T, L, C)
+            split_sizes (list[int]): [t0, t1, ...] sum_i ti=T
+            unmasked_ids (torch.LongTensor): If provided, should be in the shape of (T, L) whose value v 0<=v<=HW-1
+            output_attentions (_type_, optional): _description_. Defaults to None.
+            output_hidden_states (_type_, optional): _description_. Defaults to None.
+            return_dict (_type_, optional): _description_. Defaults to None.
+        """
+        _, L, C = visual_embeds.size()
+
+        # add spatial position embeddings
+        if self.spatial_pos_embed is not None:
+            pos_embed = self.spatial_pos_embed.view(-1, C)  # (H*W, C)
+            if unmasked_ids is None:
+                pos_embed = pos_embed.view(1, L, C)     # if not provided, L must equals to H*W
+            else:
+                pos_embed = pos_embed[unmasked_ids]     # (T, L, C)
+            visual_embeds = visual_embeds + pos_embed
+
+        # all inputs in this batch has the same t
+        if len(set(split_sizes)) == 1:
+            visual_embeds = visual_embeds.view(len(split_sizes), split_sizes[0], L, C)
+            output = self.batch_forward_no_spatial(visual_embeds)
+        else:
+            visual_embeds = visual_embeds.split(split_sizes, dim=0)
+            # group visual_embeds accoding to the number of frames
+            output, group_visual_embeds = [None] * len(split_sizes), {}
+            for idx, (embed, t) in enumerate(zip(visual_embeds, split_sizes)):
+                if t in group_visual_embeds:
+                    group_visual_embeds[t][0].append(idx)
+                    group_visual_embeds[t][1].append(embed)
+                else:
+                    group_visual_embeds[t] = [[idx], [embed]]
+            for idx, embeds in group_visual_embeds.values():
+                cur_output = self.batch_forward_no_spatial(torch.stack(embeds, dim=0))
+                for i, j in enumerate(idx):
+                    output[j] = cur_output[i]
+            output = torch.stack(output, dim=0)
+
+        return output

videoccam.py

@@ -0,0 +1,312 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+================================================
+@author: Jaron
+@time: 2024/06/23 09:52:24
+@email: fjjth98@163.com
+@description:
+================================================
+"""
+
+import torch
+import os.path as osp
+import torch.nn as nn
+
+from PIL import Image
+from peft import PeftModel
+from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, SiglipVisionModel, SiglipImageProcessor
+
+
+IGNORE_INDEX = -100
+IMAGE_TOKEN_INDEX = -200
+DEFAULT_IMAGE_TOKEN = '<image>'
+DEFAULT_VIDEO_TOKEN = '<video>'
+
+
+class VideoCCAM(nn.Module):
+
+    def __init__(
+        self,
+        model_path: str,
+        chat_template: str,
+        generation_args: dict,
+        llm_name_or_path: str = None,
+        visual_encoder_name_or_path: str = None,
+        special_tokens: list[str] = None,
+        visual_select_layer: int = -2,
+        torch_dtype: torch.dtype = torch.float16,
+        device_map: str = 'cuda:0'
+    ):
+        super().__init__()
+        self.chat_template = chat_template
+        self.generation_args = generation_args
+        self.visual_select_layer = visual_select_layer
+        self.torch_dtype = torch_dtype
+        self.device_map = device_map
+
+        if llm_name_or_path is None:
+            llm_name_or_path = model_path
+        if visual_encoder_name_or_path is None:
+            visual_encoder_name_or_path = osp.join(model_path, 'visual_encoder')
+            assert osp.exists(visual_encoder_name_or_path), f'{visual_encoder_name_or_path} does not exist, you have to specify `visual_encoder_name_or_path`'
+        projector_path = osp.join(model_path, 'projector')
+        assert osp.exists(projector_path), f'{projector_path} does not exist, you have to change `model_path`'
+
+        self.llm = AutoModelForCausalLM.from_pretrained(
+            llm_name_or_path,
+            trust_remote_code=True,
+            torch_dtype=torch_dtype,
+            device_map=device_map
+        )
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            llm_name_or_path,
+            trust_remote_code=True
+        )
+        print(f'Load LLM from {llm_name_or_path}')
+        if special_tokens is not None:
+            self.llm.resize_token_embeddings(self.llm.get_input_embeddings().weight.size(0) + len(special_tokens))
+            self.llm.requires_grad_(False)
+            self.llm.get_input_embeddings().weight[-len(special_tokens):].zero_()
+            self.tokenizer.add_tokens(special_tokens, special_tokens=True)
+            print(f'Add special_tokens {special_tokens} to LLM and tokenizer')
+        if osp.exists(adapter_path := osp.join(model_path, 'llm_adapter')):
+            self.llm = PeftModel.from_pretrained(self.llm, adapter_path)
+            print(f'Load LLM adapter from {adapter_path}')
+        self.generation_args['eos_token_id'] = self.tokenizer.convert_tokens_to_ids(self.generation_args.pop('stop_tokens'))
+
+        self.visual_encoder = SiglipVisionModel.from_pretrained(
+            visual_encoder_name_or_path,
+            torch_dtype=torch_dtype,
+            device_map=device_map
+        )
+        self.image_processor = SiglipImageProcessor.from_pretrained(visual_encoder_name_or_path)
+        print(f'Load SigLIP visual encoder from {visual_encoder_name_or_path}')
+        if osp.exists(adapter_path := osp.join(model_path, 'visual_encoder_adapter')):
+            self.visual_encoder = PeftModel.from_pretrained(self.visual_encoder, adapter_path)
+            print(f'Load visual_encoder adapter from {adapter_path}')
+
+        self.projector = AutoModel.from_pretrained(
+            projector_path,
+            torch_dtype=torch_dtype,
+            device_map=device_map,
+            trust_remote_code=True
+        )
+        print(f'Load projector from {projector_path}')
+
+    # Modified from https://github.com/InternLM/xtuner/blob/main/xtuner/model/utils.py#L138
+    def prepare_inputs_labels_for_multimodal(
+        self,
+        input_ids: torch.LongTensor = None,
+        position_ids: torch.LongTensor = None,
+        attention_mask: torch.Tensor = None,
+        past_key_values: list[torch.FloatTensor] = None,
+        labels: torch.LongTensor = None,
+        pixel_values: torch.FloatTensor = None
+    ):
+        if pixel_values is None:
+            return {
+                'input_ids': input_ids,
+                'position_ids': position_ids,
+                'attention_mask': attention_mask,
+                'past_key_values': past_key_values,
+                'inputs_embeds': None,
+                'labels': labels
+            }
+
+        _labels = labels
+        _position_ids = position_ids
+        _attention_mask = attention_mask
+        if attention_mask is None:
+            if isinstance(input_ids, torch.Tensor):
+                attention_mask = torch.ones_like(input_ids, dtype=torch.bool)
+            elif isinstance(input_ids, list):
+                attention_mask = [torch.ones_like(i, dtype=torch.bool) for i in input_ids]
+                _attention_mask = attention_mask
+            else:
+                raise ValueError(f'Do not support {type(input_ids)} type as input_ids')
+        else:
+            attention_mask = attention_mask.bool()
+        if position_ids is None:
+            position_ids = torch.arange(
+                0, input_ids[0].shape[0], dtype=torch.long, device=input_ids[0].device)
+        if labels is None:
+            if isinstance(input_ids, torch.Tensor):
+                labels = torch.full_like(input_ids, IGNORE_INDEX)
+            elif isinstance(input_ids, list):
+                labels = [torch.full_like(i, IGNORE_INDEX) for i in input_ids]
+            else:
+                raise ValueError(f'Do not support {type(input_ids)} type as input_ids')
+
+        # remove the padding using attention_mask -- TODO: double check
+        input_ids = [
+            cur_input_ids[cur_attention_mask]
+            for cur_input_ids, cur_attention_mask in zip(input_ids, attention_mask)
+        ]
+        labels = [
+            cur_labels[cur_attention_mask]
+            for cur_labels, cur_attention_mask in zip(labels, attention_mask)
+        ]
+
+        new_inputs_embeds = []
+        new_labels = []
+        cur_image_idx = 0
+        for batch_idx, cur_input_ids in enumerate(input_ids):
+            num_images = (cur_input_ids == IMAGE_TOKEN_INDEX).sum()
+            if num_images == 0:
+                cur_pixel_values = pixel_values[cur_image_idx]
+                cur_inputs_embeds_1 = self.llm.get_input_embeddings()(cur_input_ids)
+                cur_inputs_embeds = torch.cat(
+                    [cur_inputs_embeds_1, cur_pixel_values[0:0]], dim=0)
+                new_inputs_embeds.append(cur_inputs_embeds)
+                new_labels.append(labels[batch_idx])
+                cur_image_idx += 1
+                continue
+
+            image_token_indices = [-1] + torch.where(
+                cur_input_ids == IMAGE_TOKEN_INDEX)[0].tolist() + [
+                    cur_input_ids.shape[0]
+                ]
+            cur_input_ids_noim = []
+            cur_labels = labels[batch_idx]
+            cur_labels_noim = []
+            for i in range(len(image_token_indices) - 1):
+                cur_input_ids_noim.append(cur_input_ids[image_token_indices[i] +
+                                                        1:image_token_indices[i +
+                                                                            1]])
+                cur_labels_noim.append(cur_labels[image_token_indices[i] +
+                                                1:image_token_indices[i + 1]])
+            split_sizes = [x.shape[0] for x in cur_labels_noim]
+            cur_inputs_embeds = self.llm.get_input_embeddings()(
+                torch.cat(cur_input_ids_noim))
+            cur_inputs_embeds_no_im = torch.split(
+                cur_inputs_embeds, split_sizes, dim=0)
+            cur_new_inputs_embeds = []
+            cur_new_labels = []
+
+            for i in range(num_images + 1):
+                cur_new_inputs_embeds.append(cur_inputs_embeds_no_im[i])
+                cur_new_labels.append(cur_labels_noim[i])
+                if i < num_images:
+                    cur_pixel_values = pixel_values[cur_image_idx]
+                    cur_image_idx += 1
+                    cur_new_inputs_embeds.append(cur_pixel_values)
+                    cur_new_labels.append(
+                        torch.full((cur_pixel_values.shape[0], ),
+                                IGNORE_INDEX,
+                                device=cur_labels.device,
+                                dtype=cur_labels.dtype))
+
+            cur_new_inputs_embeds = torch.cat(cur_new_inputs_embeds)
+            cur_new_labels = torch.cat(cur_new_labels)
+
+            new_inputs_embeds.append(cur_new_inputs_embeds)
+            new_labels.append(cur_new_labels)
+
+        # Combine them
+        max_len = max(x.shape[0] for x in new_inputs_embeds)
+        batch_size = len(new_inputs_embeds)
+
+        new_inputs_embeds_padded = []
+        new_labels_padded = torch.full((batch_size, max_len),
+                                    IGNORE_INDEX,
+                                    dtype=new_labels[0].dtype,
+                                    device=new_labels[0].device)
+        attention_mask = torch.zeros((batch_size, max_len),
+                                    dtype=attention_mask[0].dtype,
+                                    device=attention_mask[0].device)
+        position_ids = torch.zeros((batch_size, max_len),
+                                dtype=position_ids.dtype,
+                                device=position_ids.device)
+
+        for i, (cur_new_embed,
+                cur_new_labels) in enumerate(zip(new_inputs_embeds, new_labels)):
+            cur_len = cur_new_embed.shape[0]
+            new_inputs_embeds_padded.append(
+                torch.cat((cur_new_embed,
+                        torch.zeros((max_len - cur_len, cur_new_embed.shape[1]),
+                                    dtype=cur_new_embed.dtype,
+                                    device=cur_new_embed.device)),
+                        dim=0))
+            if cur_len > 0:
+                new_labels_padded[i, :cur_len] = cur_new_labels
+                attention_mask[i, :cur_len] = True
+                position_ids[i, :cur_len] = torch.arange(
+                    0,
+                    cur_len,
+                    dtype=position_ids.dtype,
+                    device=position_ids.device)
+
+        new_inputs_embeds = torch.stack(new_inputs_embeds_padded, dim=0)
+
+        if _labels is None:
+            new_labels = None
+        else:
+            new_labels = new_labels_padded
+
+        if _attention_mask is None:
+            attention_mask = None
+        elif isinstance(_attention_mask, list):
+            attention_mask = attention_mask.to(dtype=_attention_mask[0].dtype)
+        else:
+            attention_mask = attention_mask.to(dtype=_attention_mask.dtype)
+
+        if _position_ids is None:
+            position_ids = None
+
+        return {
+            'input_ids': None,
+            'position_ids': position_ids,
+            'attention_mask': attention_mask,
+            'past_key_values': past_key_values,
+            'inputs_embeds': new_inputs_embeds,
+            'labels': new_labels
+        }
+
+    def generate(
+        self,
+        texts: list[str],
+        videos: list[list[Image.Image]] = None,
+        pixel_values: torch.Tensor = None,
+        return_pixel_values: bool = False
+    ) -> list[str] | tuple[list[str], torch.Tensor]:
+        """Genrate respoonse for video and text inputs.
+
+        Args:
+            text (list[str]): list of text inputs
+            video (list[list[Image.Image]], optional): list of frame list. Defaults to None.
+            pixel_values (torch.Tensor, optional): precomputed pixel_values. Defaults to None.
+            return_pixel_values (bool, optional): whether return pixel values or not. Defaults to False.
+
+        Returns:
+            list[str]: _description_
+        """
+        prediction = []
+        # Get visual embeddings
+        if pixel_values is None:
+            frames, split_sizes = [], []
+            for i in videos:
+                frames += i
+                split_sizes.append(len(i))
+            pixel_values = self.image_processor(frames, return_tensors='pt')['pixel_values'].to(self.torch_dtype).to(self.device_map)
+            pixel_values = self.visual_encoder(pixel_values, output_hidden_states=True).hidden_states[self.visual_select_layer]
+            pixel_values = self.projector(pixel_values, split_sizes)
+
+        for i, t in enumerate(texts):
+            et = self.chat_template.format(input=t).replace(DEFAULT_VIDEO_TOKEN, DEFAULT_IMAGE_TOKEN).split(DEFAULT_IMAGE_TOKEN)
+            assert len(et) == 2, f'Wrong input formats for {t}'
+            input_ids = [torch.tensor(self.tokenizer.encode(et[0]) + [IMAGE_TOKEN_INDEX] + self.tokenizer.encode(et[1], add_special_tokens=False), device=self.device_map)]
+            mm_inputs = self.prepare_inputs_labels_for_multimodal(
+                input_ids=input_ids,
+                pixel_values=pixel_values[i:i+1]
+            )
+            generate_output = self.llm.generate(
+                **mm_inputs,
+                **self.generation_args
+            )[0]
+            prediction.append(self.tokenizer.decode(generate_output, skip_special_tokens=True))
+
+        if return_pixel_values:
+            return prediction, pixel_values
+        else:
+            return prediction

visual_encoder_adapter/README.md

@@ -0,0 +1,202 @@
+---
+base_model: /group/40006/jaronfei/models/siglip-so400m-patch14-384
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

visual_encoder_adapter/adapter_config.json

@@ -0,0 +1,36 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": {
+    "base_model_class": "SiglipVisionModel",
+    "parent_library": "xtuner.model.modules.visual_encoder.factory"
+  },
+  "base_model_name_or_path": "/group/40006/jaronfei/models/siglip-so400m-patch14-384",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "fc2",
+    "q_proj",
+    "v_proj",
+    "fc1",
+    "out_proj"
+  ],
+  "task_type": null,
+  "use_dora": false,
+  "use_rslora": false
+}

visual_encoder_adapter/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89c005eb2005adfb28cd7e9888b3333e73422b5c0ea3f47dd6abdb21cef00328
+size 71302368