Upload folder using huggingface_hub

llm/pytorch_model-00001-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7f508240dba71e8272ef2de0331db16b0aad7b90b6de3a414f5251d2d514967
+oid sha256:f08ed91f9640a98601c3605499a586c3410914e611cf0ee85cc1f75d7061add2
 size 1947779738

llm/pytorch_model-00002-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:098d87c2b14c05b788077baf97bf0c6d85873546ec09f94e29714802133d3aba
+oid sha256:ad177ce1be7bc0f3d41f56c762e7d2d246b4dd32ab8cf10a095fdf9e81e14b89
 size 1903236688

llm/pytorch_model-00003-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aed829f676a1c69618e42023a0d543f757076b507a51406a14b3b6a0c08e0738
+oid sha256:cc698a0828a2ca8d5c3783c938e9a78442db3d2bf649af0915c4bc914bc8c215
 size 1903236688

llm/pytorch_model-00004-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:af90eb90d789cd5897aa9d6567a5fa4b6417ffc8d6d467a0c73df2d5bfa9fa79
+oid sha256:d3beb2415facca8bb85c0bcb45f94dda38afdc72bc4f8ea0f1ffeec729618e32
 size 1903236688

llm/pytorch_model-00005-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93c201b2cb75f6271c87e9dcae83dcf654c46fbd108698d01930174e173db4e6
+oid sha256:d107db54fe677de4296079bbcdf7e9b7a255cf2849d908e344bfe03a6dfcdd6a
 size 1903236752

llm/pytorch_model-00006-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8caa1e97eba055aff5bef97344f231bac6f3c1af7d59b5cd892c9559d897f857
+oid sha256:e273b5053a19ec8794f324272b19f659ad171a13df4ccbac4d69e80380c2a68e
 size 1903236752

llm/pytorch_model-00007-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9968ee2634d9e1be745ad3094804938d73f52759babaa0fecba92dd6e429d7ea
+oid sha256:bd1fb9b44a08ce42901f385d774d7139eedd3df5cda692b6fd14a174d51e09e6
 size 1903236752

llm/pytorch_model-00008-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20d5aff66a5250c276b640b235ac1127d57ce0f99fae22f67a8ab3fca0b48b18
+oid sha256:410b638a1d006ebc402a8b39f8b8a6befab94bfc9a015762659b97d7d9cc0f2d
 size 1903236752

llm/pytorch_model-00009-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70f37c7c0355b7f8946880f5a4d00827a9590f1cce334455ab84dcacc454c519
+oid sha256:0372d10309ef62b762a7f983e011162d897bb54fd4846836483befe04c1741f6
 size 1903236752

llm/pytorch_model-00010-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10263095ba406a983567fbeeb9c9eef9be441d71d231d230cbd6cd0544f9e7b9
+oid sha256:1348d915b61f874177bf4bc3c419d3e9790bf12d5cad827a8970cc2aea58c768
 size 1903236752

llm/pytorch_model-00011-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:340a1bf3dc346cf46118cd9e3d4323bcf43ba36a4223d4bb2a54706c0c799795
+oid sha256:150ec73a71f07471e2a2a2c7165febca406c81400e5ed8e9801c9241e8d0bda1
 size 1903236752

llm/pytorch_model-00012-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8936e2cd54590913d572390a199e24ef8e2545182848cf4488b1ec072fb4b62
+oid sha256:2e853bbc594dd3bfd4b7df0395719876478a7a86efa14e3f0e6ee8c4fa37db50
 size 1903236752

llm/pytorch_model-00013-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7bf405e29337f52f4fb5858db29aa1a5e4b9c518bdda14f693ccfa7f4ba16b1b
+oid sha256:216f2f90e253a2b172b183b5ed9104647a7f5bdd68598caba086e6617c1c4483
 size 1903236752

llm/pytorch_model-00014-of-00014.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53aa958d86ba409b763bfec650e727e83b700bcc521e6fa937ff8ada5c1e2224
+oid sha256:9dd3358d15dac64ee9bac9ee571605e5b26b4ce891ed0aaf5e43b14e3d8d68a9
 size 1245241080

llm_adapter/adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "gate_proj",
     "v_proj",
-    "up_proj",
+    "q_proj",
     "o_proj",
     "k_proj",
-    "q_proj",
-    "down_proj",
-    "gate_proj"
+    "up_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

llm_adapter/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a17317836e0db05dfee44bad2c4f2890207421c2b7151a695afd077ed05cc567
-size 4005637552
+oid sha256:6ce89af9bf87941d625643d64c8e807fcdd16f00149f458e82b695732b667d64
+size 8011198024

projector/config.json

@@ -12,6 +12,6 @@
   "llm_hidden_size": 5120,
   "model_type": "projector",
   "torch_dtype": "float32",
-  "transformers_version": "4.37.2",
+  "transformers_version": "4.36.0",
   "visual_hidden_size": 1280
 }

projector/configuration_projector.py

@@ -3,15 +3,15 @@ from transformers import PretrainedConfig
 
 
 class ProjectorConfig(PretrainedConfig):
-    model_type = "projector"
-    _auto_class = "AutoConfig"
+    model_type = 'projector'
+    _auto_class = 'AutoConfig'
 
     def __init__(
         self,
         visual_hidden_size=4096,
         llm_hidden_size=4096,
         depth=2,
-        hidden_act="gelu",
+        hidden_act='gelu',
         bias=True,
         **kwargs,
     ):

projector/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c2126a4ee2ac250fe6f7e67c5eb00167494e852dee344949e0f43b1c4dfe7b2
+oid sha256:53a362f3b6e223ec295c65441f7af338c113046c0367bd6111eab5b9c6d7c668
 size 131113328

projector/modeling_projector.py

@@ -8,22 +8,32 @@ from .configuration_projector import ProjectorConfig
 
 
 class ProjectorModel(PreTrainedModel):
-    _auto_class = "AutoModel"
+    _auto_class = 'AutoModel'
     config_class = ProjectorConfig
-    base_model_prefix = "model"
+    base_model_prefix = 'model'
     supports_gradient_checkpointing = True
 
     def __init__(self, config: ProjectorConfig) -> None:
         super().__init__(config)
         self.gradient_checkpointing = False
 
-        modules = [nn.Linear(config.visual_hidden_size, config.llm_hidden_size, bias=config.bias)]
+        modules = [
+            nn.Linear(
+                config.visual_hidden_size,
+                config.llm_hidden_size,
+                bias=config.bias)
+        ]
         for _ in range(1, config.depth):
             modules.append(ACT2FN[config.hidden_act])
-            modules.append(nn.Linear(config.llm_hidden_size, config.llm_hidden_size, bias=config.bias))
+            modules.append(
+                nn.Linear(
+                    config.llm_hidden_size,
+                    config.llm_hidden_size,
+                    bias=config.bias))
         self.model = nn.Sequential(*modules)
 
     def enable_input_require_grads(self):
+
         def make_inputs_require_grad(module, input, output):
             output.requires_grad_(True)
 

visual_encoder_adapter/adapter_config.json

@@ -1,8 +1,8 @@
 {
   "alpha_pattern": {},
   "auto_mapping": {
-    "base_model_class": "PikaVidEncoder",
-    "parent_library": "xtuner.model.video_encoder"
+    "base_model_class": "CLIPVisionModel",
+    "parent_library": "transformers.models.clip.modeling_clip"
   },
   "base_model_name_or_path": "apple/DFN5B-CLIP-ViT-H-14-378",
   "bias": "none",
@@ -22,12 +22,12 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "out_proj",
+    "fc1",
     "v_proj",
+    "q_proj",
     "k_proj",
-    "fc1",
     "fc2",
-    "q_proj"
+    "out_proj"
   ],
   "task_type": null
 }

visual_encoder_adapter/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0915794d1b139877e627831e1d36d2ef378872141a4c3712c220d00f6e326a74
+oid sha256:43d1f05a283041e0ba93f16f0af1c20b3c3302f2a5305d3dba36aad45671f306
 size 188800496

vit/pytorch_model-00001-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4611f601856556b5a54e0631251c9279a2610b42825a51519430efc37d1a64ce
+oid sha256:3b52fb32d49c7403927a82093851cb3efbd2deb98885356c8d068b6843cc5a10
 size 1994332295

vit/pytorch_model-00002-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d0a9038c66b6348e1b86889c0a96bcb942b95e53c950ec85ca51345cd9318e51
+oid sha256:85a66172a48b3dfd33504ba0a2e4ee44f5f063644a262c7e64d8e0ed4cb17b58
 size 531341514

xtuner_config.py

@@ -7,47 +7,44 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer,
                           BitsAndBytesConfig, 
                           CLIPImageProcessor, CLIPVisionModel)
 from mmengine.optim import AmpOptimWrapper, CosineAnnealingLR, LinearLR
+
 from peft import LoraConfig
-from math import sqrt
 from torch.optim import AdamW
-from xtuner.dataset import VideoDataset, PikaDataset, ConcatDataset, ShareGPTVideoDataset
+from xtuner.dataset import PikaDataset, ConcatDataset
 from xtuner.dataset.collate_fns import default_collate_fn
-from xtuner.dataset.map_fns import llava_video_map_fn, llava_map_fn, pika_map_fn, template_map_fn_factory
+from xtuner.dataset.map_fns import llava_map_fn, m3it_map_fn, template_map_fn_factory
 from xtuner.dataset.samplers import LengthGroupedSampler
 from xtuner.engine import DatasetInfoHook, EvaluateChatHook
 from xtuner.model import PikaModel, PikaVidEncoder
 from xtuner.utils import PROMPT_TEMPLATE
 
-
+    
 #######################################################################
 #                          PART 1  Settings                           #
 #######################################################################
 # Model
 llm_name_or_path = 'lmsys/vicuna-13b-v1.5-16k'
 visual_encoder_name_or_path = 'apple/DFN5B-CLIP-ViT-H-14-378'
-# Specify the s3 pretrained pth
-pretrained_pth = 'work_dirs/13b_16k_s5/iter_400.pth'
-prompt_template = PROMPT_TEMPLATE.vicuna
+# Specify the s2 pretrained pth
+pretrained_pth = 'work_dirs/13b_16k_s2/epoch_1.pth'
 
+prompt_template = PROMPT_TEMPLATE.vicuna
+max_length = 4096
 size = 378
-# None for sampling all the video frames
-n_sample_frames = 32
-visual_token_merge_ratio = 0.1
-accumulative_counts = 32
-lr = 1e-4
-batch_size = 1  # per_device can only be set to 1 to support image and video mix training
 
-max_length = 4096
+batch_size = 16  # per_device
+accumulative_counts = 1
+lr = 2e-4
 dataloader_num_workers = 0
 max_epochs = 1
 optim_type = AdamW
 betas = (0.9, 0.999)
-weight_decay = 0.1
+weight_decay = 0
 max_norm = 1  # grad clip
 warmup_ratio = 0.03
 
 # Save
-save_steps = 500
+save_steps = 100
 save_total_limit = 2  # Maximum checkpoints to keep (-1 means unlimited)
 
 #######################################################################
@@ -93,21 +90,21 @@ model = dict(
         bias='none',
         task_type='CAUSAL_LM'),
     visual_encoder=dict(
-        # type=CLIPVisionModel.from_pretrained,
-        type=PikaVidEncoder.from_pretrained,
-        pretrained_model_name_or_path=visual_encoder_name_or_path,
-        visual_token_merge_ratio=visual_token_merge_ratio),
+        type=CLIPVisionModel.from_pretrained,
+        pretrained_model_name_or_path=visual_encoder_name_or_path),
     visual_encoder_lora=dict(
-        type=LoraConfig, r=64, lora_alpha=16, lora_dropout=0.05, bias='none'),
-)
-
+        type=LoraConfig,
+        r=64, 
+        lora_alpha=16, 
+        lora_dropout=0.05, 
+        bias='none'))
 
 #######################################################################
 #                      PART 3  Dataset & Dataloader                   #
 #######################################################################
-allava_image_caption_dataset = dict(
+llava_dataset = dict(
     type=PikaDataset,
-    data_path='./data/image_finetune/ALLaVA-Caption-LAION-4V',
+    data_path='./data/image_finetune/llava_v1_5_mix665k',
     image_folder='./data/image_data',
     tokenizer=tokenizer,
     image_processor=image_processor,
@@ -115,40 +112,35 @@ allava_image_caption_dataset = dict(
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     max_length=max_length,
-    pad_image_to_square=False,
-    keep_aspect_ratio=True,)
+    pad_image_to_square=True)
 
-sharegpt4v_video_caption_dataset = dict(
-    type=ShareGPTVideoDataset,
-    data_path='./data/video_finetune/sharegptvideo_caption_full_frame',
-    image_folder='./data/video_data/sharegptvideo_900k',
+train_dataset = dict(
+    type=PikaDataset,
+    data_path='./data/stage_3_part2',
+    image_folder='./data/image_data',
     tokenizer=tokenizer,
     image_processor=image_processor,
-    dataset_map_fn=llava_video_map_fn,
+    dataset_map_fn=llava_map_fn,
     template_map_fn=dict(
         type=template_map_fn_factory, template=prompt_template),
     max_length=max_length,
-    pad_image_to_square=False,
-    frame_number=n_sample_frames,
-    keep_aspect_ratio=True,)
+    pad_image_to_square=True)
 
-# mix video and image
 train_dataset = dict(
     type=ConcatDataset,
     datasets=[
-        allava_image_caption_dataset,
-        sharegpt4v_video_caption_dataset,
-    ])
+        llava_dataset,  
+        train_dataset])
 
 train_dataloader = dict(
     batch_size=batch_size,
     num_workers=dataloader_num_workers,
     dataset=train_dataset,
+    sampler=dict(type=DefaultSampler, shuffle=True),
     # sampler=dict(
     #     type=LengthGroupedSampler,
     #     length_property='modality_length',
     #     per_device_batch_size=batch_size * accumulative_counts),
-    sampler=dict(type=DefaultSampler, shuffle=True),
     collate_fn=dict(type=default_collate_fn))
 
 #######################################################################
@@ -190,7 +182,7 @@ train_cfg = dict(by_epoch=True, max_epochs=max_epochs, val_interval=1)
 #                           PART 5  Runtime                           #
 #######################################################################
 # Evaluate the generation performance during the training
-evaluation_freq = 500
+evaluation_freq = 100
 SYSTEM = ''
 evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg'
 evaluation_inputs = ['请描述一下这张照片', 'Please describe this picture']
@@ -219,12 +211,7 @@ default_hooks = dict(
     # enable the parameter scheduler.
     param_scheduler=dict(type=ParamSchedulerHook),
     # save checkpoint per epoch.
-    # checkpoint=dict(type=CheckpointHook, interval=1),
-    checkpoint=dict(
-        type=CheckpointHook,
-        by_epoch=False,
-        interval=save_steps,
-        max_keep_ckpts=save_total_limit),
+    checkpoint=dict(type=CheckpointHook, interval=1),
     # set sampler seed in distributed evrionment.
     sampler_seed=dict(type=DistSamplerSeedHook),
 )