import torch from mmengine.dataset import DefaultSampler from mmengine.hooks import (CheckpointHook, DistSamplerSeedHook, IterTimerHook, LoggerHook, ParamSchedulerHook) from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, CLIPImageProcessor, CLIPVisionModel) from mmengine.optim import AmpOptimWrapper, CosineAnnealingLR, LinearLR from peft import LoraConfig from torch.optim import AdamW from xtuner.dataset import PikaDataset, ConcatDataset from xtuner.dataset.collate_fns import default_collate_fn from xtuner.dataset.map_fns import llava_map_fn, m3it_map_fn, template_map_fn_factory from xtuner.dataset.samplers import LengthGroupedSampler from xtuner.engine import DatasetInfoHook, EvaluateChatHook from xtuner.model import PikaModel, PikaVidEncoder from xtuner.utils import PROMPT_TEMPLATE ####################################################################### # PART 1 Settings # ####################################################################### # Model llm_name_or_path = 'lmsys/vicuna-7b-v1.5-16k' visual_encoder_name_or_path = 'apple/DFN5B-CLIP-ViT-H-14-378' # Specify the s2 pretrained pth pretrained_pth = 'work_dirs/7b_16k_s2/epoch_1.pth' prompt_template = PROMPT_TEMPLATE.vicuna max_length = 4096 size = 378 batch_size = 16 # per_device accumulative_counts = 1 lr = 2e-4 dataloader_num_workers = 0 max_epochs = 1 optim_type = AdamW betas = (0.9, 0.999) weight_decay = 0 max_norm = 1 # grad clip warmup_ratio = 0.03 # Save save_steps = 100 save_total_limit = 2 # Maximum checkpoints to keep (-1 means unlimited) ####################################################################### # PART 2 Model & Tokenizer & Image Processor # ####################################################################### tokenizer = dict( type=AutoTokenizer.from_pretrained, pretrained_model_name_or_path=llm_name_or_path, trust_remote_code=True, padding_side='right') image_processor = dict( type=CLIPImageProcessor.from_pretrained, pretrained_model_name_or_path='laion/CLIP-ViT-bigG-14-laion2B-39B-b160k', trust_remote_code=True, size=size, crop_size=size) model = dict( type=PikaModel, freeze_llm=True, freeze_visual_encoder=True, pretrained_pth=pretrained_pth, llm=dict( type=AutoModelForCausalLM.from_pretrained, pretrained_model_name_or_path=llm_name_or_path, trust_remote_code=True, torch_dtype=torch.float16, quantization_config=dict( type=BitsAndBytesConfig, load_in_4bit=True, load_in_8bit=False, llm_int8_threshold=6.0, llm_int8_has_fp16_weight=False, bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type='nf4')), llm_lora=dict( type=LoraConfig, r=512, lora_alpha=256, lora_dropout=0.05, bias='none', task_type='CAUSAL_LM'), visual_encoder=dict( type=CLIPVisionModel.from_pretrained, pretrained_model_name_or_path=visual_encoder_name_or_path), visual_encoder_lora=dict( type=LoraConfig, r=64, lora_alpha=16, lora_dropout=0.05, bias='none')) ####################################################################### # PART 3 Dataset & Dataloader # ####################################################################### llava_dataset = dict( type=PikaDataset, data_path='./data/image_finetune/llava_v1_5_mix665k', image_folder='./data/image_data', tokenizer=tokenizer, image_processor=image_processor, dataset_map_fn=llava_map_fn, template_map_fn=dict( type=template_map_fn_factory, template=prompt_template), max_length=max_length, pad_image_to_square=True) train_dataset = dict( type=PikaDataset, data_path='./data/stage_3_part2', image_folder='./data/image_data', tokenizer=tokenizer, image_processor=image_processor, dataset_map_fn=llava_map_fn, template_map_fn=dict( type=template_map_fn_factory, template=prompt_template), max_length=max_length, pad_image_to_square=True) train_dataset = dict( type=ConcatDataset, datasets=[ llava_dataset, train_dataset]) train_dataloader = dict( batch_size=batch_size, num_workers=dataloader_num_workers, dataset=train_dataset, sampler=dict(type=DefaultSampler, shuffle=True), # sampler=dict( # type=LengthGroupedSampler, # length_property='modality_length', # per_device_batch_size=batch_size * accumulative_counts), collate_fn=dict(type=default_collate_fn)) ####################################################################### # PART 4 Scheduler & Optimizer # ####################################################################### # optimizer optim_wrapper = dict( type=AmpOptimWrapper, optimizer=dict( type=optim_type, lr=lr, betas=betas, weight_decay=weight_decay), clip_grad=dict(max_norm=max_norm, error_if_nonfinite=False), accumulative_counts=accumulative_counts, loss_scale='dynamic', dtype='float16') # learning policy # More information: https://github.com/open-mmlab/mmengine/blob/main/docs/en/tutorials/param_scheduler.md # noqa: E501 param_scheduler = [ dict( type=LinearLR, start_factor=1e-5, by_epoch=True, begin=0, end=warmup_ratio * max_epochs, convert_to_iter_based=True), dict( type=CosineAnnealingLR, eta_min=0.0, by_epoch=True, begin=warmup_ratio * max_epochs, T_max=max_epochs, convert_to_iter_based=True) ] # train, val, test setting train_cfg = dict(by_epoch=True, max_epochs=max_epochs, val_interval=1) ####################################################################### # PART 5 Runtime # ####################################################################### # Evaluate the generation performance during the training evaluation_freq = 100 SYSTEM = '' evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg' evaluation_inputs = ['请描述一下这张照片', 'Please describe this picture'] # Log the dialogue periodically during the training process, optional custom_hooks = [ dict(type=DatasetInfoHook, tokenizer=tokenizer), dict( type=EvaluateChatHook, tokenizer=tokenizer, image_processor=image_processor, every_n_iters=evaluation_freq, evaluation_inputs=evaluation_inputs, evaluation_images=evaluation_images, system=SYSTEM, prompt_template=prompt_template) ] # configure default hooks default_hooks = dict( # record the time of every iteration. timer=dict(type=IterTimerHook), # print log every 100 iterations. logger=dict(type=LoggerHook, interval=10), # enable the parameter scheduler. param_scheduler=dict(type=ParamSchedulerHook), # save checkpoint per epoch. checkpoint=dict(type=CheckpointHook, interval=1), # set sampler seed in distributed evrionment. sampler_seed=dict(type=DistSamplerSeedHook), ) # configure environment env_cfg = dict( # whether to enable cudnn benchmark cudnn_benchmark=False, # set multi process parameters mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0), # set distributed parameters dist_cfg=dict(backend='nccl'), ) # set visualizer visualizer = None # set log level log_level = 'INFO' # load from which checkpoint load_from = None # whether to resume training from the loaded checkpoint resume = False # Defaults to use random seed and disable `deterministic` randomness = dict(seed=None, deterministic=False)