| SYSTEM = '' |
| accumulative_counts = 1 |
| batch_size = 16 |
| betas = ( |
| 0.9, |
| 0.999, |
| ) |
| custom_hooks = [ |
| dict( |
| tokenizer=dict( |
| padding_side='right', |
| pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5', |
| trust_remote_code=True, |
| type='transformers.AutoTokenizer.from_pretrained'), |
| type='xtuner.engine.DatasetInfoHook'), |
| dict( |
| evaluation_images='https://llava-vl.github.io/static/images/view.jpg', |
| evaluation_inputs=[ |
| '请描述一下这张照片', |
| 'Please describe this picture', |
| ], |
| every_n_iters=500, |
| image_processor=dict( |
| pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', |
| trust_remote_code=True, |
| type='transformers.CLIPImageProcessor.from_pretrained'), |
| prompt_template='xtuner.utils.PROMPT_TEMPLATE.vicuna', |
| system='', |
| tokenizer=dict( |
| padding_side='right', |
| pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5', |
| trust_remote_code=True, |
| type='transformers.AutoTokenizer.from_pretrained'), |
| type='xtuner.engine.EvaluateChatHook'), |
| ] |
| data_path = './data/llava_data/LLaVA-Instruct-150K/llava_v1_5_mix665k.json' |
| dataloader_num_workers = 0 |
| default_hooks = dict( |
| checkpoint=dict(interval=1, type='mmengine.hooks.CheckpointHook'), |
| logger=dict(interval=10, type='mmengine.hooks.LoggerHook'), |
| param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'), |
| sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'), |
| timer=dict(type='mmengine.hooks.IterTimerHook')) |
| env_cfg = dict( |
| cudnn_benchmark=False, |
| dist_cfg=dict(backend='nccl'), |
| mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0)) |
| evaluation_freq = 500 |
| evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg' |
| evaluation_inputs = [ |
| '请描述一下这张照片', |
| 'Please describe this picture', |
| ] |
| image_folder = './data/llava_data/llava_images' |
| launcher = 'pytorch' |
| llava_data_root = './data/llava_data/' |
| llava_dataset = dict( |
| data_path='./data/llava_data/LLaVA-Instruct-150K/llava_v1_5_mix665k.json', |
| dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn', |
| image_folder='./data/llava_data/llava_images', |
| max_length=1472, |
| pad_image_to_square=True, |
| image_processor=dict( |
| pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', |
| trust_remote_code=True, |
| type='transformers.CLIPImageProcessor.from_pretrained'), |
| template_map_fn=dict( |
| template='xtuner.utils.PROMPT_TEMPLATE.vicuna', |
| type='xtuner.dataset.map_fns.template_map_fn_factory'), |
| tokenizer=dict( |
| padding_side='right', |
| pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5', |
| trust_remote_code=True, |
| type='transformers.AutoTokenizer.from_pretrained'), |
| type='xtuner.dataset.LLaVADataset') |
| llm_name_or_path = 'lmsys/vicuna-7b-v1.5' |
| load_from = None |
| log_level = 'INFO' |
| lr = 0.0002 |
| max_epochs = 1 |
| max_length = 1472 |
| max_norm = 1 |
| model = dict( |
| freeze_llm=True, |
| freeze_visual_encoder=True, |
| llm=dict( |
| pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5', |
| quantization_config=dict( |
| bnb_4bit_compute_dtype='torch.float16', |
| bnb_4bit_quant_type='nf4', |
| bnb_4bit_use_double_quant=True, |
| llm_int8_has_fp16_weight=False, |
| llm_int8_threshold=6.0, |
| load_in_4bit=True, |
| load_in_8bit=False, |
| type='transformers.BitsAndBytesConfig'), |
| torch_dtype='torch.float16', |
| trust_remote_code=True, |
| type='transformers.AutoModelForCausalLM.from_pretrained'), |
| llm_lora=dict( |
| bias='none', |
| lora_alpha=256, |
| lora_dropout=0.05, |
| r=512, |
| task_type='CAUSAL_LM', |
| type='peft.LoraConfig'), |
| pretrained_pth= |
| './work_dirs/llava_vicuna_7b_v15_clip_vit_large_p14_336_e1_gpu8_pretrain/epoch_1.pth', |
| type='xtuner.model.LLaVAModel', |
| visual_encoder=dict( |
| pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', |
| type='transformers.CLIPVisionModel.from_pretrained'), |
| visual_encoder_lora=dict( |
| bias='none', |
| lora_alpha=16, |
| lora_dropout=0.05, |
| r=64, |
| type='peft.LoraConfig')) |
| optim_type = 'torch.optim.AdamW' |
| optim_wrapper = dict( |
| optimizer=dict( |
| betas=( |
| 0.9, |
| 0.999, |
| ), |
| lr=0.0002, |
| type='torch.optim.AdamW', |
| weight_decay=0), |
| type='DeepSpeedOptimWrapper') |
| param_scheduler = [ |
| dict( |
| begin=0, |
| by_epoch=True, |
| convert_to_iter_based=True, |
| end=0.03, |
| start_factor=1e-05, |
| type='mmengine.optim.LinearLR'), |
| dict( |
| T_max=1, |
| begin=0.03, |
| by_epoch=True, |
| convert_to_iter_based=True, |
| eta_min=0.0, |
| type='mmengine.optim.CosineAnnealingLR'), |
| ] |
| pretrained_pth = './work_dirs/llava_vicuna_7b_v15_clip_vit_large_p14_336_e1_gpu8_pretrain/epoch_1.pth' |
| image_processor = dict( |
| pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', |
| trust_remote_code=True, |
| type='transformers.CLIPImageProcessor.from_pretrained') |
| prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.vicuna' |
| randomness = dict(deterministic=False, seed=None) |
| resume = False |
| runner_type = 'FlexibleRunner' |
| strategy = dict( |
| config=dict( |
| bf16=dict(enabled=True), |
| fp16=dict(enabled=False, initial_scale_power=16), |
| gradient_accumulation_steps='auto', |
| gradient_clipping='auto', |
| train_micro_batch_size_per_gpu='auto', |
| zero_allow_untested_optimizer=True, |
| zero_force_ds_cpu_optimizer=False, |
| zero_optimization=dict(overlap_comm=True, stage=2)), |
| exclude_frozen_parameters=True, |
| gradient_accumulation_steps=1, |
| gradient_clipping=1, |
| train_micro_batch_size_per_gpu=16, |
| type='xtuner.engine.DeepSpeedStrategy') |
| tokenizer = dict( |
| padding_side='right', |
| pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5', |
| trust_remote_code=True, |
| type='transformers.AutoTokenizer.from_pretrained') |
| train_cfg = dict(by_epoch=True, max_epochs=1, val_interval=1) |
| train_dataloader = dict( |
| batch_size=16, |
| collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'), |
| dataset=dict( |
| data_path= |
| './data/llava_data/LLaVA-Instruct-150K/llava_v1_5_mix665k.json', |
| dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn', |
| image_folder='./data/llava_data/llava_images', |
| max_length=1472, |
| pad_image_to_square=True, |
| image_processor=dict( |
| pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', |
| trust_remote_code=True, |
| type='transformers.CLIPImageProcessor.from_pretrained'), |
| template_map_fn=dict( |
| template='xtuner.utils.PROMPT_TEMPLATE.vicuna', |
| type='xtuner.dataset.map_fns.template_map_fn_factory'), |
| tokenizer=dict( |
| padding_side='right', |
| pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5', |
| trust_remote_code=True, |
| type='transformers.AutoTokenizer.from_pretrained'), |
| type='xtuner.dataset.LLaVADataset'), |
| num_workers=0, |
| sampler=dict( |
| length_property='modality_length', |
| per_device_batch_size=16, |
| type='xtuner.dataset.samplers.LengthGroupedSampler')) |
| visual_encoder_name_or_path = 'openai/clip-vit-large-patch14-336' |
| visualizer = None |
| warmup_ratio = 0.03 |
| weight_decay = 0 |
| work_dir = './work_dirs/llava_vicuna_7b_v15_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune' |
|
|