SYSTEM = '' accumulative_counts = 1 batch_size = 32 betas = ( 0.9, 0.999, ) custom_hooks = [ dict( tokenizer=dict( padding_side='right', pretrained_model_name_or_path='internlm/internlm-chat-7b', trust_remote_code=True, type='transformers.AutoTokenizer.from_pretrained'), type='xtuner.engine.DatasetInfoHook'), dict( evaluation_images='https://llava-vl.github.io/static/images/view.jpg', evaluation_inputs=[ '', ], every_n_iters=500, image_processor=dict( pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', trust_remote_code=True, type='transformers.CLIPImageProcessor.from_pretrained'), prompt_template='xtuner.utils.PROMPT_TEMPLATE.internlm_chat', system='', tokenizer=dict( padding_side='right', pretrained_model_name_or_path='internlm/internlm-chat-7b', trust_remote_code=True, type='transformers.AutoTokenizer.from_pretrained'), type='xtuner.engine.EvaluateChatHook'), ] data_path = './data/llava_data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json' dataloader_num_workers = 0 default_hooks = dict( checkpoint=dict(interval=1, type='mmengine.hooks.CheckpointHook'), logger=dict(interval=10, type='mmengine.hooks.LoggerHook'), param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'), sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'), timer=dict(type='mmengine.hooks.IterTimerHook')) env_cfg = dict( cudnn_benchmark=False, dist_cfg=dict(backend='nccl'), mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0)) evaluation_freq = 500 evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg' evaluation_inputs = [ '', ] image_folder = './data/llava_data/LLaVA-Pretrain/images' launcher = 'pytorch' llava_data_root = './data/llava_data/' llava_dataset = dict( data_path='./data/llava_data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json', dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn', image_folder='./data/llava_data/LLaVA-Pretrain/images', max_length=1472, pad_image_to_square=False, image_processor=dict( pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', trust_remote_code=True, type='transformers.CLIPImageProcessor.from_pretrained'), template_map_fn=dict( template='xtuner.utils.PROMPT_TEMPLATE.internlm_chat', type='xtuner.dataset.map_fns.template_map_fn_factory'), tokenizer=dict( padding_side='right', pretrained_model_name_or_path='internlm/internlm-chat-7b', trust_remote_code=True, type='transformers.AutoTokenizer.from_pretrained'), type='xtuner.dataset.LLaVADataset') llm_name_or_path = 'internlm/internlm-chat-7b' load_from = None log_level = 'INFO' lr = 0.001 max_epochs = 1 max_length = 1472 max_norm = 1 model = dict( freeze_llm=True, freeze_visual_encoder=True, llm=dict( pretrained_model_name_or_path='internlm/internlm-chat-7b', quantization_config=dict( bnb_4bit_compute_dtype='torch.float16', bnb_4bit_quant_type='nf4', bnb_4bit_use_double_quant=True, llm_int8_has_fp16_weight=False, llm_int8_threshold=6.0, load_in_4bit=True, load_in_8bit=False, type='transformers.BitsAndBytesConfig'), torch_dtype='torch.float16', trust_remote_code=True, type='transformers.AutoModelForCausalLM.from_pretrained'), type='xtuner.model.LLaVAModel', visual_encoder=dict( pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', type='transformers.CLIPVisionModel.from_pretrained')) optim_type = 'torch.optim.AdamW' optim_wrapper = dict( optimizer=dict( betas=( 0.9, 0.999, ), lr=0.001, type='torch.optim.AdamW', weight_decay=0), type='DeepSpeedOptimWrapper') param_scheduler = [ dict( begin=0, by_epoch=True, convert_to_iter_based=True, end=0.03, start_factor=1e-05, type='mmengine.optim.LinearLR'), dict( T_max=1, begin=0.03, by_epoch=True, convert_to_iter_based=True, eta_min=0.0, type='mmengine.optim.CosineAnnealingLR'), ] image_processor = dict( pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', trust_remote_code=True, type='transformers.CLIPImageProcessor.from_pretrained') prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.internlm_chat' randomness = dict(deterministic=False, seed=None) resume = False runner_type = 'FlexibleRunner' strategy = dict( config=dict( bf16=dict(enabled=True), fp16=dict(enabled=False, initial_scale_power=16), gradient_accumulation_steps='auto', gradient_clipping='auto', train_micro_batch_size_per_gpu='auto', zero_allow_untested_optimizer=True, zero_force_ds_cpu_optimizer=False, zero_optimization=dict(overlap_comm=True, stage=2)), exclude_frozen_parameters=True, gradient_accumulation_steps=1, gradient_clipping=1, train_micro_batch_size_per_gpu=32, type='xtuner.engine.DeepSpeedStrategy') tokenizer = dict( padding_side='right', pretrained_model_name_or_path='internlm/internlm-chat-7b', trust_remote_code=True, type='transformers.AutoTokenizer.from_pretrained') train_cfg = dict(by_epoch=True, max_epochs=1, val_interval=1) train_dataloader = dict( batch_size=32, collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'), dataset=dict( data_path= './data/llava_data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json', dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn', image_folder='./data/llava_data/LLaVA-Pretrain/images', max_length=1472, pad_image_to_square=False, image_processor=dict( pretrained_model_name_or_path='openai/clip-vit-large-patch14-336', trust_remote_code=True, type='transformers.CLIPImageProcessor.from_pretrained'), template_map_fn=dict( template='xtuner.utils.PROMPT_TEMPLATE.internlm_chat', type='xtuner.dataset.map_fns.template_map_fn_factory'), tokenizer=dict( padding_side='right', pretrained_model_name_or_path='internlm/internlm-chat-7b', trust_remote_code=True, type='transformers.AutoTokenizer.from_pretrained'), type='xtuner.dataset.LLaVADataset'), num_workers=0, sampler=dict(shuffle=True, type='mmengine.dataset.DefaultSampler')) visual_encoder_name_or_path = 'openai/clip-vit-large-patch14-336' visualizer = None warmup_ratio = 0.03 weight_decay = 0 work_dir = './work_dirs/llava_internlm_chat_7b_clip_vit_large_p14_336_e1_gpu8_pretrain'