Hugging Face's logo

xtuner
/
llava-v1.5-7b-xtuner-pretrain

Visual Question Answering
Transformers
Inference Endpoints
Model card Files Community
llava-v1.5-7b-xtuner-pretrain
File size: 6,896 Bytes 
d86310d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
 
SYSTEM = ''
accumulative_counts = 1
batch_size = 32
betas = (
    0.9,
    0.999,
)
custom_hooks = [
    dict(
        tokenizer=dict(
            padding_side='right',
            pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
            trust_remote_code=True,
            type='transformers.AutoTokenizer.from_pretrained'),
        type='xtuner.engine.DatasetInfoHook'),
    dict(
        evaluation_images='https://llava-vl.github.io/static/images/view.jpg',
        evaluation_inputs=[
            '',
        ],
        every_n_iters=500,
        processor=dict(
            pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
            trust_remote_code=True,
            type='transformers.CLIPImageProcessor.from_pretrained'),
        prompt_template='xtuner.utils.PROMPT_TEMPLATE.vicuna',
        system='',
        tokenizer=dict(
            padding_side='right',
            pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
            trust_remote_code=True,
            type='transformers.AutoTokenizer.from_pretrained'),
        type='xtuner.engine.EvaluateChatHook'),
]
data_path = './data/llava_data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json'
dataloader_num_workers = 0
default_hooks = dict(
    checkpoint=dict(interval=1, type='mmengine.hooks.CheckpointHook'),
    logger=dict(interval=10, type='mmengine.hooks.LoggerHook'),
    param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
    sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
    timer=dict(type='mmengine.hooks.IterTimerHook'))
env_cfg = dict(
    cudnn_benchmark=False,
    dist_cfg=dict(backend='nccl'),
    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
evaluation_freq = 500
evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg'
evaluation_inputs = [
    '',
]
image_folder = './data/llava_data/LLaVA-Pretrain/images'
launcher = 'pytorch'
llava_data_root = './data/llava_data/'
llava_dataset = dict(
    data_path='./data/llava_data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json',
    dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
    image_folder='./data/llava_data/LLaVA-Pretrain/images',
    max_length=1472,
    pad_image_to_square=False,
    processor=dict(
        pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
        trust_remote_code=True,
        type='transformers.CLIPImageProcessor.from_pretrained'),
    template_map_fn=dict(
        template='xtuner.utils.PROMPT_TEMPLATE.vicuna',
        type='xtuner.dataset.map_fns.template_map_fn_factory'),
    tokenizer=dict(
        padding_side='right',
        pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
        trust_remote_code=True,
        type='transformers.AutoTokenizer.from_pretrained'),
    type='xtuner.dataset.LLaVADataset')
llm_name_or_path = 'lmsys/vicuna-7b-v1.5'
load_from = None
log_level = 'INFO'
lr = 0.001
max_epochs = 1
max_length = 1472
max_norm = 1
model = dict(
    freeze_llm=True,
    freeze_visual_encoder=True,
    llm=dict(
        pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
        quantization_config=dict(
            bnb_4bit_compute_dtype='torch.float16',
            bnb_4bit_quant_type='nf4',
            bnb_4bit_use_double_quant=True,
            llm_int8_has_fp16_weight=False,
            llm_int8_threshold=6.0,
            load_in_4bit=True,
            load_in_8bit=False,
            type='transformers.BitsAndBytesConfig'),
        torch_dtype='torch.float16',
        trust_remote_code=True,
        type='transformers.AutoModelForCausalLM.from_pretrained'),
    type='xtuner.model.LLaVAModel',
    visual_encoder=dict(
        pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
        type='transformers.CLIPVisionModel.from_pretrained'))
optim_type = 'torch.optim.AdamW'
optim_wrapper = dict(
    optimizer=dict(
        betas=(
            0.9,
            0.999,
        ),
        lr=0.001,
        type='torch.optim.AdamW',
        weight_decay=0),
    type='DeepSpeedOptimWrapper')
param_scheduler = [
    dict(
        begin=0,
        by_epoch=True,
        convert_to_iter_based=True,
        end=0.03,
        start_factor=1e-05,
        type='mmengine.optim.LinearLR'),
    dict(
        T_max=1,
        begin=0.03,
        by_epoch=True,
        convert_to_iter_based=True,
        eta_min=0.0,
        type='mmengine.optim.CosineAnnealingLR'),
]
processor = dict(
    pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
    trust_remote_code=True,
    type='transformers.CLIPImageProcessor.from_pretrained')
prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.vicuna'
randomness = dict(deterministic=False, seed=None)
resume = False
runner_type = 'FlexibleRunner'
strategy = dict(
    config=dict(
        bf16=dict(enabled=True),
        fp16=dict(enabled=False, initial_scale_power=16),
        gradient_accumulation_steps='auto',
        gradient_clipping='auto',
        train_micro_batch_size_per_gpu='auto',
        zero_allow_untested_optimizer=True,
        zero_force_ds_cpu_optimizer=False,
        zero_optimization=dict(overlap_comm=True, stage=2)),
    exclude_frozen_parameters=True,
    gradient_accumulation_steps=1,
    gradient_clipping=1,
    train_micro_batch_size_per_gpu=32,
    type='xtuner.engine.DeepSpeedStrategy')
tokenizer = dict(
    padding_side='right',
    pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
    trust_remote_code=True,
    type='transformers.AutoTokenizer.from_pretrained')
train_cfg = dict(by_epoch=True, max_epochs=1, val_interval=1)
train_dataloader = dict(
    batch_size=32,
    collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'),
    dataset=dict(
        data_path=
        './data/llava_data/LLaVA-Pretrain/blip_laion_cc_sbu_558k.json',
        dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
        image_folder='./data/llava_data/LLaVA-Pretrain/images',
        max_length=1472,
        pad_image_to_square=False,
        processor=dict(
            pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
            trust_remote_code=True,
            type='transformers.CLIPImageProcessor.from_pretrained'),
        template_map_fn=dict(
            template='xtuner.utils.PROMPT_TEMPLATE.vicuna',
            type='xtuner.dataset.map_fns.template_map_fn_factory'),
        tokenizer=dict(
            padding_side='right',
            pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
            trust_remote_code=True,
            type='transformers.AutoTokenizer.from_pretrained'),
        type='xtuner.dataset.LLaVADataset'),
    num_workers=0,
    sampler=dict(shuffle=True, type='mmengine.dataset.DefaultSampler'))
visual_encoder_name_or_path = 'openai/clip-vit-large-patch14-336'
visualizer = None
warmup_ratio = 0.03
weight_decay = 0
work_dir = './work_dirs/llava_vicuna_7b_v15_clip_vit_large_p14_336_e1_gpu8_pretrain'