Upload LlavaT5ForConditionalGeneration

chartinstruct_flant5_modeling.py

@@ -8,592 +8,14 @@ from transformers import AutoConfig, AutoModelForSeq2SeqLM, \
                          T5Config, T5Model, T5ForConditionalGeneration
 
 from transformers.models.t5.modeling_t5 import T5Stack
-from transformers.modeling_outputs import CausalLMOutputWithPast, Seq2SeqLMOutput, BaseModelOutput
+from transformers.modeling_outputs import CausalLMOutputWithPast, Seq2SeqLMOutput, BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions
 from transformers.utils import ModelOutput
 from transformers import DonutSwinModel, DonutImageProcessor, DonutSwinConfig
-
 from abc import ABC, abstractmethod
 import re
 
-# Model Constants
-IGNORE_INDEX = -100
-IMAGE_TOKEN_INDEX = -200
-DEFAULT_IMAGE_TOKEN = "<image>"
-DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
-DEFAULT_IM_START_TOKEN = "<im_start>"
-DEFAULT_IM_END_TOKEN = "<im_end>"
-
-class UniChartVisionTower(nn.Module):
-    def __init__(self, vision_tower, args, delay_load=False):
-        super().__init__()
-
-        self.is_loaded = False
-
-        self.vision_tower_name = vision_tower
-        self.select_layer = args.mm_vision_select_layer
-        self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch')
-
-        if not delay_load:
-            self.load_model()
-        else:
-            self.cfg_only = DonutSwinConfig.from_pretrained(self.vision_tower_name)
-
-    def load_model(self):
-        self.image_processor = DonutImageProcessor.from_pretrained(self.vision_tower_name)
-        self.vision_tower = DonutSwinModel.from_pretrained(self.vision_tower_name)
-
-        # Changed. Check for this variable. It's false by default.
-        if not self.tune_vision_encoder:
-            self.vision_tower.requires_grad_(False)
-
-        self.is_loaded = True
-
-    def feature_select(self, image_forward_outs):
-        image_features = image_forward_outs.hidden_states[self.select_layer]
-        if self.select_feature == 'patch':
-            image_features = image_features[:, 1:]
-        elif self.select_feature == 'cls_patch':
-            image_features = image_features
-        else:
-            raise ValueError(f'Unexpected select feature: {self.select_feature}')
-        return image_features
-
-    @torch.no_grad()
-    def forward(self, images):
-        if type(images) is list:
-            image_features = []
-            for image in images:
-                image_forward_out = self.vision_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True)
-                image_feature = self.feature_select(image_forward_out).to(image.dtype)
-                image_features.append(image_feature)
-        else:
-            image_forward_outs = self.vision_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True)
-            image_features = self.feature_select(image_forward_outs).to(images.dtype)
-
-        return image_features
-
-    @property
-    def dummy_feature(self):
-        return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
-
-    @property
-    def dtype(self):
-        return self.vision_tower.dtype
-
-    @property
-    def device(self):
-        return self.vision_tower.device
-
-    @property
-    def config(self):
-        if self.is_loaded:
-            return self.vision_tower.config
-        else:
-            return self.cfg_only
-
-    @property
-    def hidden_size(self):
-        return self.config.hidden_size
-
-    @property
-    def num_patches(self):
-        return (self.config.image_size // self.config.patch_size) ** 2
-
-def build_vision_tower(vision_tower_cfg, **kwargs):
-    vision_tower = getattr(vision_tower_cfg, 'mm_vision_tower', getattr(vision_tower_cfg, 'vision_tower', None))
-    is_absolute_path_exists = os.path.exists(vision_tower)
-    if is_absolute_path_exists:
-        if 'unichart' in vision_tower:
-            return UniChartVisionTower(vision_tower, args=vision_tower_cfg, **kwargs)
-
-    raise ValueError(f'Unknown vision tower: {vision_tower}')
-
-def build_vision_projector(config, delay_load=False, **kwargs):
-    projector_type = getattr(config, 'mm_projector_type', 'mlp3x_gelu')
-
-    if projector_type == 'linear':
-        return nn.Linear(config.mm_hidden_size, config.hidden_size)
-
-    mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type)
-    if mlp_gelu_match:
-        mlp_depth = int(mlp_gelu_match.group(1))
-        modules = [nn.Linear(config.mm_hidden_size, config.hidden_size)]
-        for _ in range(1, mlp_depth):
-            modules.append(nn.GELU())
-            modules.append(nn.Linear(config.hidden_size, config.hidden_size))
-        return nn.Sequential(*modules)
-
-    raise ValueError(f'Unknown projector type: {projector_type}')
-
-#    Copyright 2023 Haotian Liu
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
-
-class LlavaMetaModel:
-
-    def __init__(self, config): #, embed_tokens):
-        super(LlavaMetaModel, self).__init__(config) #, embed_tokens)
-        if hasattr(config, "mm_vision_tower"):
-            self.vision_tower = build_vision_tower(config, delay_load=True)
-            self.mm_projector = build_vision_projector(self.config) #nn.Linear(config.mm_hidden_size, config.hidden_size)
-
-    def get_vision_tower(self):
-        vision_tower = getattr(self, 'vision_tower', None)
-        if type(vision_tower) is list:
-            vision_tower = vision_tower[0]
-        return vision_tower
-
-    def initialize_vision_modules(self, model_args, fsdp=None):
-        vision_tower = model_args.vision_tower
-        mm_vision_select_layer = model_args.mm_vision_select_layer
-        mm_vision_select_feature = model_args.mm_vision_select_feature
-        pretrain_mm_mlp_adapter = model_args.pretrain_mm_mlp_adapter
-
-        self.config.mm_vision_tower = vision_tower
-
-        vision_tower = build_vision_tower(model_args)
-
-        if fsdp is not None and len(fsdp) > 0:
-            self.vision_tower = [vision_tower]
-        else:
-            self.vision_tower = vision_tower
-
-        self.config.use_mm_proj = True
-        self.config.mm_projector_type = getattr(model_args, 'mm_projector_type', 'linear')
-        self.config.mm_hidden_size = vision_tower.hidden_size
-        self.config.mm_vision_select_layer = mm_vision_select_layer
-        self.config.mm_vision_select_feature = mm_vision_select_feature
-
-        if not hasattr(self, 'mm_projector'):
-            self.mm_projector = build_vision_projector(self.config) #nn.Linear(self.config.mm_hidden_size, self.config.hidden_size)
-
-        if pretrain_mm_mlp_adapter is not None:
-            mm_projector_weights = torch.load(pretrain_mm_mlp_adapter, map_location='cpu')
-            def get_w(weights, keyword):
-                return {k.split(keyword + '.')[1]: v for k, v in weights.items() if keyword in k}
-
-            self.mm_projector.load_state_dict(get_w(mm_projector_weights, 'mm_projector'))
-
-
-class LlavaMetaForCausalLM(ABC):
-
-    @abstractmethod
-    def get_model(self):
-        pass
-
-    def get_vision_tower(self):
-        return self.get_model().get_vision_tower()
-
-    def encode_images(self, images):
-        image_features = self.get_model().get_vision_tower()(images)
-        image_features = self.get_model().mm_projector(image_features)
-        return image_features
-
-    def prepare_inputs_labels_for_multimodal(
-        self, input_ids, attention_mask, past_key_values, labels, images
-    ):
-        vision_tower = self.get_vision_tower()
-        if vision_tower is None or images is None or input_ids.shape[1] == 1:
-            if past_key_values is not None and vision_tower is not None and images is not None and input_ids.shape[1] == 1:
-                attention_mask = torch.ones((attention_mask.shape[0], past_key_values[-1][-1].shape[-2] + 1), dtype=attention_mask.dtype, device=attention_mask.device)
-            return input_ids, attention_mask, past_key_values, None, labels
-
-        if type(images) is list or images.ndim == 5:
-            concat_images = torch.cat([image for image in images], dim=0)
-            image_features = self.encode_images(concat_images)
-            split_sizes = [image.shape[0] for image in images]
-            image_features = torch.split(image_features, split_sizes, dim=0)
-            image_features = [x.flatten(0, 1) for x in image_features]
-        else:
-            image_features = self.encode_images(images)
-
-        new_input_embeds = []
-        new_labels = [] if labels is not None else None
-        cur_image_idx = 0
-        for batch_idx, cur_input_ids in enumerate(input_ids):
-            if (cur_input_ids == IMAGE_TOKEN_INDEX).sum() == 0:
-                # multimodal LLM, but the current sample is not multimodal
-                cur_input_embeds = self.get_model().embed_tokens(cur_input_ids)
-                cur_input_embeds = cur_input_embeds + (0. * self.get_model().mm_projector(vision_tower.dummy_feature)).sum()
-                new_input_embeds.append(cur_input_embeds)
-                if labels is not None:
-                    new_labels.append(labels[batch_idx])
-                cur_image_idx += 1
-                continue
-            image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
-            cur_new_input_embeds = []
-            if labels is not None:
-                cur_labels = labels[batch_idx]
-                cur_new_labels = []
-                assert cur_labels.shape == cur_input_ids.shape
-            while image_token_indices.numel() > 0:
-                cur_image_features = image_features[cur_image_idx]
-                image_token_start = image_token_indices[0]
-                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
-                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[:image_token_start-1]).detach())
-                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[image_token_start-1:image_token_start]))
-                    cur_new_input_embeds.append(cur_image_features)
-                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[image_token_start+1:image_token_start+2]))
-                    if labels is not None:
-                        cur_new_labels.append(cur_labels[:image_token_start])
-                        cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device, dtype=labels.dtype))
-                        cur_new_labels.append(cur_labels[image_token_start:image_token_start+1])
-                        cur_labels = cur_labels[image_token_start+2:]
-                else:
-                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[:image_token_start]))
-                    cur_new_input_embeds.append(cur_image_features)
-                    if labels is not None:
-                        cur_new_labels.append(cur_labels[:image_token_start])
-                        cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device, dtype=labels.dtype))
-                        cur_labels = cur_labels[image_token_start+1:]
-                cur_image_idx += 1
-                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
-                    cur_input_ids = cur_input_ids[image_token_start+2:]
-                else:
-                    cur_input_ids = cur_input_ids[image_token_start+1:]
-                image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
-            if cur_input_ids.numel() > 0:
-                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
-                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids).detach())
-                else:
-                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids))
-                if labels is not None:
-                    cur_new_labels.append(cur_labels)
-            cur_new_input_embeds = [x.to(device=self.device) for x in cur_new_input_embeds]
-            cur_new_input_embeds = torch.cat(cur_new_input_embeds, dim=0)
-            new_input_embeds.append(cur_new_input_embeds)
-            if labels is not None:
-                cur_new_labels = torch.cat(cur_new_labels, dim=0)
-                new_labels.append(cur_new_labels)
-
-        if any(x.shape != new_input_embeds[0].shape for x in new_input_embeds):
-            max_len = max(x.shape[0] for x in new_input_embeds)
-
-            new_input_embeds_align = []
-            for cur_new_embed in new_input_embeds:
-                cur_new_embed = torch.cat((cur_new_embed, torch.zeros((max_len - cur_new_embed.shape[0], cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device)), dim=0)
-                new_input_embeds_align.append(cur_new_embed)
-            new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
-
-            if labels is not None:
-                new_labels_align = []
-                _new_labels = new_labels
-                for cur_new_label in new_labels:
-                    cur_new_label = torch.cat((cur_new_label, torch.full((max_len - cur_new_label.shape[0],), IGNORE_INDEX, dtype=cur_new_label.dtype, device=cur_new_label.device)), dim=0)
-                    new_labels_align.append(cur_new_label)
-                new_labels = torch.stack(new_labels_align, dim=0)
-
-            if attention_mask is not None:
-                new_attention_mask = []
-                for cur_attention_mask, cur_new_labels, cur_new_labels_align in zip(attention_mask, _new_labels, new_labels):
-                    new_attn_mask_pad_left = torch.full((cur_new_labels.shape[0] - labels.shape[1],), True, dtype=attention_mask.dtype, device=attention_mask.device)
-                    new_attn_mask_pad_right = torch.full((cur_new_labels_align.shape[0] - cur_new_labels.shape[0],), False, dtype=attention_mask.dtype, device=attention_mask.device)
-                    cur_new_attention_mask = torch.cat((new_attn_mask_pad_left, cur_attention_mask, new_attn_mask_pad_right), dim=0)
-                    new_attention_mask.append(cur_new_attention_mask)
-                attention_mask = torch.stack(new_attention_mask, dim=0)
-                assert attention_mask.shape == new_labels.shape
-        else:
-            new_input_embeds = torch.stack(new_input_embeds, dim=0)
-            if labels is not None:
-                new_labels  = torch.stack(new_labels, dim=0)
-
-            if attention_mask is not None:
-                new_attn_mask_pad_left = torch.full((attention_mask.shape[0], new_input_embeds.shape[1] - input_ids.shape[1]), True, dtype=attention_mask.dtype, device=attention_mask.device)
-                attention_mask = torch.cat((new_attn_mask_pad_left, attention_mask), dim=1)
-                assert attention_mask.shape == new_input_embeds.shape[:2]
-
-        return None, attention_mask, past_key_values, new_input_embeds, new_labels
-
-    def initialize_vision_tokenizer(self, model_args, tokenizer):
-        if model_args.mm_use_im_patch_token:
-            tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
-            self.resize_token_embeddings(len(tokenizer))
-
-        if model_args.mm_use_im_start_end:
-            num_new_tokens = tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
-            self.resize_token_embeddings(len(tokenizer))
-
-            if num_new_tokens > 0:
-                input_embeddings = self.get_input_embeddings().weight.data
-                output_embeddings = self.get_output_embeddings().weight.data
-
-                input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
-                    dim=0, keepdim=True)
-                output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
-                    dim=0, keepdim=True)
-
-                input_embeddings[-num_new_tokens:] = input_embeddings_avg
-                output_embeddings[-num_new_tokens:] = output_embeddings_avg
-
-            if model_args.tune_mm_mlp_adapter:
-                for p in self.get_input_embeddings().parameters():
-                    p.requires_grad = True
-                for p in self.get_output_embeddings().parameters():
-                    p.requires_grad = False
-
-            if model_args.pretrain_mm_mlp_adapter:
-                mm_projector_weights = torch.load(model_args.pretrain_mm_mlp_adapter, map_location='cpu')
-                embed_tokens_weight = mm_projector_weights['model.embed_tokens.weight']
-                assert num_new_tokens == 2
-                if input_embeddings.shape == embed_tokens_weight.shape:
-                    input_embeddings[-num_new_tokens:] = embed_tokens_weight[-num_new_tokens:]
-                elif embed_tokens_weight.shape[0] == num_new_tokens:
-                    input_embeddings[-num_new_tokens:] = embed_tokens_weight
-                else:
-                    raise ValueError(f"Unexpected embed_tokens_weight shape. Pretrained: {embed_tokens_weight.shape}. Current: {input_embeddings.shape}. Numer of new tokens: {num_new_tokens}.")
-        elif model_args.mm_use_im_patch_token:
-            if model_args.tune_mm_mlp_adapter:
-                for p in self.get_input_embeddings().parameters():
-                    p.requires_grad = False
-                for p in self.get_output_embeddings().parameters():
-                    p.requires_grad = False
-
-
-
-class LlavaMetaForConditionalGeneration(ABC):
-
-    def get_vision_tower(self):
-        return self.get_encoder().get_vision_tower()
-
-    def encode_images(self, images):
-        image_features = self.get_encoder().get_vision_tower()(images)
-        image_features = self.get_encoder().mm_projector(image_features)
-        return image_features
-
-    def prepare_inputs_labels_for_multimodal(
-        self, input_ids, attention_mask, labels, images
-    ):
-        vision_tower = self.get_vision_tower()
-        if vision_tower is None or images is None or input_ids.shape[1] == 1:
-            return input_ids, attention_mask, None
-
-        if type(images) is list or images.ndim == 5:
-            concat_images = torch.cat([image for image in images], dim=0)
-            image_features = self.encode_images(concat_images)
-            split_sizes = [image.shape[0] for image in images]
-            image_features = torch.split(image_features, split_sizes, dim=0)
-            image_features = [x.flatten(0, 1) for x in image_features]
-        else:
-            image_features = self.encode_images(images)
-
-        # TODO: double check.
-        if labels is None:
-            labels = torch.full_like(input_ids, IGNORE_INDEX)
-        ######
-
-        new_input_embeds = []
-        new_labels = [] if labels is not None else None
-        cur_image_idx = 0
-        for batch_idx, cur_input_ids in enumerate(input_ids):
-            if (cur_input_ids == IMAGE_TOKEN_INDEX).sum() == 0:
-                # multimodal LLM, but the current sample is not multimodal
-                cur_input_embeds = self.get_encoder().embed_tokens(cur_input_ids)
-                cur_input_embeds = cur_input_embeds + (0. * self.get_encoder().mm_projector(vision_tower.dummy_feature)).sum()
-                new_input_embeds.append(cur_input_embeds)
-                if labels is not None:
-                    new_labels.append(labels[batch_idx])
-                cur_image_idx += 1
-                continue
-            image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
-            cur_new_input_embeds = []
-            if labels is not None:
-                cur_labels = labels[batch_idx]
-                cur_new_labels = []
-                assert cur_labels.shape == cur_input_ids.shape
-            while image_token_indices.numel() > 0:
-                cur_image_features = image_features[cur_image_idx]
-                image_token_start = image_token_indices[0]
-                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
-                    cur_new_input_embeds.append(self.get_encoder().embed_tokens(cur_input_ids[:image_token_start-1]).detach())
-                    cur_new_input_embeds.append(self.get_encoder().embed_tokens(cur_input_ids[image_token_start-1:image_token_start]))
-                    cur_new_input_embeds.append(cur_image_features)
-                    cur_new_input_embeds.append(self.get_encoder().embed_tokens(cur_input_ids[image_token_start+1:image_token_start+2]))
-                    if labels is not None:
-                        cur_new_labels.append(cur_labels[:image_token_start])
-                        cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device, dtype=labels.dtype))
-                        cur_new_labels.append(cur_labels[image_token_start:image_token_start+1])
-                        cur_labels = cur_labels[image_token_start+2:]
-                else:
-                    cur_new_input_embeds.append(self.get_encoder().embed_tokens(cur_input_ids[:image_token_start]))
-                    cur_new_input_embeds.append(cur_image_features)
-                    if labels is not None:
-                        cur_new_labels.append(cur_labels[:image_token_start])
-                        cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device, dtype=labels.dtype))
-                        cur_labels = cur_labels[image_token_start+1:]
-                cur_image_idx += 1
-                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
-                    cur_input_ids = cur_input_ids[image_token_start+2:]
-                else:
-                    cur_input_ids = cur_input_ids[image_token_start+1:]
-                image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
-            if cur_input_ids.numel() > 0:
-                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
-                    cur_new_input_embeds.append(self.get_encoder().embed_tokens(cur_input_ids).detach())
-                else:
-                    cur_new_input_embeds.append(self.get_encoder().embed_tokens(cur_input_ids))
-                if labels is not None:
-                    cur_new_labels.append(cur_labels)
-            cur_new_input_embeds = [x.to(device=self.device) for x in cur_new_input_embeds]
-            cur_new_input_embeds = torch.cat(cur_new_input_embeds, dim=0)
-            new_input_embeds.append(cur_new_input_embeds)
-            if labels is not None:
-                cur_new_labels = torch.cat(cur_new_labels, dim=0)
-                new_labels.append(cur_new_labels)
-
-        if any(x.shape != new_input_embeds[0].shape for x in new_input_embeds):
-            max_len = max(x.shape[0] for x in new_input_embeds)
-
-            new_input_embeds_align = []
-            for cur_new_embed in new_input_embeds:
-                cur_new_embed = torch.cat((cur_new_embed, torch.zeros((max_len - cur_new_embed.shape[0], cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device)), dim=0)
-                new_input_embeds_align.append(cur_new_embed)
-            new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
-
-            if labels is not None:
-                new_labels_align = []
-                _new_labels = new_labels
-                for cur_new_label in new_labels:
-                    cur_new_label = torch.cat((cur_new_label, torch.full((max_len - cur_new_label.shape[0],), IGNORE_INDEX, dtype=cur_new_label.dtype, device=cur_new_label.device)), dim=0)
-                    new_labels_align.append(cur_new_label)
-                new_labels = torch.stack(new_labels_align, dim=0)
-
-            if attention_mask is not None:
-                new_attention_mask = []
-                for cur_attention_mask, cur_new_labels, cur_new_labels_align in zip(attention_mask, _new_labels, new_labels):
-                    new_attn_mask_pad_left = torch.full((cur_new_labels.shape[0] - labels.shape[1],), True, dtype=attention_mask.dtype, device=attention_mask.device)
-                    new_attn_mask_pad_right = torch.full((cur_new_labels_align.shape[0] - cur_new_labels.shape[0],), False, dtype=attention_mask.dtype, device=attention_mask.device)
-                    cur_new_attention_mask = torch.cat((new_attn_mask_pad_left, cur_attention_mask, new_attn_mask_pad_right), dim=0)
-                    new_attention_mask.append(cur_new_attention_mask)
-                attention_mask = torch.stack(new_attention_mask, dim=0)
-                assert attention_mask.shape == new_labels.shape
-        else:
-            new_input_embeds = torch.stack(new_input_embeds, dim=0)
-            if labels is not None:
-                new_labels  = torch.stack(new_labels, dim=0)
-
-            if attention_mask is not None:
-                new_attn_mask_pad_left = torch.full((attention_mask.shape[0], new_input_embeds.shape[1] - input_ids.shape[1]), True, dtype=attention_mask.dtype, device=attention_mask.device)
-                attention_mask = torch.cat((new_attn_mask_pad_left, attention_mask), dim=1)
-                assert attention_mask.shape == new_input_embeds.shape[:2]
-
-        return None, attention_mask, new_input_embeds, new_labels
-
-    def initialize_vision_tokenizer(self, model_args, tokenizer):
-        if model_args.mm_use_im_patch_token:
-            tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
-            self.resize_token_embeddings(len(tokenizer))
-
-        if model_args.mm_use_im_start_end:
-            num_new_tokens = tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
-            self.resize_token_embeddings(len(tokenizer))
-
-            if num_new_tokens > 0:
-                input_embeddings = self.get_input_embeddings().weight.data
-                output_embeddings = self.get_output_embeddings().weight.data
-
-                input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
-                    dim=0, keepdim=True)
-                output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
-                    dim=0, keepdim=True)
-
-                input_embeddings[-num_new_tokens:] = input_embeddings_avg
-                output_embeddings[-num_new_tokens:] = output_embeddings_avg
-
-            if model_args.tune_mm_mlp_adapter:
-                for p in self.get_input_embeddings().parameters():
-                    p.requires_grad = True
-                for p in self.get_output_embeddings().parameters():
-                    p.requires_grad = False
-
-            if model_args.pretrain_mm_mlp_adapter:
-                mm_projector_weights = torch.load(model_args.pretrain_mm_mlp_adapter, map_location='cpu')
-                embed_tokens_weight = mm_projector_weights['model.embed_tokens.weight']
-                assert num_new_tokens == 2
-                if input_embeddings.shape == embed_tokens_weight.shape:
-                    input_embeddings[-num_new_tokens:] = embed_tokens_weight[-num_new_tokens:]
-                elif embed_tokens_weight.shape[0] == num_new_tokens:
-                    input_embeddings[-num_new_tokens:] = embed_tokens_weight
-                else:
-                    raise ValueError(f"Unexpected embed_tokens_weight shape. Pretrained: {embed_tokens_weight.shape}. Current: {input_embeddings.shape}. Numer of new tokens: {num_new_tokens}.")
-        elif model_args.mm_use_im_patch_token:
-            if model_args.tune_mm_mlp_adapter:
-                for p in self.get_input_embeddings().parameters():
-                    p.requires_grad = False
-                for p in self.get_output_embeddings().parameters():
-                    p.requires_grad = False
-
-class LlavaMetaT5Model:
-
-    def __init__(self, config, embed_tokens):
-        super(LlavaMetaT5Model, self).__init__(config, embed_tokens)
-        if hasattr(config, "mm_vision_tower"):
-            self.vision_tower = build_vision_tower(config, delay_load=True)
-            self.mm_projector = nn.Linear(config.mm_hidden_size, config.hidden_size)
-
-    def get_vision_tower(self):
-        vision_tower = getattr(self, 'vision_tower', None)
-        if type(vision_tower) is list:
-            vision_tower = vision_tower[0]
-        return vision_tower
-
-    def initialize_vision_modules(self, model_args, fsdp=None):
-        vision_tower = model_args.vision_tower
-        mm_vision_select_layer = model_args.mm_vision_select_layer
-        mm_vision_select_feature = model_args.mm_vision_select_feature
-        pretrain_mm_mlp_adapter = model_args.pretrain_mm_mlp_adapter
-
-        self.config.mm_vision_tower = vision_tower
-
-        vision_tower = build_vision_tower(model_args)
-
-        if fsdp is not None and len(fsdp) > 0:
-            self.vision_tower = [vision_tower]
-        else:
-            self.vision_tower = vision_tower
-
-        self.config.use_mm_proj = True
-        self.config.mm_projector_type = getattr(model_args, 'mm_projector_type', 'linear')
-        self.config.mm_hidden_size = vision_tower.hidden_size
-        self.config.mm_vision_select_layer = mm_vision_select_layer
-        self.config.mm_vision_select_feature = mm_vision_select_feature
-
-        if not hasattr(self, 'mm_projector'):
-            self.mm_projector = build_vision_projector(self.config) #nn.Linear(self.config.mm_hidden_size, self.config.hidden_size)
-
-        if pretrain_mm_mlp_adapter is not None:
-            mm_projector_weights = torch.load(pretrain_mm_mlp_adapter, map_location='cpu')
-            def get_w(weights, keyword):
-                return {k.split(keyword + '.')[1]: v for k, v in weights.items() if keyword in k}
-
-            self.mm_projector.load_state_dict(get_w(mm_projector_weights, 'mm_projector'))
-
-
-
-#    Copyright 2023 Haotian Liu
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
+from transformers import T5PreTrainedModel
+from transformers.models.t5.modeling_t5 import T5Block, T5LayerNorm
 
 
 @dataclass
@@ -645,14 +67,340 @@ class BaseModelOutputWithPastAndCrossAttentionsWithAttentionMask(ModelOutput):
 class LlavaT5Config(T5Config):
     model_type = "llava_t5"
 
-class LlavaT5Stack(LlavaMetaT5Model, T5Stack):
+
+
+class LlavaT5Stack(T5PreTrainedModel):
     config_class = LlavaT5Config
 
-    def __init__(self, config: T5Config, embed_tokens=None):
-        super(LlavaT5Stack, self).__init__(config, embed_tokens)
+    def __init__(self, config, embed_tokens=None):
+        super().__init__(config)
 
+        self.embed_tokens = embed_tokens
+        self.is_decoder = config.is_decoder
+
+        self.block = nn.ModuleList(
+            [T5Block(config, has_relative_attention_bias=bool(i == 0)) for i in range(config.num_layers)]
+        )
+        self.final_layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon)
+        self.dropout = nn.Dropout(config.dropout_rate)
+
+        ## Vision
+        self.vision_tower =  DonutSwinModel(config=config.vision_config)
+        self.mm_projector = nn.Linear(config.mm_hidden_size, config.hidden_size)
+        self.pad_token_id = 0
+        self.image_token_index = 32100
+        ##
+
+        # Initialize weights and apply final processing
+        self.post_init()
+        # Model parallel
+        self.model_parallel = False
+        self.device_map = None
+        self.gradient_checkpointing = False
+
+    def _merge_input_ids_with_image_features(self, image_features, inputs_embeds, input_ids, attention_mask):
+        num_images, num_image_patches, embed_dim = image_features.shape
+        batch_size, sequence_length = input_ids.shape
+        left_padding = not torch.sum(input_ids[:, -1] == torch.tensor(self.pad_token_id))
+        # 1. Create a mask to know where special image tokens are
+        special_image_token_mask = input_ids == self.image_token_index
+        num_special_image_tokens = torch.sum(special_image_token_mask, dim=-1)
+        # Compute the maximum embed dimension
+        max_embed_dim = (num_special_image_tokens.max() * (num_image_patches - 1)) + sequence_length
+        batch_indices, non_image_indices = torch.where(input_ids != self.image_token_index)
+
+        # 2. Compute the positions where text should be written
+        # Calculate new positions for text tokens in merged image-text sequence.
+        # `special_image_token_mask` identifies image tokens. Each image token will be replaced by `nb_text_tokens_per_images - 1` text tokens.
+        # `torch.cumsum` computes how each image token shifts subsequent text token positions.
+        # - 1 to adjust for zero-based indexing, as `cumsum` inherently increases indices by one.
+        new_token_positions = torch.cumsum((special_image_token_mask * (num_image_patches - 1) + 1), -1) - 1
+        nb_image_pad = max_embed_dim - 1 - new_token_positions[:, -1]
+        if left_padding:
+            new_token_positions += nb_image_pad[:, None]  # offset for left padding
+        text_to_overwrite = new_token_positions[batch_indices, non_image_indices]
+
+        # 3. Create the full embedding, already padded to the maximum position
+        final_embedding = torch.zeros(
+            batch_size, max_embed_dim, embed_dim, dtype=inputs_embeds.dtype, device=inputs_embeds.device
+        )
+        final_attention_mask = torch.zeros(
+            batch_size, max_embed_dim, dtype=attention_mask.dtype, device=inputs_embeds.device
+        )
 
-class LlavaT5ForConditionalGeneration(T5ForConditionalGeneration, LlavaMetaForConditionalGeneration):
+        # In case the Vision model or the Language model has been offloaded to CPU, we need to manually
+        # set the corresponding tensors into their correct target device.
+        target_device = inputs_embeds.device
+        batch_indices, non_image_indices, text_to_overwrite = (
+            batch_indices.to(target_device),
+            non_image_indices.to(target_device),
+            text_to_overwrite.to(target_device),
+        )
+        attention_mask = attention_mask.to(target_device)
+
+        # 4. Fill the embeddings based on the mask. If we have ["hey" "<image>", "how", "are"]
+        # we need to index copy on [0, 577, 578, 579] for the text and [1:576] for the image features
+        final_embedding[batch_indices, text_to_overwrite] = inputs_embeds[batch_indices, non_image_indices]
+        final_attention_mask[batch_indices, text_to_overwrite] = attention_mask[batch_indices, non_image_indices]
+
+        # 5. Fill the embeddings corresponding to the images. Anything that is not `text_positions` needs filling (#29835)
+        image_to_overwrite = torch.full(
+            (batch_size, max_embed_dim), True, dtype=torch.bool, device=inputs_embeds.device
+        )
+        image_to_overwrite[batch_indices, text_to_overwrite] = False
+        image_to_overwrite &= image_to_overwrite.cumsum(-1) - 1 >= nb_image_pad[:, None].to(target_device)
+
+        if image_to_overwrite.sum() != image_features.shape[:-1].numel():
+            raise ValueError(
+                f"The input provided to the model are wrong. The number of image tokens is {torch.sum(special_image_token_mask)} while"
+                f" the number of image given to the model is {num_images}. This prevents correct indexing and breaks batch generation."
+            )
+
+        final_embedding[image_to_overwrite] = image_features.contiguous().reshape(-1, embed_dim).to(target_device)
+        final_attention_mask |= image_to_overwrite
+
+        # 6. Mask out the embedding at padding positions, as we later use the past_key_value value to determine the non-attended tokens.
+        batch_indices, pad_indices = torch.where(input_ids == self.pad_token_id)
+        indices_to_mask = new_token_positions[batch_indices, pad_indices]
+
+        final_embedding[batch_indices, indices_to_mask] = 0
+
+        return final_embedding, final_attention_mask
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        pixel_values=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        inputs_embeds=None,
+        head_mask=None,
+        cross_attn_head_mask=None,
+        past_key_values=None,
+        use_cache=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        # Model parallel
+        if self.model_parallel:
+            torch.cuda.set_device(self.first_device)
+            self.embed_tokens = self.embed_tokens.to(self.first_device)
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if input_ids is not None and inputs_embeds is not None:
+            err_msg_prefix = "decoder_" if self.is_decoder else ""
+            raise ValueError(
+                f"You cannot specify both {err_msg_prefix}input_ids and {err_msg_prefix}inputs_embeds at the same time"
+            )
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            input_ids = input_ids.view(-1, input_shape[-1])
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+        else:
+            err_msg_prefix = "decoder_" if self.is_decoder else ""
+            raise ValueError(f"You have to specify either {err_msg_prefix}input_ids or {err_msg_prefix}inputs_embeds")
+
+        if inputs_embeds is None:
+            if self.embed_tokens is None:
+                raise ValueError("You have to initialize the model with valid token embeddings")
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        ### Multimodal 
+        vision_feature_layer = -1
+        vision_feature_select_strategy = "default"
+        image_outputs = self.vision_tower(pixel_values, output_hidden_states=True)
+        # this is not memory efficient at all (output_hidden_states=True) will save all the hidden stated.
+        selected_image_feature = image_outputs.hidden_states[vision_feature_layer]
+
+        if vision_feature_select_strategy == "default":
+            selected_image_feature = selected_image_feature[:, 1:]
+        elif vision_feature_select_strategy == "full":
+            selected_image_feature = selected_image_feature
+        else:
+            raise ValueError(
+                f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}"
+            )
+
+        image_features = self.mm_projector(selected_image_feature)
+        inputs_embeds = inputs_embeds.to(image_features.dtype)
+        inputs_embeds, attention_mask = self._merge_input_ids_with_image_features(
+            image_features, inputs_embeds, input_ids, attention_mask
+        )
+        input_shape = inputs_embeds.size()[:-1]
+        #################
+
+        batch_size, seq_length = input_shape
+
+        # required mask seq length can be calculated via length of past
+        mask_seq_length = past_key_values[0][0].shape[2] + seq_length if past_key_values is not None else seq_length
+
+        if use_cache is True:
+            if not self.is_decoder:
+                raise ValueError(f"`use_cache` can only be set to `True` if {self} is used as a decoder")
+
+        # initialize past_key_values with `None` if past does not exist
+        if past_key_values is None:
+            past_key_values = [None] * len(self.block)
+
+        if attention_mask is None:
+            attention_mask = torch.ones(batch_size, mask_seq_length, device=inputs_embeds.device)
+
+        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
+        # ourselves in which case we just need to make it broadcastable to all heads.
+        extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape)
+
+        # If a 2D or 3D attention mask is provided for the cross-attention
+        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
+        if self.is_decoder and encoder_hidden_states is not None:
+            encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
+            encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
+            if encoder_attention_mask is None:
+                encoder_attention_mask = torch.ones(
+                    encoder_hidden_shape, device=inputs_embeds.device, dtype=torch.long
+                )
+            encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask)
+        else:
+            encoder_extended_attention_mask = None
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                # logger.warning_once(
+                #     "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                # )
+                use_cache = False
+
+        # Prepare head mask if needed
+        head_mask = self.get_head_mask(head_mask, self.config.num_layers)
+        cross_attn_head_mask = self.get_head_mask(cross_attn_head_mask, self.config.num_layers)
+        present_key_value_states = () if use_cache else None
+        all_hidden_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+        all_cross_attentions = () if (output_attentions and self.is_decoder) else None
+        position_bias = None
+        encoder_decoder_position_bias = None
+
+        hidden_states = self.dropout(inputs_embeds)
+
+        for i, (layer_module, past_key_value) in enumerate(zip(self.block, past_key_values)):
+            layer_head_mask = head_mask[i]
+            cross_attn_layer_head_mask = cross_attn_head_mask[i]
+            # Model parallel
+            if self.model_parallel:
+                torch.cuda.set_device(hidden_states.device)
+                # Ensure that attention_mask is always on the same device as hidden_states
+                if attention_mask is not None:
+                    attention_mask = attention_mask.to(hidden_states.device)
+                if position_bias is not None:
+                    position_bias = position_bias.to(hidden_states.device)
+                if encoder_hidden_states is not None:
+                    encoder_hidden_states = encoder_hidden_states.to(hidden_states.device)
+                if encoder_extended_attention_mask is not None:
+                    encoder_extended_attention_mask = encoder_extended_attention_mask.to(hidden_states.device)
+                if encoder_decoder_position_bias is not None:
+                    encoder_decoder_position_bias = encoder_decoder_position_bias.to(hidden_states.device)
+                if layer_head_mask is not None:
+                    layer_head_mask = layer_head_mask.to(hidden_states.device)
+                if cross_attn_layer_head_mask is not None:
+                    cross_attn_layer_head_mask = cross_attn_layer_head_mask.to(hidden_states.device)
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    layer_module.forward,
+                    hidden_states,
+                    extended_attention_mask,
+                    position_bias,
+                    encoder_hidden_states,
+                    encoder_extended_attention_mask,
+                    encoder_decoder_position_bias,
+                    layer_head_mask,
+                    cross_attn_layer_head_mask,
+                    None,  # past_key_value is always None with gradient checkpointing
+                    use_cache,
+                    output_attentions,
+                )
+            else:
+                layer_outputs = layer_module(
+                    hidden_states,
+                    attention_mask=extended_attention_mask,
+                    position_bias=position_bias,
+                    encoder_hidden_states=encoder_hidden_states,
+                    encoder_attention_mask=encoder_extended_attention_mask,
+                    encoder_decoder_position_bias=encoder_decoder_position_bias,
+                    layer_head_mask=layer_head_mask,
+                    cross_attn_layer_head_mask=cross_attn_layer_head_mask,
+                    past_key_value=past_key_value,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                )
+
+            # layer_outputs is a tuple with:
+            # hidden-states, key-value-states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights)
+            if use_cache is False:
+                layer_outputs = layer_outputs[:1] + (None,) + layer_outputs[1:]
+
+            hidden_states, present_key_value_state = layer_outputs[:2]
+
+            # We share the position biases between the layers - the first layer store them
+            # layer_outputs = hidden-states, key-value-states (self-attention position bias), (self-attention weights),
+            # (cross-attention position bias), (cross-attention weights)
+            position_bias = layer_outputs[2]
+            if self.is_decoder and encoder_hidden_states is not None:
+                encoder_decoder_position_bias = layer_outputs[4 if output_attentions else 3]
+            # append next layer key value states
+            if use_cache:
+                present_key_value_states = present_key_value_states + (present_key_value_state,)
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[3],)
+                if self.is_decoder:
+                    all_cross_attentions = all_cross_attentions + (layer_outputs[5],)
+
+            # Model Parallel: If it's the last layer for that device, put things on the next device
+            if self.model_parallel:
+                for k, v in self.device_map.items():
+                    if i == v[-1] and "cuda:" + str(k) != self.last_device:
+                        hidden_states = hidden_states.to("cuda:" + str(k + 1))
+
+        hidden_states = self.final_layer_norm(hidden_states)
+        hidden_states = self.dropout(hidden_states)
+
+        # Add last layer
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(
+                v
+                for v in [
+                    hidden_states,
+                    present_key_value_states,
+                    all_hidden_states,
+                    all_attentions,
+                    all_cross_attentions,
+                ]
+                if v is not None
+            )
+        return BaseModelOutputWithPastAndCrossAttentionsWithAttentionMask(
+            last_hidden_state=hidden_states,
+            past_key_values=present_key_value_states,
+            hidden_states=all_hidden_states,
+            attentions=all_attentions,
+            cross_attentions=all_cross_attentions,
+            attention_mask=attention_mask,
+        )
+
+
+class LlavaT5ForConditionalGeneration(T5ForConditionalGeneration):
     config_class = LlavaT5Config
 
     def __init__(self, config):
@@ -700,7 +448,7 @@ class LlavaT5ForConditionalGeneration(T5ForConditionalGeneration, LlavaMetaForCo
         use_cache: Optional[bool] = None,
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
-        images: Optional[torch.FloatTensor] = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
         return_dict: Optional[bool] = None,
 
         decoder_input_ids: Optional[torch.LongTensor] = None,
@@ -723,17 +471,16 @@ class LlavaT5ForConditionalGeneration(T5ForConditionalGeneration, LlavaMetaForCo
                 #warnings.warn(__HEAD_MASK_WARNING_MSG, FutureWarning)
                 decoder_head_mask = head_mask
 
+        if encoder_outputs is not None:
+          attention_mask = encoder_outputs.attention_mask
+
         # Encode if needed (training, first prediction pass)
         if encoder_outputs is None:
-            input_ids, attention_mask, inputs_embeds, _ = self.prepare_inputs_labels_for_multimodal(input_ids,
-                                                                                            attention_mask,
-                                                                                            None,  # Important: keep it None
-                                                                                            images
-                                                                                            )
             # Convert encoder inputs in embeddings if needed
             encoder_outputs = self.encoder(
                 input_ids=input_ids,
                 attention_mask=attention_mask,
+                pixel_values=pixel_values,
                 inputs_embeds=inputs_embeds,
                 head_mask=head_mask,
                 output_attentions=output_attentions,
@@ -746,6 +493,7 @@ class LlavaT5ForConditionalGeneration(T5ForConditionalGeneration, LlavaMetaForCo
                 hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                 attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
             )
+        
 
         hidden_states = encoder_outputs[0]
 
@@ -768,7 +516,6 @@ class LlavaT5ForConditionalGeneration(T5ForConditionalGeneration, LlavaMetaForCo
                 decoder_attention_mask = decoder_attention_mask.to(self.decoder.first_device)
 
 
-        # Decode
         decoder_outputs = self.decoder(
             input_ids=decoder_input_ids,
             attention_mask=decoder_attention_mask,
@@ -858,6 +605,5 @@ class LlavaT5ForConditionalGeneration(T5ForConditionalGeneration, LlavaMetaForCo
             "decoder_attention_mask": decoder_attention_mask,
             "cross_attn_head_mask": cross_attn_head_mask,
             "use_cache": use_cache,
-            "images": kwargs.get("images", None),
+            "pixel_values": kwargs.get("pixel_values", None),
         }
-  

config.json

@@ -28,7 +28,7 @@
   "mm_use_im_start_end": false,
   "mm_vision_select_feature": "patch",
   "mm_vision_select_layer": -1,
-  "mm_vision_tower": "/content/unichart-encoder-512",
+  "mm_vision_tower": "/home/masry20/projects/def-enamul/masry20/llms-models/unichart-encoder-512",
   "model_type": "llava_t5",
   "n_positions": 512,
   "num_decoder_layers": 24,
@@ -73,5 +73,102 @@
   "tune_mm_mlp_adapter": false,
   "use_cache": false,
   "use_mm_proj": true,
+  "vision_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": [
+      "DonutSwinModel"
+    ],
+    "attention_probs_dropout_prob": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "depths": [
+      2,
+      2,
+      14,
+      2
+    ],
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "drop_path_rate": 0.1,
+    "early_stopping": false,
+    "embed_dim": 128,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": [
+      512,
+      512
+    ],
+    "initializer_range": 0.02,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "mlp_ratio": 4.0,
+    "model_type": "donut-swin",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_heads": [
+      4,
+      8,
+      16,
+      32
+    ],
+    "num_layers": 4,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 4,
+    "path_norm": true,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qkv_bias": true,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "float32",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_absolute_embeddings": false,
+    "use_bfloat16": false,
+    "window_size": 10
+  },
   "vocab_size": 32128
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:719f6af29df4fe3cae844752de131258afdebe6bad05a513533d654a7f3a2ad9
-size 4986432872
+oid sha256:e6b8f697fd0f6cd28ad003060372ec0787530fb6aec464ec575039f5b089001c
+size 4998389912

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71428b8d193b4234668328861cd30d0abd6ce25bccdaa12de5ce0aa90fde81e9
-size 4991730200
+oid sha256:3d52e625d16b7cd51b5d0cd2e4f8ee1a9e593b540be21346150e7a68c2124990
+size 4984514272

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0059c19a7881bdff91877bedc0b46cd6d2cc6ee1b3ab6ea0581fa41604e6aaa
-size 1429331352
+oid sha256:3bfe7d13f294454178cf4857235c8d521fc45fa02a8f0a9e876a18ae5e58ce8d
+size 1722967504

model.safetensors.index.json

@@ -1,17 +1,17 @@
 {
   "metadata": {
-    "total_size": 11407425536
+    "total_size": 11705748448
   },
   "weight_map": {
-    "decoder.block.0.layer.0.SelfAttention.k.weight": "model-00001-of-00003.safetensors",
-    "decoder.block.0.layer.0.SelfAttention.o.weight": "model-00001-of-00003.safetensors",
-    "decoder.block.0.layer.0.SelfAttention.q.weight": "model-00001-of-00003.safetensors",
-    "decoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight": "model-00001-of-00003.safetensors",
-    "decoder.block.0.layer.0.SelfAttention.v.weight": "model-00001-of-00003.safetensors",
-    "decoder.block.0.layer.0.layer_norm.weight": "model-00001-of-00003.safetensors",
+    "decoder.block.0.layer.0.SelfAttention.k.weight": "model-00002-of-00003.safetensors",
+    "decoder.block.0.layer.0.SelfAttention.o.weight": "model-00002-of-00003.safetensors",
+    "decoder.block.0.layer.0.SelfAttention.q.weight": "model-00002-of-00003.safetensors",
+    "decoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight": "model-00002-of-00003.safetensors",
+    "decoder.block.0.layer.0.SelfAttention.v.weight": "model-00002-of-00003.safetensors",
+    "decoder.block.0.layer.0.layer_norm.weight": "model-00002-of-00003.safetensors",
     "decoder.block.0.layer.1.EncDecAttention.k.weight": "model-00002-of-00003.safetensors",
     "decoder.block.0.layer.1.EncDecAttention.o.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.0.layer.1.EncDecAttention.q.weight": "model-00001-of-00003.safetensors",
+    "decoder.block.0.layer.1.EncDecAttention.q.weight": "model-00002-of-00003.safetensors",
     "decoder.block.0.layer.1.EncDecAttention.v.weight": "model-00002-of-00003.safetensors",
     "decoder.block.0.layer.1.layer_norm.weight": "model-00002-of-00003.safetensors",
     "decoder.block.0.layer.2.DenseReluDense.wi_0.weight": "model-00002-of-00003.safetensors",
@@ -150,24 +150,24 @@
     "decoder.block.18.layer.0.SelfAttention.v.weight": "model-00002-of-00003.safetensors",
     "decoder.block.18.layer.0.layer_norm.weight": "model-00002-of-00003.safetensors",
     "decoder.block.18.layer.1.EncDecAttention.k.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.18.layer.1.EncDecAttention.o.weight": "model-00002-of-00003.safetensors",
+    "decoder.block.18.layer.1.EncDecAttention.o.weight": "model-00003-of-00003.safetensors",
     "decoder.block.18.layer.1.EncDecAttention.q.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.18.layer.1.EncDecAttention.v.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.18.layer.1.layer_norm.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.18.layer.2.DenseReluDense.wi_0.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.18.layer.2.DenseReluDense.wi_1.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.18.layer.2.DenseReluDense.wo.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.18.layer.2.layer_norm.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.0.SelfAttention.k.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.0.SelfAttention.o.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.0.SelfAttention.q.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.0.SelfAttention.v.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.0.layer_norm.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.1.EncDecAttention.k.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.1.EncDecAttention.o.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.1.EncDecAttention.q.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.1.EncDecAttention.v.weight": "model-00002-of-00003.safetensors",
-    "decoder.block.19.layer.1.layer_norm.weight": "model-00002-of-00003.safetensors",
+    "decoder.block.18.layer.1.EncDecAttention.v.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.18.layer.1.layer_norm.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.18.layer.2.DenseReluDense.wi_0.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.18.layer.2.DenseReluDense.wi_1.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.18.layer.2.DenseReluDense.wo.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.18.layer.2.layer_norm.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.0.SelfAttention.k.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.0.SelfAttention.o.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.0.SelfAttention.q.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.0.SelfAttention.v.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.0.layer_norm.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.1.EncDecAttention.k.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.1.EncDecAttention.o.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.1.EncDecAttention.q.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.1.EncDecAttention.v.weight": "model-00003-of-00003.safetensors",
+    "decoder.block.19.layer.1.layer_norm.weight": "model-00003-of-00003.safetensors",
     "decoder.block.19.layer.2.DenseReluDense.wi_0.weight": "model-00003-of-00003.safetensors",
     "decoder.block.19.layer.2.DenseReluDense.wi_1.weight": "model-00003-of-00003.safetensors",
     "decoder.block.19.layer.2.DenseReluDense.wo.weight": "model-00003-of-00003.safetensors",
@@ -559,8 +559,381 @@
     "encoder.block.9.layer.1.DenseReluDense.wo.weight": "model-00001-of-00003.safetensors",
     "encoder.block.9.layer.1.layer_norm.weight": "model-00001-of-00003.safetensors",
     "encoder.final_layer_norm.weight": "model-00001-of-00003.safetensors",
-    "encoder.mm_projector.bias": "model-00001-of-00003.safetensors",
-    "encoder.mm_projector.weight": "model-00001-of-00003.safetensors",
+    "encoder.mm_projector.bias": "model-00002-of-00003.safetensors",
+    "encoder.mm_projector.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.embeddings.norm.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.embeddings.norm.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.embeddings.patch_embeddings.projection.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.embeddings.patch_embeddings.projection.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.0.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.blocks.1.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.downsample.norm.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.downsample.norm.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.0.downsample.reduction.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.0.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.blocks.1.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.downsample.norm.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.downsample.norm.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.1.downsample.reduction.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.0.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.1.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.self.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.self.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.self.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.self.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.self.relative_position_bias_table": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.self.relative_position_index": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.self.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.attention.self.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.10.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.self.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.self.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.self.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.self.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.self.relative_position_bias_table": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.self.relative_position_index": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.self.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.attention.self.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.11.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.self.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.self.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.self.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.self.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.self.relative_position_bias_table": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.self.relative_position_index": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.self.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.attention.self.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.12.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.self.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.self.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.self.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.self.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.self.relative_position_bias_table": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.self.relative_position_index": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.self.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.attention.self.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.13.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.2.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.3.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.4.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.5.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.intermediate.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.intermediate.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.6.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.output.dense.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.output.dense.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.self.key.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.self.key.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.self.query.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.self.query.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.self.relative_position_bias_table": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.self.relative_position_index": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.self.value.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.attention.self.value.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.layernorm_after.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.layernorm_after.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.layernorm_before.bias": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.layernorm_before.weight": "model-00001-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.7.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.self.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.self.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.self.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.self.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.self.relative_position_bias_table": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.self.relative_position_index": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.self.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.attention.self.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.8.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.self.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.self.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.self.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.self.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.self.relative_position_bias_table": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.self.relative_position_index": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.self.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.attention.self.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.blocks.9.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.downsample.norm.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.downsample.norm.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.2.downsample.reduction.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.self.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.self.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.self.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.self.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.self.relative_position_bias_table": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.self.relative_position_index": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.self.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.attention.self.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.0.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.output.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.self.key.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.self.key.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.self.query.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.self.query.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.self.relative_position_bias_table": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.self.relative_position_index": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.self.value.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.attention.self.value.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.intermediate.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.intermediate.dense.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.layernorm_after.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.layernorm_after.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.layernorm_before.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.layernorm_before.weight": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.output.dense.bias": "model-00002-of-00003.safetensors",
+    "encoder.vision_tower.encoder.layers.3.blocks.1.output.dense.weight": "model-00002-of-00003.safetensors",
     "lm_head.weight": "model-00003-of-00003.safetensors",
     "shared.weight": "model-00001-of-00003.safetensors"
   }