Upload model

CXR_LLAVA_HF.py

@@ -0,0 +1,642 @@
+from transformers import PretrainedConfig, PreTrainedModel
+import torch, transformers
+from typing import List, Optional, Tuple, Union
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from .VisualTransformer import  VisionTransformer, LayerNorm
+from functools import partial
+from transformers import TextIteratorStreamer
+from transformers import StoppingCriteria, GenerationConfig
+from threading import Thread
+
+# Model Constants
+IGNORE_INDEX = -100
+IMAGE_TOKEN_INDEX = -200
+DEFAULT_IMAGE_TOKEN = "<image>"
+DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
+DEFAULT_IM_START_TOKEN = "<im_start>"
+DEFAULT_IM_END_TOKEN = "<im_end>"
+class AttrDict(dict):
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self
+
+class CXRLLAVAConfig(PretrainedConfig):
+    model_type = "CXR-LLAVA"
+
+    def __init__(self,  **kwargs,):
+
+        if 'llama' in kwargs:
+            self.llama = AttrDict(kwargs['llama'])
+            del kwargs['llama']
+
+        self.__dict__.update(kwargs)
+        super().__init__(**kwargs)
+
+
+class CXRLLAVAModel(PreTrainedModel):
+    config_class = CXRLLAVAConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.tokenizer = transformers.LlamaTokenizer.from_pretrained(config._name_or_path, add_special_tokens=False)
+        self.tokenizer.pad_token = self.tokenizer.unk_token
+        self.tokenizer.sep_token = self.tokenizer.unk_token
+        self.tokenizer.cls_token = self.tokenizer.unk_token
+        self.tokenizer.mask_token = self.tokenizer.unk_token
+
+        from open_clip.model import CLIPVisionCfg
+        vision_cfg = CLIPVisionCfg(**config.clip_vision_cfg)
+
+        self.generation_config =  GenerationConfig.from_pretrained(config._name_or_path)
+
+        vision_heads = vision_cfg.width // vision_cfg.head_width
+        norm_layer = LayerNorm
+        act_layer = torch.nn.GELU
+        if vision_cfg.norm_kwargs:
+            norm_layer = partial(norm_layer, **vision_cfg.norm_kwargs)
+        if vision_cfg.act_kwargs is not None:
+            act_layer = partial(act_layer, **vision_cfg.act_kwargs)
+
+        self.vision_tower = VisionTransformer(
+            in_channels=1,
+            image_size=vision_cfg.image_size,
+            patch_size=vision_cfg.patch_size,
+            width=vision_cfg.width,
+            layers=vision_cfg.layers,
+            heads=vision_heads,
+            mlp_ratio=vision_cfg.mlp_ratio,
+            ls_init_value=vision_cfg.ls_init_value,
+            patch_dropout=vision_cfg.patch_dropout,
+            attentional_pool=vision_cfg.attentional_pool,
+            attn_pooler_queries=vision_cfg.attn_pooler_queries,
+            attn_pooler_heads=vision_cfg.attn_pooler_heads,
+            pos_embed_type=vision_cfg.pos_embed_type,
+            no_ln_pre=vision_cfg.no_ln_pre,
+            final_ln_after_pool=vision_cfg.final_ln_after_pool,
+            pool_type=vision_cfg.pool_type,
+            output_tokens=vision_cfg.output_tokens,
+            output_dim=config.clip_embed_dim,
+            act_layer=act_layer,
+            norm_layer=norm_layer,
+        )
+
+        self.vision_tower.image_processor = transformers.CLIPImageProcessor(
+            do_resize=True,
+            size={'shortest_edge': config.clip_vision_cfg['image_size']},
+            resample=True,
+            do_center_crop=True,
+            crop_size=config.clip_vision_cfg['image_size'],
+            do_rescale=True,
+            rescale_factor=1 / 255,
+            do_normalize=True,
+            image_mean=config.image_preprocess_cfg['mean'],
+            image_std=config.image_preprocess_cfg['std'],
+            do_convert_rgb=False
+        )
+
+        def convert_dtype(dtype):
+            if dtype == 'fp32':
+                dtype = torch.float32
+            elif dtype == 'fp16':
+                dtype = torch.float16
+            elif dtype == 'bf16':
+                dtype = torch.bfloat16
+            else:
+                raise Exception("Unsupported dtype")
+            return dtype
+
+        self.clip_cast_dtype = convert_dtype(config.clip_vision_tower_dtype)
+        self.mm_projector = torch.nn.Linear(config.mm_projector_dim, config.llama['hidden_size'])
+        self.lm_head = torch.nn.Linear(config.llama.hidden_size, config.llama.vocab_size, bias=False)
+        self.llama = transformers.LlamaModel(transformers.LlamaConfig(**config.llama))
+
+        self.llama = self.llama.to(torch.bfloat16)
+        self.lm_head = self.lm_head.to(torch.bfloat16)
+        self.vision_tower = self.vision_tower.to(torch.bfloat16)
+        self.mm_projector = self.mm_projector.to(torch.bfloat16)
+
+    def get_input_embeddings(self):
+        return self.llama.get_input_embeddings()
+
+    def get_vision_tower(self):
+        return self.vision_tower
+
+    def gradient_checkpointing_enable(self):
+        return self.llama.gradient_checkpointing_enable()
+
+    def encode_images(self, images):
+        images = images.to(torch.bfloat16)
+
+        def _expand_token(token, batch_size: int):
+            return token.view(1, 1, -1).expand(batch_size, -1, -1)
+
+        # open_clip ViT
+        # https://github.com/mlfoundations/open_clip/blob/main/src/open_clip/transformer.py
+        x = images
+        x = self.vision_tower.conv1(x)  # shape = [*, width, grid, grid]
+        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+
+        # class embeddings and positional embeddings
+        x = torch.cat([_expand_token(self.vision_tower.class_embedding, x.shape[0]).to(x.dtype), x], dim=1)
+        # shape = [*, grid ** 2 + 1, width]
+        x = x + self.vision_tower.positional_embedding.to(x.dtype)
+
+        x = self.vision_tower.patch_dropout(x)
+        x = self.vision_tower.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.vision_tower.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        if self.vision_tower.attn_pool is not None:
+            if self.vision_tower.attn_pool_contrastive is not None:
+                # This is untested, WIP pooling that should match paper
+                x = self.vision_tower.ln_post(x)  # TBD LN first or separate one after each pool?
+                tokens = self.vision_tower.attn_pool(x)
+                if self.vision_tower.attn_pool_type == 'parallel':
+                    pooled = self.vision_tower.attn_pool_contrastive(x)
+                else:
+                    assert self.vision_tower.attn_pool_type == 'cascade'
+                    pooled = self.vision_tower.attn_pool_contrastive(tokens)
+            else:
+                # this is the original OpenCLIP CoCa setup, does not match paper
+                x = self.vision_tower.attn_pool(x)
+                x = self.vision_tower.ln_post(x)
+                pooled, tokens = self.vision_tower._global_pool(x)
+        elif self.vision_tower.final_ln_after_pool:
+            pooled, tokens = self.vision_tower._global_pool(x)
+            pooled = self.vision_tower.ln_post(pooled)
+        else:
+            x = self.vision_tower.ln_post(x)
+            pooled, tokens = self.vision_tower._global_pool(x)
+
+        if self.vision_tower.proj is not None:
+            pooled = pooled @ self.vision_tower.proj
+
+        image_features = tokens
+        image_features = image_features.to(torch.bfloat16)
+        image_features = self.mm_projector(image_features)
+
+        image_features = image_features.to(torch.bfloat16)
+        return image_features
+
+    def forward(
+            self,
+            input_ids: torch.LongTensor = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            past_key_values: Optional[List[torch.FloatTensor]] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            labels: Optional[torch.LongTensor] = None,  # (1,4317)
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            images: Optional[torch.FloatTensor] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+
+        input_ids, attention_mask, past_key_values, inputs_embeds, labels = self.prepare_inputs_labels_for_multimodal(
+            input_ids, attention_mask, past_key_values, labels, images)
+
+        outputs = self.llama(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    # original multimodal code
+    def prepare_inputs_labels_for_multimodal(
+            self, input_ids, attention_mask, past_key_values, labels, images
+    ):
+        vision_tower = self.vision_tower
+        if vision_tower is None or images is None or input_ids.shape[1] == 1:
+            if past_key_values is not None and vision_tower is not None and images is not None and input_ids.shape[
+                1] == 1:
+                attention_mask = torch.ones((attention_mask.shape[0], past_key_values[-1][-1].shape[-2] + 1),
+                                            dtype=attention_mask.dtype, device=attention_mask.device)
+            return input_ids, attention_mask, past_key_values, None, labels
+
+        if type(images) is list or images.ndim == 5:
+            concat_images = torch.cat([image for image in images], dim=0)
+            image_features = self.encode_images(concat_images)
+            split_sizes = [image.shape[0] for image in images]
+            image_features = torch.split(image_features, split_sizes, dim=0)
+            image_features = [x.flatten(0, 1) for x in image_features]
+        else:
+            image_features = self.encode_images(images)
+
+        new_input_embeds = []
+        new_labels = [] if labels is not None else None
+        cur_image_idx = 0
+        for batch_idx, cur_input_ids in enumerate(input_ids):
+            if (cur_input_ids == IMAGE_TOKEN_INDEX).sum() == 0:
+                # multimodal LLM, but the current sample is not multimodal
+                cur_input_embeds = self.llama.embed_tokens(cur_input_ids)
+                cur_input_embeds = cur_input_embeds + (0. * self.mm_projector(vision_tower.dummy_feature)).sum()
+                new_input_embeds.append(cur_input_embeds)
+                if labels is not None:
+                    new_labels.append(labels[batch_idx])
+                cur_image_idx += 1
+                continue
+            image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
+            cur_new_input_embeds = []
+            if labels is not None:
+                cur_labels = labels[batch_idx]
+                cur_new_labels = []
+                assert cur_labels.shape == cur_input_ids.shape
+            while image_token_indices.numel() > 0:
+                cur_image_features = image_features[cur_image_idx]
+                image_token_start = image_token_indices[0]
+                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end',
+                                                                                  False):
+                    cur_new_input_embeds.append(self.llama.embed_tokens(cur_input_ids[:image_token_start - 1]).detach())
+                    cur_new_input_embeds.append(
+                        self.llama.embed_tokens(cur_input_ids[image_token_start - 1:image_token_start]))
+                    cur_new_input_embeds.append(cur_image_features)
+                    cur_new_input_embeds.append(
+                        self.llama.embed_tokens(cur_input_ids[image_token_start + 1:image_token_start + 2]))
+                    if labels is not None:
+                        cur_new_labels.append(cur_labels[:image_token_start])
+                        cur_new_labels.append(
+                            torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device,
+                                       dtype=labels.dtype))
+                        cur_new_labels.append(cur_labels[image_token_start:image_token_start + 1])
+                        cur_labels = cur_labels[image_token_start + 2:]
+                else:
+                    cur_new_input_embeds.append(self.llama.embed_tokens(cur_input_ids[:image_token_start]))
+                    cur_new_input_embeds.append(cur_image_features)
+                    if labels is not None:
+                        cur_new_labels.append(cur_labels[:image_token_start])
+                        cur_new_labels.append(
+                            torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device,
+                                       dtype=labels.dtype))
+                        cur_labels = cur_labels[image_token_start + 1:]
+                cur_image_idx += 1
+                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end',
+                                                                                  False):
+                    cur_input_ids = cur_input_ids[image_token_start + 2:]
+                else:
+                    cur_input_ids = cur_input_ids[image_token_start + 1:]
+                image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
+            if cur_input_ids.numel() > 0:
+                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end',
+                                                                                  False):
+                    cur_new_input_embeds.append(self.llama.embed_tokens(cur_input_ids).detach())
+                else:
+                    cur_new_input_embeds.append(self.llama.embed_tokens(cur_input_ids))
+                if labels is not None:
+                    cur_new_labels.append(cur_labels)
+            cur_new_input_embeds = [x.to(device=self.device) for x in cur_new_input_embeds]
+
+            cur_new_input_embeds = torch.cat(cur_new_input_embeds, dim=0)
+            new_input_embeds.append(cur_new_input_embeds)
+            if labels is not None:
+                cur_new_labels = torch.cat(cur_new_labels, dim=0)
+                new_labels.append(cur_new_labels)
+
+        if any(x.shape != new_input_embeds[0].shape for x in new_input_embeds):
+            max_len = max(x.shape[0] for x in new_input_embeds)
+
+            new_input_embeds_align = []
+            for cur_new_embed in new_input_embeds:
+                cur_new_embed = torch.cat((cur_new_embed,
+                                           torch.zeros((max_len - cur_new_embed.shape[0], cur_new_embed.shape[1]),
+                                                       dtype=cur_new_embed.dtype, device=cur_new_embed.device)), dim=0)
+                new_input_embeds_align.append(cur_new_embed)
+            new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
+
+            if labels is not None:
+                new_labels_align = []
+                _new_labels = new_labels
+                for cur_new_label in new_labels:
+                    cur_new_label = torch.cat((cur_new_label,
+                                               torch.full((max_len - cur_new_label.shape[0],), IGNORE_INDEX,
+                                                          dtype=cur_new_label.dtype, device=cur_new_label.device)),
+                                              dim=0)
+                    new_labels_align.append(cur_new_label)
+                new_labels = torch.stack(new_labels_align, dim=0)
+
+            if attention_mask is not None:
+                new_attention_mask = []
+                for cur_attention_mask, cur_new_labels, cur_new_labels_align in zip(attention_mask, _new_labels,
+                                                                                    new_labels):
+                    new_attn_mask_pad_left = torch.full((cur_new_labels.shape[0] - labels.shape[1],), True,
+                                                        dtype=attention_mask.dtype, device=attention_mask.device)
+                    new_attn_mask_pad_right = torch.full((cur_new_labels_align.shape[0] - cur_new_labels.shape[0],),
+                                                         False, dtype=attention_mask.dtype,
+                                                         device=attention_mask.device)
+                    cur_new_attention_mask = torch.cat(
+                        (new_attn_mask_pad_left, cur_attention_mask, new_attn_mask_pad_right), dim=0)
+                    new_attention_mask.append(cur_new_attention_mask)
+                attention_mask = torch.stack(new_attention_mask, dim=0)
+                assert attention_mask.shape == new_labels.shape
+        else:
+            new_input_embeds = torch.stack(new_input_embeds, dim=0)
+            if labels is not None:
+                new_labels = torch.stack(new_labels, dim=0)
+
+            if attention_mask is not None:
+                new_attn_mask_pad_left = torch.full(
+                    (attention_mask.shape[0], new_input_embeds.shape[1] - input_ids.shape[1]), True,
+                    dtype=attention_mask.dtype, device=attention_mask.device)
+                attention_mask = torch.cat((new_attn_mask_pad_left, attention_mask), dim=1)
+                assert attention_mask.shape == new_input_embeds.shape[:2]
+
+        return None, attention_mask, past_key_values, new_input_embeds, new_labels
+
+    # sw-modified code
+
+    def prepare_inputs_labels_for_multimodal_use_final_vector(
+            self, input_ids, attention_mask, past_key_values, labels, images
+    ):
+        vision_tower = self.vision_tower
+        if vision_tower is None or images is None or input_ids.shape[1] == 1:
+            if past_key_values is not None and vision_tower is not None and images is not None and input_ids.shape[
+                1] == 1:
+                attention_mask = torch.ones((attention_mask.shape[0], past_key_values[-1][-1].shape[-2] + 1),
+                                            dtype=attention_mask.dtype, device=attention_mask.device)
+            return input_ids, attention_mask, past_key_values, None, labels
+
+        if type(images) is list or images.ndim == 5:
+            concat_images = torch.cat([image for image in images], dim=0)
+            image_features = self.encode_images(concat_images)
+            split_sizes = [image.shape[0] for image in images]
+            image_features = torch.split(image_features, split_sizes, dim=0)
+            image_features = [x.flatten(0, 1) for x in image_features]
+        else:
+            image_features = self.encode_images(images)
+
+        new_input_embeds = []
+        new_labels = [] if labels is not None else None
+        cur_image_idx = 0
+        for batch_idx, cur_input_ids in enumerate(input_ids):
+            if (cur_input_ids == IMAGE_TOKEN_INDEX).sum() == 0:
+                # multimodal LLM, but the current sample is not multimodal
+                cur_input_embeds = self.llama.embed_tokens(cur_input_ids)
+                cur_input_embeds = cur_input_embeds + (0. * self.mm_projector(vision_tower.dummy_feature)).sum()
+                new_input_embeds.append(cur_input_embeds)
+                if labels is not None:
+                    new_labels.append(labels[batch_idx])
+                cur_image_idx += 1
+                continue
+            image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
+            cur_new_input_embeds = []
+            if labels is not None:
+                cur_labels = labels[batch_idx]
+                cur_new_labels = []
+                assert cur_labels.shape == cur_input_ids.shape
+            while image_token_indices.numel() > 0:
+                cur_image_features = image_features[cur_image_idx]
+                image_token_start = image_token_indices[0]
+                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end',
+                                                                                  False):
+                    cur_new_input_embeds.append(self.llama.embed_tokens(cur_input_ids[:image_token_start - 1]).detach())
+                    cur_new_input_embeds.append(
+                        self.llama.embed_tokens(cur_input_ids[image_token_start - 1:image_token_start]))
+                    cur_new_input_embeds.append(cur_image_features)
+                    cur_new_input_embeds.append(
+                        self.llama.embed_tokens(cur_input_ids[image_token_start + 1:image_token_start + 2]))
+                    if labels is not None:
+                        cur_new_labels.append(cur_labels[:image_token_start])
+                        cur_new_labels.append(
+                            torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device,
+                                       dtype=labels.dtype))
+                        cur_new_labels.append(cur_labels[image_token_start:image_token_start + 1])
+                        cur_labels = cur_labels[image_token_start + 2:]
+                else:
+                    cur_new_input_embeds.append(
+                        self.llama.embed_tokens(cur_input_ids[:image_token_start].to(self.device)))
+                    cur_new_input_embeds.append(cur_image_features)
+                    if labels is not None:
+                        cur_new_labels.append(cur_labels[:image_token_start])
+                        cur_new_labels.append(
+                            torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device,
+                                       dtype=labels.dtype))
+                        cur_labels = cur_labels[image_token_start + 1:]
+                cur_image_idx += 1
+                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end',
+                                                                                  False):
+                    cur_input_ids = cur_input_ids[image_token_start + 2:]
+                else:
+                    cur_input_ids = cur_input_ids[image_token_start + 1:]
+                image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
+            if cur_input_ids.numel() > 0:
+                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end',
+                                                                                  False):
+                    cur_new_input_embeds.append(self.llama.embed_tokens(cur_input_ids).detach())
+                else:
+                    cur_new_input_embeds.append(self.llama.embed_tokens(cur_input_ids.to(self.device)))
+                if labels is not None:
+                    # seowoo-edit
+                    cur_labels = labels[batch_idx]
+                    cur_new_labels.append(cur_labels)
+            # [5120] -> [1, 5120]
+            cur_new_input_embeds[1] = torch.unsqueeze(cur_new_input_embeds[1], dim=0)
+            cur_new_input_embeds = [x.to(device=self.device) for x in cur_new_input_embeds]
+            cur_new_input_embeds = torch.cat(cur_new_input_embeds, dim=0)
+            new_input_embeds.append(cur_new_input_embeds)
+            if labels is not None:
+                cur_new_labels = torch.cat(cur_new_labels, dim=0)
+                new_labels.append(cur_new_labels)
+
+        if any(x.shape != new_input_embeds[0].shape for x in new_input_embeds):
+            # print("if 204")
+            max_len = max(x.shape[0] for x in new_input_embeds)
+
+            new_input_embeds_align = []
+            for cur_new_embed in new_input_embeds:
+                cur_new_embed = torch.cat((cur_new_embed,
+                                           torch.zeros((max_len - cur_new_embed.shape[0], cur_new_embed.shape[1]),
+                                                       dtype=cur_new_embed.dtype, device=cur_new_embed.device)), dim=0)
+                new_input_embeds_align.append(cur_new_embed)
+            new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
+
+            if labels is not None:
+                new_labels_align = []
+                _new_labels = new_labels
+                for cur_new_label in new_labels:
+                    cur_new_label = torch.cat((cur_new_label,
+                                               torch.full((max_len - cur_new_label.shape[0],), IGNORE_INDEX,
+                                                          dtype=cur_new_label.dtype, device=cur_new_label.device)),
+                                              dim=0)
+                    new_labels_align.append(cur_new_label)
+                new_labels = torch.stack(new_labels_align, dim=0)
+
+            if attention_mask is not None:
+                new_attention_mask = []
+                for cur_attention_mask, cur_new_labels, cur_new_labels_align in zip(attention_mask, _new_labels,
+                                                                                    new_labels):
+                    new_attn_mask_pad_left = torch.full((cur_new_labels.shape[0] - labels.shape[1],), True,
+                                                        dtype=attention_mask.dtype, device=attention_mask.device)
+                    new_attn_mask_pad_right = torch.full((cur_new_labels_align.shape[0] - cur_new_labels.shape[0],),
+                                                         False, dtype=attention_mask.dtype,
+                                                         device=attention_mask.device)
+                    cur_new_attention_mask = torch.cat(
+                        (new_attn_mask_pad_left, cur_attention_mask, new_attn_mask_pad_right), dim=0)
+                    new_attention_mask.append(cur_new_attention_mask)
+                attention_mask = torch.stack(new_attention_mask, dim=0)
+                assert attention_mask.shape == new_labels.shape
+        else:
+            new_input_embeds = torch.stack(new_input_embeds, dim=0)
+            if labels is not None:
+                new_labels = torch.stack(new_labels, dim=0)
+
+            if attention_mask is not None:
+                new_attn_mask_pad_left = torch.full(
+                    (attention_mask.shape[0], new_input_embeds.shape[1] - input_ids.shape[1]), True,
+                    dtype=attention_mask.dtype, device=attention_mask.device)
+                attention_mask = torch.cat((new_attn_mask_pad_left, attention_mask), dim=1)
+                assert attention_mask.shape == new_input_embeds.shape[:2]
+
+            return None, attention_mask, past_key_values, new_input_embeds, labels
+
+    def prepare_inputs_for_generation(
+            self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        if past_key_values:
+            input_ids = input_ids[:, -1:]
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+        model_inputs.update(
+            {
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "images": kwargs.get("images", None),
+            }
+        )
+        return model_inputs
+
+    def apply_chat_template(self, chat):
+        return self.tokenizer.apply_chat_template(chat, tokenize=False)
+
+    def tokenizer_image_token(self, prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
+        prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
+
+        def insert_separator(X, sep):
+            return [ele for sublist in zip(X, [sep] * len(X)) for ele in sublist][:-1]
+
+        input_ids = []
+        offset = 0
+        if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id:
+            offset = 1
+            input_ids.append(prompt_chunks[0][0])
+
+        for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)):
+            input_ids.extend(x[offset:])
+
+        if return_tensors is not None:
+            if return_tensors == 'pt':
+                return torch.tensor(input_ids, dtype=torch.long)
+            raise ValueError(f'Unsupported tensor type: {return_tensors}')
+        return input_ids
+
+    def generate_cxr_repsonse(self, chat, pil_image, temperature=0.2, top_p=0.8):
+        with torch.no_grad():
+            streamer = TextIteratorStreamer(self.tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=15)
+            import numpy as np
+            pil_image = np.expand_dims(pil_image,axis=-1)
+            prompt = self.apply_chat_template(chat)
+            images = self.vision_tower.image_processor(pil_image, return_tensors='pt')['pixel_values']
+            images = images.to(self.device)
+            input_ids = self.tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+            stopping_criteria = KeywordsStoppingCriteria(["</s>"], self.tokenizer, input_ids)
+
+            image_args = {"images": images}
+            do_sample = True if temperature > 0.001 else False
+            num_image_tokens = 1
+            max_context_length = getattr(self.config, 'max_position_embeddings', 2048)
+
+            max_new_tokens = min(512, max_context_length - input_ids.shape[-1] - num_image_tokens)
+
+            thread = Thread(target=self.generate, kwargs=dict(
+                inputs=input_ids,
+                do_sample=do_sample,
+                temperature=temperature,
+                top_p=top_p,
+                max_new_tokens=max_new_tokens,
+                streamer=streamer,
+                stopping_criteria=[stopping_criteria],
+                use_cache=True,
+                generation_config=self.generation_config,
+                **image_args
+            ))
+            thread.start()
+            generated_text = ""
+            for new_text in streamer:
+                generated_text += new_text
+
+        return generated_text
+
+    def tokenizer_image_token(self, prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
+        prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
+
+        def insert_separator(X, sep):
+            return [ele for sublist in zip(X, [sep] * len(X)) for ele in sublist][:-1]
+
+        input_ids = []
+        offset = 0
+        if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id:
+            offset = 1
+            input_ids.append(prompt_chunks[0][0])
+
+        for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)):
+            input_ids.extend(x[offset:])
+
+        if return_tensors is not None:
+            if return_tensors == 'pt':
+                return torch.tensor(input_ids, dtype=torch.long)
+            raise ValueError(f'Unsupported tensor type: {return_tensors}')
+        return input_ids
+class KeywordsStoppingCriteria(StoppingCriteria):
+    def __init__(self, keywords, tokenizer, input_ids):
+        self.keywords = keywords
+        self.keyword_ids = []
+        for keyword in keywords:
+            cur_keyword_ids = tokenizer(keyword).input_ids
+            if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id:
+                cur_keyword_ids = cur_keyword_ids[1:]
+            self.keyword_ids.append(torch.tensor(cur_keyword_ids))
+        self.tokenizer = tokenizer
+        self.start_len = input_ids.shape[1]
+
+    def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        assert output_ids.shape[0] == 1, "Only support batch size 1 (yet)"  # TODO
+        offset = min(output_ids.shape[1] - self.start_len, 3)
+        self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids]
+        for keyword_id in self.keyword_ids:
+            if output_ids[0, -keyword_id.shape[0]:] == keyword_id:
+                return True
+        outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0]
+        for keyword in self.keywords:
+            if keyword in outputs:
+                return True
+        return False

VisualTransformer.py

@@ -0,0 +1,917 @@
+from collections import OrderedDict
+import math
+from typing import Callable, Optional, Sequence, Tuple
+from functools import partial
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from torch.utils.checkpoint import checkpoint
+
+from itertools import repeat
+import collections.abc
+
+# From PyTorch internals
+def _ntuple(n):
+    def parse(x):
+        if isinstance(x, collections.abc.Iterable):
+            return x
+        return tuple(repeat(x, n))
+    return parse
+
+to_1tuple = _ntuple(1)
+to_2tuple = _ntuple(2)
+to_3tuple = _ntuple(3)
+to_4tuple = _ntuple(4)
+to_ntuple = lambda n, x: _ntuple(n)(x)
+
+class LayerNormFp32(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16 (by casting to float32 and back)."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        x = F.layer_norm(x.to(torch.float32), self.normalized_shape, self.weight, self.bias, self.eps)
+
+        #x = F.layer_norm(x.to(torch.bfloat16), self.normalized_shape, self.weight, self.bias, self.eps)
+        return x.to(orig_type)
+
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm (with cast back to input dtype)."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        x = F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)
+        return x.to(orig_type)
+
+
+class QuickGELU(nn.Module):
+    # NOTE This is slower than nn.GELU or nn.SiLU and uses more GPU memory
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+
+class LayerScale(nn.Module):
+    def __init__(self, dim, init_values=1e-5, inplace=False):
+        super().__init__()
+        self.inplace = inplace
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+    def forward(self, x):
+        return x.mul_(self.gamma) if self.inplace else x * self.gamma
+
+
+class PatchDropout(nn.Module):
+    """
+    https://arxiv.org/abs/2212.00794
+    """
+
+    def __init__(self, prob, exclude_first_token=True):
+        super().__init__()
+        assert 0 <= prob < 1.
+        self.prob = prob
+        self.exclude_first_token = exclude_first_token  # exclude CLS token
+
+    def forward(self, x):
+        if not self.training or self.prob == 0.:
+            return x
+
+        if self.exclude_first_token:
+            cls_tokens, x = x[:, :1], x[:, 1:]
+        else:
+            cls_tokens = torch.jit.annotate(torch.Tensor, x[:, :1])
+
+        batch = x.size()[0]
+        num_tokens = x.size()[1]
+
+        batch_indices = torch.arange(batch)
+        batch_indices = batch_indices[..., None]
+
+        keep_prob = 1 - self.prob
+        num_patches_keep = max(1, int(num_tokens * keep_prob))
+
+        rand = torch.randn(batch, num_tokens)
+        patch_indices_keep = rand.topk(num_patches_keep, dim=-1).indices
+
+        x = x[batch_indices, patch_indices_keep]
+
+        if self.exclude_first_token:
+            x = torch.cat((cls_tokens, x), dim=1)
+
+        return x
+
+
+class Attention(nn.Module):
+    def __init__(
+            self,
+            dim,
+            num_heads=8,
+            qkv_bias=True,
+            scaled_cosine=False,
+            scale_heads=False,
+            logit_scale_max=math.log(1. / 0.01),
+            attn_drop=0.,
+            proj_drop=0.
+    ):
+        super().__init__()
+        self.scaled_cosine = scaled_cosine
+        self.scale_heads = scale_heads
+        assert dim % num_heads == 0, 'dim should be divisible by num_heads'
+        self.num_heads = num_heads
+        self.head_dim = dim // num_heads
+        self.scale = self.head_dim ** -0.5
+        self.logit_scale_max = logit_scale_max
+
+        # keeping in_proj in this form (instead of nn.Linear) to match weight scheme of original
+        self.in_proj_weight = nn.Parameter(torch.randn((dim * 3, dim)) * self.scale)
+        if qkv_bias:
+            self.in_proj_bias = nn.Parameter(torch.zeros(dim * 3))
+        else:
+            self.in_proj_bias = None
+
+        if self.scaled_cosine:
+            self.logit_scale = nn.Parameter(torch.log(10 * torch.ones((num_heads, 1, 1))))
+        else:
+            self.logit_scale = None
+        self.attn_drop = nn.Dropout(attn_drop)
+        if self.scale_heads:
+            self.head_scale = nn.Parameter(torch.ones((num_heads, 1, 1)))
+        else:
+            self.head_scale = None
+        self.out_proj = nn.Linear(dim, dim)
+        self.out_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x, attn_mask: Optional[torch.Tensor] = None):
+        L, N, C = x.shape
+        q, k, v = F.linear(x, self.in_proj_weight, self.in_proj_bias).chunk(3, dim=-1)
+        q = q.contiguous().view(L, N * self.num_heads, -1).transpose(0, 1)
+        k = k.contiguous().view(L, N * self.num_heads, -1).transpose(0, 1)
+        v = v.contiguous().view(L, N * self.num_heads, -1).transpose(0, 1)
+
+        if self.logit_scale is not None:
+            attn = torch.bmm(F.normalize(q, dim=-1), F.normalize(k, dim=-1).transpose(-1, -2))
+            logit_scale = torch.clamp(self.logit_scale, max=self.logit_scale_max).exp()
+            attn = attn.view(N, self.num_heads, L, L) * logit_scale
+            attn = attn.view(-1, L, L)
+        else:
+            q = q * self.scale
+            attn = torch.bmm(q, k.transpose(-1, -2))
+
+        if attn_mask is not None:
+            if attn_mask.dtype == torch.bool:
+                new_attn_mask = torch.zeros_like(attn_mask, dtype=q.dtype)
+                new_attn_mask.masked_fill_(attn_mask, float("-inf"))
+                attn_mask = new_attn_mask
+            attn += attn_mask
+
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = torch.bmm(attn, v)
+        if self.head_scale is not None:
+            x = x.view(N, self.num_heads, L, C) * self.head_scale
+            x = x.view(-1, L, C)
+        x = x.transpose(0, 1).reshape(L, N, C)
+        x = self.out_proj(x)
+        x = self.out_drop(x)
+        return x
+
+
+class AttentionalPooler(nn.Module):
+    def __init__(
+            self,
+            d_model: int,
+            context_dim: int,
+            n_head: int = 8,
+            n_queries: int = 256,
+            norm_layer: Callable = LayerNorm
+    ):
+        super().__init__()
+        self.query = nn.Parameter(torch.randn(n_queries, d_model))
+        self.attn = nn.MultiheadAttention(d_model, n_head, kdim=context_dim, vdim=context_dim)
+        self.ln_q = norm_layer(d_model)
+        self.ln_k = norm_layer(context_dim)
+
+    def forward(self, x: torch.Tensor):
+        x = self.ln_k(x).permute(1, 0, 2)  # NLD -> LND
+        N = x.shape[1]
+        q = self.ln_q(self.query)
+        out = self.attn(q.unsqueeze(1).expand(-1, N, -1), x, x, need_weights=False)[0]
+        return out.permute(1, 0, 2)  # LND -> NLD
+
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(
+            self,
+            d_model: int,
+            n_head: int,
+            mlp_ratio: float = 4.0,
+            ls_init_value: float = None,
+            act_layer: Callable = nn.GELU,
+            norm_layer: Callable = LayerNorm,
+            is_cross_attention: bool = False,
+    ):
+        super().__init__()
+
+        self.ln_1 = norm_layer(d_model)
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ls_1 = LayerScale(d_model, ls_init_value) if ls_init_value is not None else nn.Identity()
+        if is_cross_attention:
+            self.ln_1_kv = norm_layer(d_model)
+
+        self.ln_2 = norm_layer(d_model)
+        mlp_width = int(d_model * mlp_ratio)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, mlp_width)),
+            ("gelu", act_layer()),
+            ("c_proj", nn.Linear(mlp_width, d_model))
+        ]))
+        self.ls_2 = LayerScale(d_model, ls_init_value) if ls_init_value is not None else nn.Identity()
+
+    def attention(
+            self,
+            q_x: torch.Tensor,
+            k_x: Optional[torch.Tensor] = None,
+            v_x: Optional[torch.Tensor] = None,
+            attn_mask: Optional[torch.Tensor] = None,
+    ):
+        k_x = k_x if k_x is not None else q_x
+        v_x = v_x if v_x is not None else q_x
+
+        attn_mask = attn_mask.to(q_x.dtype) if attn_mask is not None else None
+        return self.attn(
+            q_x, k_x, v_x, need_weights=False, attn_mask=attn_mask
+        )[0]
+
+    def forward(
+            self,
+            q_x: torch.Tensor,
+            k_x: Optional[torch.Tensor] = None,
+            v_x: Optional[torch.Tensor] = None,
+            attn_mask: Optional[torch.Tensor] = None,
+    ):
+        k_x = self.ln_1_kv(k_x) if hasattr(self, "ln_1_kv") and k_x is not None else None
+        v_x = self.ln_1_kv(v_x) if hasattr(self, "ln_1_kv") and v_x is not None else None
+
+        x = q_x + self.ls_1(self.attention(q_x=self.ln_1(q_x), k_x=k_x, v_x=v_x, attn_mask=attn_mask))
+        x = x + self.ls_2(self.mlp(self.ln_2(x)))
+        return x
+
+
+class CustomResidualAttentionBlock(nn.Module):
+    def __init__(
+            self,
+            d_model: int,
+            n_head: int,
+            mlp_ratio: float = 4.0,
+            ls_init_value: float = None,
+            act_layer: Callable = nn.GELU,
+            norm_layer: Callable = LayerNorm,
+            scale_cosine_attn: bool = False,
+            scale_heads: bool = False,
+            scale_attn: bool = False,
+            scale_fc: bool = False,
+    ):
+        super().__init__()
+
+        self.ln_1 = norm_layer(d_model)
+        self.attn = Attention(
+            d_model, n_head,
+            scaled_cosine=scale_cosine_attn,
+            scale_heads=scale_heads,
+        )
+        self.ln_attn = norm_layer(d_model) if scale_attn else nn.Identity()
+        self.ls_1 = LayerScale(d_model, ls_init_value) if ls_init_value is not None else nn.Identity()
+
+        self.ln_2 = norm_layer(d_model)
+        mlp_width = int(d_model * mlp_ratio)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, mlp_width)),
+            ("gelu", act_layer()),
+            ('ln', norm_layer(mlp_width) if scale_fc else nn.Identity()),
+            ("c_proj", nn.Linear(mlp_width, d_model))
+        ]))
+        self.ls_2 = LayerScale(d_model, ls_init_value) if ls_init_value is not None else nn.Identity()
+
+    def forward(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] = None):
+        x = x + self.ls_1(self.ln_attn(self.attn(self.ln_1(x), attn_mask=attn_mask)))
+        x = x + self.ls_2(self.mlp(self.ln_2(x)))
+        return x
+
+
+def _expand_token(token, batch_size: int):
+    return token.view(1, 1, -1).expand(batch_size, -1, -1)
+
+
+class Transformer(nn.Module):
+    def __init__(
+            self,
+            width: int,
+            layers: int,
+            heads: int,
+            mlp_ratio: float = 4.0,
+            ls_init_value: float = None,
+            act_layer: Callable = nn.GELU,
+            norm_layer: Callable = LayerNorm,
+    ):
+        super().__init__()
+        self.width = width
+        self.layers = layers
+        self.grad_checkpointing = False
+
+        self.resblocks = nn.ModuleList([
+            ResidualAttentionBlock(
+                width, heads, mlp_ratio, ls_init_value=ls_init_value, act_layer=act_layer, norm_layer=norm_layer)
+            for _ in range(layers)
+        ])
+
+    def get_cast_dtype(self) -> torch.dtype:
+        if hasattr(self.resblocks[0].mlp.c_fc, 'int8_original_dtype'):
+            return self.resblocks[0].mlp.c_fc.int8_original_dtype
+        return self.resblocks[0].mlp.c_fc.weight.dtype
+
+    def forward(self, x: torch.Tensor, attn_mask: Optional[torch.Tensor] = None):
+        for r in self.resblocks:
+            if self.grad_checkpointing and not torch.jit.is_scripting():
+                # TODO: handle kwargs https://github.com/pytorch/pytorch/issues/79887#issuecomment-1161758372
+                x = checkpoint(r, x, None, None, attn_mask)
+            else:
+                x = r(x, attn_mask=attn_mask)
+        return x
+
+
+class VisionTransformer(nn.Module):
+    output_tokens: torch.jit.Final[bool]
+
+    def __init__(
+            self,
+            in_channels:int,
+            image_size: int,
+            patch_size: int,
+            width: int,
+            layers: int,
+            heads: int,
+            mlp_ratio: float,
+            ls_init_value: float = None,
+            attentional_pool: bool = False,
+            attn_pooler_queries: int = 256,
+            attn_pooler_heads: int = 8,
+            output_dim: int = 512,
+            patch_dropout: float = 0.,
+            no_ln_pre: bool = False,
+            pos_embed_type: str = 'learnable',
+            pool_type: str = 'tok',
+            final_ln_after_pool: bool = False,
+            act_layer: Callable = nn.GELU,
+            norm_layer: Callable = LayerNorm,
+            output_tokens: bool = False,
+    ):
+        super().__init__()
+        assert pool_type in ('tok', 'avg', 'none')
+        self.output_tokens = output_tokens
+        image_height, image_width = self.image_size = to_2tuple(image_size)
+        patch_height, patch_width = self.patch_size = to_2tuple(patch_size)
+        self.grid_size = (image_height // patch_height, image_width // patch_width)
+        self.final_ln_after_pool = final_ln_after_pool  # currently ignored w/ attn pool enabled
+        self.output_dim = output_dim
+
+        self.conv1 = nn.Conv2d(in_channels=in_channels, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
+
+        # class embeddings and positional embeddings
+        scale = width ** -0.5
+        self.class_embedding = nn.Parameter(scale * torch.randn(width))
+        if pos_embed_type == 'learnable':
+            self.positional_embedding = nn.Parameter(
+                scale * torch.randn(self.grid_size[0] * self.grid_size[1] + 1, width))
+        elif pos_embed_type == 'sin_cos_2d':
+            # fixed sin-cos embedding
+            assert self.grid_size[0] == self.grid_size[1], \
+                'currently sin cos 2d pos embedding only supports square input'
+            self.positional_embedding = nn.Parameter(
+                torch.zeros(self.grid_size[0] * self.grid_size[1] + 1, width), requires_grad=False)
+            pos_embed_type = get_2d_sincos_pos_embed(width, self.grid_size[0], cls_token=True)
+            self.positional_embedding.data.copy_(torch.from_numpy(pos_embed_type).float())
+        else:
+            raise ValueError
+
+        # setting a patch_dropout of 0. would mean it is disabled and this function would be the identity fn
+        self.patch_dropout = PatchDropout(patch_dropout) if patch_dropout > 0. else nn.Identity()
+
+        self.ln_pre = nn.Identity() if no_ln_pre else norm_layer(width)
+        self.transformer = Transformer(
+            width,
+            layers,
+            heads,
+            mlp_ratio,
+            ls_init_value=ls_init_value,
+            act_layer=act_layer,
+            norm_layer=norm_layer,
+        )
+
+        if attentional_pool:
+            if isinstance(attentional_pool, str):
+                self.attn_pool_type = attentional_pool
+                self.pool_type = 'none'
+                if attentional_pool in ('parallel', 'cascade'):
+                    self.attn_pool = AttentionalPooler(
+                        output_dim,
+                        width,
+                        n_head=attn_pooler_heads,
+                        n_queries=attn_pooler_queries,
+                    )
+                    self.attn_pool_contrastive = AttentionalPooler(
+                        output_dim,
+                        width,
+                        n_head=attn_pooler_heads,
+                        n_queries=1,
+                    )
+                else:
+                    assert False
+            else:
+                self.attn_pool_type = ''
+                self.pool_type = pool_type
+                self.attn_pool = AttentionalPooler(
+                    output_dim,
+                    width,
+                    n_head=attn_pooler_heads,
+                    n_queries=attn_pooler_queries,
+                )
+                self.attn_pool_contrastive = None
+            pool_dim = output_dim
+        else:
+            self.attn_pool = None
+            pool_dim = width
+            self.pool_type = pool_type
+
+        self.ln_post = norm_layer(pool_dim)
+        self.proj = nn.Parameter(scale * torch.randn(pool_dim, output_dim))
+
+        self.init_parameters()
+
+    def lock(self, unlocked_groups=0, freeze_bn_stats=False):
+        for param in self.parameters():
+            param.requires_grad = False
+
+        if unlocked_groups != 0:
+            groups = [
+                [
+                    self.conv1,
+                    self.class_embedding,
+                    self.positional_embedding,
+                    self.ln_pre,
+                ],
+                *self.transformer.resblocks[:-1],
+                [
+                    self.transformer.resblocks[-1],
+                    self.ln_post,
+                ],
+                self.proj,
+            ]
+
+            def _unlock(x):
+                if isinstance(x, Sequence):
+                    for g in x:
+                        _unlock(g)
+                else:
+                    if isinstance(x, torch.nn.Parameter):
+                        x.requires_grad = True
+                    else:
+                        for p in x.parameters():
+                            p.requires_grad = True
+
+            _unlock(groups[-unlocked_groups:])
+
+    def init_parameters(self):
+        # FIXME OpenAI CLIP did not define an init for the VisualTransformer
+        # TODO experiment if default PyTorch init, below, or alternate init is best.
+
+        # nn.init.normal_(self.class_embedding, std=self.scale)
+        # nn.init.normal_(self.positional_embedding, std=self.scale)
+        #
+        # proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
+        # attn_std = self.transformer.width ** -0.5
+        # fc_std = (2 * self.transformer.width) ** -0.5
+        # for block in self.transformer.resblocks:
+        #     nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+        #     nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+        #     nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+        #     nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+        #
+        # if self.text_projection is not None:
+        #     nn.init.normal_(self.text_projection, std=self.scale)
+        pass
+
+    @torch.jit.ignore
+    def set_grad_checkpointing(self, enable=True):
+        self.transformer.grad_checkpointing = enable
+
+    def _global_pool(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        if self.pool_type == 'avg':
+            pooled, tokens = x[:, 1:].mean(dim=1), x[:, 1:]
+        elif self.pool_type == 'tok':
+            pooled, tokens = x[:, 0], x[:, 1:]
+        else:
+            pooled = tokens = x
+
+        return pooled, tokens
+
+    def forward(self, x: torch.Tensor):
+        x = self.conv1(x)  # shape = [*, width, grid, grid]
+        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+
+        # class embeddings and positional embeddings
+        x = torch.cat([_expand_token(self.class_embedding, x.shape[0]).to(x.dtype), x], dim=1)
+        # shape = [*, grid ** 2 + 1, width]
+        x = x + self.positional_embedding.to(x.dtype)
+
+        x = self.patch_dropout(x)
+        x = self.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        if self.attn_pool is not None:
+            if self.attn_pool_contrastive is not None:
+                # This is untested, WIP pooling that should match paper
+                x = self.ln_post(x)  # TBD LN first or separate one after each pool?
+                tokens = self.attn_pool(x)
+                if self.attn_pool_type == 'parallel':
+                    pooled = self.attn_pool_contrastive(x)
+                else:
+                    assert self.attn_pool_type == 'cascade'
+                    pooled = self.attn_pool_contrastive(tokens)
+            else:
+                # this is the original OpenCLIP CoCa setup, does not match paper
+                x = self.attn_pool(x)
+                x = self.ln_post(x)
+                pooled, tokens = self._global_pool(x)
+        elif self.final_ln_after_pool:
+            pooled, tokens = self._global_pool(x)
+            pooled = self.ln_post(pooled)
+        else:
+            x = self.ln_post(x)
+            pooled, tokens = self._global_pool(x)
+
+        if self.proj is not None:
+            pooled = pooled @ self.proj
+
+        if self.output_tokens:
+            return pooled, tokens
+
+        return pooled
+
+
+def text_global_pool(x, text: Optional[torch.Tensor] = None, pool_type: str = 'argmax'):
+    if pool_type == 'first':
+        pooled, tokens = x[:, 0], x[:, 1:]
+    elif pool_type == 'last':
+        pooled, tokens = x[:, -1], x[:, :-1]
+    elif pool_type == 'argmax':
+        # take features from the eot embedding (eot_token is the highest number in each sequence)
+        assert text is not None
+        pooled, tokens = x[torch.arange(x.shape[0]), text.argmax(dim=-1)], x
+    else:
+        pooled = tokens = x
+
+    return pooled, tokens
+
+
+class TextTransformer(nn.Module):
+    output_tokens: torch.jit.Final[bool]
+
+    def __init__(
+            self,
+            context_length: int = 77,
+            vocab_size: int = 49408,
+            width: int = 512,
+            heads: int = 8,
+            layers: int = 12,
+            mlp_ratio: float = 4.0,
+            ls_init_value: float = None,
+            output_dim: int = 512,
+            embed_cls: bool = False,
+            no_causal_mask: bool = False,
+            pad_id: int = 0,
+            pool_type: str = 'argmax',
+            proj_bias: bool = False,
+            act_layer: Callable = nn.GELU,
+            norm_layer: Callable = LayerNorm,
+            output_tokens: bool = False,
+    ):
+        super().__init__()
+        assert pool_type in ('first', 'last', 'argmax', 'none')
+        self.output_tokens = output_tokens
+        self.num_pos = self.context_length = context_length
+        self.vocab_size = vocab_size
+        self.width = width
+        self.output_dim = output_dim
+        self.heads = heads
+        self.pad_id = pad_id
+        self.pool_type = pool_type
+
+        self.token_embedding = nn.Embedding(vocab_size, width)
+        if embed_cls:
+            self.cls_emb = nn.Parameter(torch.empty(width))
+            self.num_pos += 1
+        else:
+            self.cls_emb = None
+        self.positional_embedding = nn.Parameter(torch.empty(self.num_pos, width))
+        self.transformer = Transformer(
+            width=width,
+            layers=layers,
+            heads=heads,
+            mlp_ratio=mlp_ratio,
+            ls_init_value=ls_init_value,
+            act_layer=act_layer,
+            norm_layer=norm_layer,
+        )
+        self.ln_final = norm_layer(width)
+
+        if no_causal_mask:
+            self.attn_mask = None
+        else:
+            self.register_buffer('attn_mask', self.build_causal_mask(), persistent=False)
+
+        if proj_bias:
+            self.text_projection = nn.Linear(width, output_dim)
+        else:
+            self.text_projection = nn.Parameter(torch.empty(width, output_dim))
+
+        self.init_parameters()
+
+    def init_parameters(self):
+        nn.init.normal_(self.token_embedding.weight, std=0.02)
+        nn.init.normal_(self.positional_embedding, std=0.01)
+        if self.cls_emb is not None:
+            nn.init.normal_(self.cls_emb, std=0.01)
+
+        proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
+        attn_std = self.transformer.width ** -0.5
+        fc_std = (2 * self.transformer.width) ** -0.5
+        for block in self.transformer.resblocks:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+
+        if self.text_projection is not None:
+            if isinstance(self.text_projection, nn.Linear):
+                nn.init.normal_(self.text_projection.weight, std=self.transformer.width ** -0.5)
+                if self.text_projection.bias is not None:
+                    nn.init.zeros_(self.text_projection.bias)
+            else:
+                nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
+
+    @torch.jit.ignore
+    def set_grad_checkpointing(self, enable=True):
+        self.transformer.grad_checkpointing = enable
+
+    def build_causal_mask(self):
+        # lazily create causal attention mask, with full attention between the tokens
+        # pytorch uses additive attention mask; fill with -inf
+        mask = torch.empty(self.num_pos, self.num_pos)
+        mask.fill_(float("-inf"))
+        mask.triu_(1)  # zero out the lower diagonal
+        return mask
+
+    def build_cls_mask(self, text, cast_dtype: torch.dtype):
+        cls_mask = (text != self.pad_id).unsqueeze(1)
+        cls_mask = F.pad(cls_mask, (1, 0, cls_mask.shape[2], 0), value=True)
+        additive_mask = torch.empty(cls_mask.shape, dtype=cast_dtype, device=cls_mask.device)
+        additive_mask.fill_(0)
+        additive_mask.masked_fill_(~cls_mask, float("-inf"))
+        additive_mask = torch.repeat_interleave(additive_mask, self.heads, 0)
+        return additive_mask
+
+    def forward(self, text):
+        cast_dtype = self.transformer.get_cast_dtype()
+        seq_len = text.shape[1]
+
+        x = self.token_embedding(text).to(cast_dtype)  # [batch_size, n_ctx, d_model]
+        attn_mask = self.attn_mask
+        if self.cls_emb is not None:
+            seq_len += 1
+            x = torch.cat([x, _expand_token(self.cls_emb, x.shape[0])], dim=1)
+            cls_mask = self.build_cls_mask(text, cast_dtype)
+            if attn_mask is not None:
+                attn_mask = attn_mask[None, :seq_len, :seq_len] + cls_mask[:, :seq_len, :seq_len]
+
+        x = x + self.positional_embedding[:seq_len].to(cast_dtype)
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x, attn_mask=attn_mask)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        # x.shape = [batch_size, n_ctx, transformer.width]
+        if self.cls_emb is not None:
+            # presence of appended cls embed (CoCa) overrides pool_type, always take last token
+            pooled, tokens = text_global_pool(x, pool_type='last')
+            pooled = self.ln_final(pooled)  # final LN applied after pooling in this case
+        else:
+            x = self.ln_final(x)
+            pooled, tokens = text_global_pool(x, text, pool_type=self.pool_type)
+
+        if self.text_projection is not None:
+            if isinstance(self.text_projection, nn.Linear):
+                pooled = self.text_projection(pooled)
+            else:
+                pooled = pooled @ self.text_projection
+
+        if self.output_tokens:
+            return pooled, tokens
+
+        return pooled
+
+
+class MultimodalTransformer(Transformer):
+    def __init__(
+            self,
+            width: int,
+            layers: int,
+            heads: int,
+            context_length: int = 77,
+            mlp_ratio: float = 4.0,
+            ls_init_value: float = None,
+            act_layer: Callable = nn.GELU,
+            norm_layer: Callable = LayerNorm,
+            output_dim: int = 512,
+    ):
+
+        super().__init__(
+            width=width,
+            layers=layers,
+            heads=heads,
+            mlp_ratio=mlp_ratio,
+            ls_init_value=ls_init_value,
+            act_layer=act_layer,
+            norm_layer=norm_layer,
+        )
+        self.context_length = context_length
+        self.cross_attn = nn.ModuleList([
+            ResidualAttentionBlock(
+                width,
+                heads,
+                mlp_ratio,
+                ls_init_value=ls_init_value,
+                act_layer=act_layer,
+                norm_layer=norm_layer,
+                is_cross_attention=True,
+            )
+            for _ in range(layers)
+        ])
+
+        self.register_buffer('attn_mask', self.build_attention_mask(), persistent=False)
+
+        self.ln_final = norm_layer(width)
+        self.text_projection = nn.Parameter(torch.empty(width, output_dim))
+
+    def init_parameters(self):
+        proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
+        attn_std = self.transformer.width ** -0.5
+        fc_std = (2 * self.transformer.width) ** -0.5
+        for block in self.transformer.resblocks:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+        for block in self.transformer.cross_attn:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+
+        if self.text_projection is not None:
+            nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
+
+    def build_attention_mask(self):
+        # lazily create causal attention mask, with full attention between the tokens
+        # pytorch uses additive attention mask; fill with -inf
+        mask = torch.empty(self.context_length, self.context_length)
+        mask.fill_(float("-inf"))
+        mask.triu_(1)  # zero out the lower diagonal
+        return mask
+
+    def forward(self, image_embs, text_embs):
+        text_embs = text_embs.permute(1, 0, 2)  # NLD -> LNDsq
+        image_embs = image_embs.permute(1, 0, 2)  # NLD -> LND
+        seq_len = text_embs.shape[0]
+
+        for resblock, cross_attn in zip(self.resblocks, self.cross_attn):
+            if self.grad_checkpointing and not torch.jit.is_scripting():
+                # TODO: handle kwargs https://github.com/pytorch/pytorch/issues/79887#issuecomment-1161758372
+                text_embs = checkpoint(resblock, text_embs, None, None, self.attn_mask[:seq_len, :seq_len])
+                text_embs = checkpoint(cross_attn, text_embs, image_embs, image_embs, None)
+            else:
+                text_embs = resblock(text_embs, attn_mask=self.attn_mask[:seq_len, :seq_len])
+                text_embs = cross_attn(text_embs, k_x=image_embs, v_x=image_embs)
+
+        x = text_embs.permute(1, 0, 2)  # LND -> NLD
+        x = self.ln_final(x)
+
+        if self.text_projection is not None:
+            x = x @ self.text_projection
+
+        return x
+
+    @torch.jit.ignore
+    def set_grad_checkpointing(self, enable=True):
+        self.grad_checkpointing = enable
+
+
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# --------------------------------------------------------
+# Position embedding utils
+# --------------------------------------------------------
+
+import numpy as np
+
+import torch
+
+# --------------------------------------------------------
+# 2D sine-cosine position embedding
+# References:
+# Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py
+# MoCo v3: https://github.com/facebookresearch/moco-v3
+# --------------------------------------------------------
+def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
+    """
+    grid_size: int of the grid height and width
+    return:
+    pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
+    """
+    grid_h = np.arange(grid_size, dtype=np.float32)
+    grid_w = np.arange(grid_size, dtype=np.float32)
+    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
+    grid = np.stack(grid, axis=0)
+
+    grid = grid.reshape([2, 1, grid_size, grid_size])
+    pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
+    if cls_token:
+        pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
+    return pos_embed
+
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+    assert embed_dim % 2 == 0
+
+    # use half of dimensions to encode grid_h
+    emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0])  # (H*W, D/2)
+    emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1])  # (H*W, D/2)
+
+    emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
+    return emb
+
+
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
+    """
+    embed_dim: output dimension for each position
+    pos: a list of positions to be encoded: size (M,)
+    out: (M, D)
+    """
+    assert embed_dim % 2 == 0
+    omega = np.arange(embed_dim // 2, dtype=float)
+    omega /= embed_dim / 2.
+    omega = 1. / 10000**omega  # (D/2,)
+
+    pos = pos.reshape(-1)  # (M,)
+    out = np.einsum('m,d->md', pos, omega)  # (M, D/2), outer product
+
+    emb_sin = np.sin(out) # (M, D/2)
+    emb_cos = np.cos(out) # (M, D/2)
+
+    emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
+    return emb
+
+
+# --------------------------------------------------------
+# Interpolate position embeddings for high-resolution
+# References:
+# DeiT: https://github.com/facebookresearch/deit
+# --------------------------------------------------------
+def interpolate_pos_embed(model, checkpoint_model):
+    if 'pos_embed' in checkpoint_model:
+        pos_embed_checkpoint = checkpoint_model['pos_embed']
+        embedding_size = pos_embed_checkpoint.shape[-1]
+        num_patches = model.patch_embed.num_patches
+        num_extra_tokens = model.pos_embed.shape[-2] - num_patches
+        # height (== width) for the checkpoint position embedding
+        orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5)
+        # height (== width) for the new position embedding
+        new_size = int(num_patches ** 0.5)
+        # class_token and dist_token are kept unchanged
+        if orig_size != new_size:
+            print("Position interpolate from %dx%d to %dx%d" % (orig_size, orig_size, new_size, new_size))
+            extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens]
+            # only the position tokens are interpolated
+            pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:]
+            pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2)
+            pos_tokens = torch.nn.functional.interpolate(
+                pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False)
+            pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2)
+            new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1)
+            checkpoint_model['pos_embed'] = new_pos_embed

config.json

@@ -0,0 +1,118 @@
+{
+  "_name_or_path": "G:\\Temp\\finetune_result\\LLAMA2-7B-CHAT_ViT-L-16-512_MOREKEYWORD_LN_PATCH_FINETUNE_ChexpertJSON_POSTTRAIN_25000_DIST",
+  "architectures": [
+    "CXRLLAVAModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "CXR_LLAVA_HF.CXRLLAVAConfig",
+    "AutoModel": "CXR_LLAVA_HF.CXRLLAVAModel"
+  },
+  "clip_embed_dim": 128,
+  "clip_quick_gelu": false,
+  "clip_vision_cfg": {
+    "image_size": 512,
+    "layers": 24,
+    "patch_size": 16,
+    "width": 1024
+  },
+  "clip_vision_tower_dtype": "bf16",
+  "clip_vision_tower_path": null,
+  "freeze_mm_mlp_adapter": false,
+  "image_aspect_ratio": "square",
+  "image_grid_pinpoints": null,
+  "image_preprocess_cfg": {
+    "mean": 0.5518136078431373,
+    "std": 0.3821719215686275
+  },
+  "llama": {
+    "_name_or_path": "/home/jovyan/llava/SW_LLAVA/LLAMA2-7B-CHAT_ViT-L-16-512_MOREKEYWORD_LN_PATCH_FINETUNE_ChexpertJSON_POSTTRAIN",
+    "add_cross_attention": false,
+    "architectures": [
+      "LlamaForCausalLM"
+    ],
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 11008,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 4096,
+    "min_length": 0,
+    "model_type": "llama",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 32,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 32,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "pretraining_tp": 1,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 10000.0,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "float16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "vocab_size": 32000
+  },
+  "llama_model_dtype": "bf16",
+  "llama_model_path": "/home/jovyan/llava/SW_LLAVA/LLAMA2-7B-CHAT_ViT-L-16-512_MOREKEYWORD_LN_PATCH_FINETUNE_ChexpertJSON_POSTTRAIN",
+  "mm_projector_dim": 1024,
+  "mm_projector_dtype": "fp32",
+  "mm_projector_path": null,
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "model_type": "CXR-LLAVA",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.34.0",
+  "tune_mm_mlp_adapter": false,
+  "use_cache": false
+}

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.34.0"
+}

pytorch_model-00001-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8687c706d0d68518b4636a40eeaafed137946620c164f77473484e86f20c540a
+size 9955046591

pytorch_model-00002-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1179867a0a8cb96669b8c46670bd82928c6597e805982c623652a0bda776b4da
+size 4137901382

pytorch_model.bin.index.json

@@ -0,0 +1,596 @@
+{
+  "metadata": {
+    "total_size": 14092742656
+  },
+  "weight_map": {
+    "llama.embed_tokens.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.10.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.10.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.10.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.11.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.11.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.11.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.12.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.12.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.12.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.13.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.13.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.13.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.13.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.14.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.14.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.14.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.14.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.14.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.14.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.14.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.15.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.15.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.15.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.15.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.15.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.15.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.15.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.16.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.16.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.16.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.16.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.16.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.16.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.16.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.17.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.17.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.17.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.17.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.17.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.17.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.17.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.17.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.18.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.18.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.18.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.18.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.18.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.18.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.18.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.18.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.19.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.19.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.19.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.19.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.19.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.19.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.19.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.20.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.20.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.20.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.20.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.20.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.20.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.20.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.21.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.21.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.21.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.21.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.21.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.21.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.21.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.21.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.21.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.22.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.22.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.22.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.22.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.22.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.22.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.22.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.22.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.22.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.23.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.23.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.23.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.23.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.23.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.23.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.23.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.23.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.23.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.24.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.24.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.24.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.24.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.24.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.24.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.24.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.25.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.25.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.25.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.25.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.25.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.25.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.25.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.26.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.26.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.26.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.26.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.26.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.26.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.26.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.27.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.27.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.27.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.27.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.27.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.27.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.27.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.28.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.28.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.28.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.29.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.29.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.29.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.29.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.29.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.30.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.30.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.30.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.30.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.30.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.30.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.30.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.30.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.30.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.31.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.31.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.31.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.31.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.31.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.31.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.31.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.31.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.31.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "llama.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.layers.9.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "llama.norm.weight": "pytorch_model-00002-of-00002.bin",
+    "lm_head.weight": "pytorch_model-00001-of-00002.bin",
+    "mm_projector.bias": "pytorch_model-00001-of-00002.bin",
+    "mm_projector.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.class_embedding": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.conv1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.ln_post.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.ln_post.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.ln_pre.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.ln_pre.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.positional_embedding": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.proj": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.0.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.1.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.10.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.11.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.12.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.13.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.14.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.15.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.16.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.17.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.18.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.19.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.2.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.20.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.21.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.22.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.23.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.3.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.4.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.5.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.6.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.7.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.8.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.attn.in_proj_bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.attn.in_proj_weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.attn.out_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.attn.out_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.ln_1.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.ln_1.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.ln_2.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.ln_2.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.mlp.c_fc.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.mlp.c_fc.weight": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.mlp.c_proj.bias": "pytorch_model-00001-of-00002.bin",
+    "vision_tower.transformer.resblocks.9.mlp.c_proj.weight": "pytorch_model-00001-of-00002.bin"
+  }
+}