FIRST: add model weight

config.json

@@ -0,0 +1,250 @@
+{
+  "architectures": [
+    "ZiyaBLIP2ForConditionalGeneration"
+  ],
+  "assistant_name": "<bot>",
+  "human_name": "<human>",
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "model_type": "blip-2",
+  "num_query_tokens": 32,
+  "prompt_prefix": "",
+  "qformer_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_probs_dropout_prob": 0.1,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "classifier_dropout": null,
+    "cross_attention_frequency": 2,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_hidden_size": 1408,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-12,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 512,
+    "min_length": 0,
+    "model_type": "blip_2_qformer",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 0,
+    "position_embedding_type": "absolute",
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.29.0.dev0",
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "vocab_size": 30522
+  },
+  "text_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": [
+      "LlamaForCausalLM"
+    ],
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 5120,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 13824,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 2048,
+    "min_length": 0,
+    "model_type": "llama",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 40,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 40,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 0,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "float32",
+    "torchscript": false,
+    "transformers_version": "4.29.0.dev0",
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "vocab_size": 39424
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": null,
+  "use_decoder_only_language_model": true,
+  "vision_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0.0,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_size": 1408,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 224,
+    "initializer_factor": 1.0,
+    "initializer_range": 1e-10,
+    "intermediate_size": 6144,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "blip_2_vision_model",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 39,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "projection_dim": 512,
+    "pruned_heads": {},
+    "qkv_bias": true,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.29.0.dev0",
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  }
+}

demo.py

@@ -0,0 +1,38 @@
+from transformers import LlamaForCausalLM, LlamaTokenizer, BlipImageProcessor
+from modeling_ziya_blip2 import ZiyaBLIP2ForConditionalGeneration
+from PIL import Image
+
+# model path of IDEA-CCNL/Ziya-LLaMA-13B-v1
+LM_MODEL_PATH="local path of model IDEA-CCNL/Ziya-LLaMA-13B-v1"
+LM_MODEL_PATH="/cognitive_comp/gaoxinyu/huggingface_model/Ziya-LLaMA-13B-v1"
+lm_model = LlamaForCausalLM.from_pretrained(LM_MODEL_PATH)
+tokenizer = LlamaTokenizer.from_pretrained(LM_MODEL_PATH)
+
+
+# visual model
+OPENAI_CLIP_MEAN = [0.48145466, 0.4578275, 0.40821073]
+OPENAI_CLIP_STD = [0.26862954, 0.26130258, 0.27577711]
+# demo.py is in the project path, so we can use local path ".". Otherwise you should use "IDEA-CCNL/Ziya-BLIP2-14B-Visual-v1"
+model = ZiyaBLIP2ForConditionalGeneration.from_pretrained(".", language_model=lm_model)
+image_size = model.config.vision_config.image_size
+image_processor = BlipImageProcessor(
+    size={"height": image_size, "width": image_size},
+    image_mean=OPENAI_CLIP_MEAN,
+    image_std=OPENAI_CLIP_STD,
+)
+model.cuda() # if you use on cpu, comment this line
+generate_config = {
+    "max_new_tokens": 128,
+    "top_p": 0.1,
+    "temperature": 0.7
+}
+output = model.chat(
+    tokenizer=tokenizer,
+    pixel_values=image_processor(Image.open("wzry.jpg"), return_tensors="pt").pixel_values.to(model.device),
+    query="这是什么游戏",
+    previous_querys=[],
+    previous_outputs=[],
+    **generate_config,
+    )
+print(output)
+# 这是一款名为《王者荣耀》的多人在线竞技游戏。在游戏中，玩家扮演不同的角色，并与其他玩家进行战斗。游戏中的人物和环境都是虚拟的，但它们看起来非常逼真。玩家需要使用各种技能和策略来击败对手，并获得胜利。

modeling_ziya_blip2.py

@@ -0,0 +1,287 @@
+from typing import Optional, Tuple, Union, List
+
+import torch
+import torch.utils.checkpoint
+from torch import nn
+from transformers.utils import (
+    logging,
+)
+from transformers.models.blip_2.configuration_blip_2 import Blip2Config
+from transformers.models.blip_2.modeling_blip_2 import Blip2ForConditionalGenerationModelOutput
+from transformers import (
+    Blip2PreTrainedModel,
+    Blip2VisionModel,
+    AutoModelForCausalLM,
+    Blip2QFormerModel,
+    PreTrainedTokenizer,
+    PreTrainedModel,
+)
+
+
+logger = logging.get_logger(__name__)
+
+
+class ZiyaBLIP2ForConditionalGeneration(Blip2PreTrainedModel):
+    config_class = Blip2Config
+    main_input_name = "pixel_values"
+    _keys_to_ignore_on_load_missing = [
+        r"language_model",
+    ]
+    def __init__(self, config: Blip2Config, language_model: PreTrainedModel = None):
+        super().__init__(config)
+
+        self.vision_model = Blip2VisionModel(config.vision_config)
+
+        self.query_tokens = nn.Parameter(torch.zeros(
+            1, config.num_query_tokens, config.qformer_config.hidden_size))
+        self.qformer = Blip2QFormerModel(config.qformer_config)
+
+        self.language_projection = nn.Linear(
+            config.qformer_config.hidden_size, config.text_config.hidden_size)
+        if language_model is None:
+            if config.use_decoder_only_language_model:
+                language_model = AutoModelForCausalLM.from_config(config.text_config)
+            else:
+                raise Exception("not impl")
+        self.language_model = language_model
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def get_output_embeddings(self) -> nn.Module:
+        return self.language_model.get_output_embeddings()
+
+    def get_encoder(self):
+        return self.language_model.get_encoder()
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
+    def _tie_weights(self):
+        if not self.config.use_decoder_only_language_model:
+            self.language_model.encoder.embed_tokens = self.language_model.shared
+            self.language_model.decoder.embed_tokens = self.language_model.shared
+
+    def _preprocess_accelerate(self):
+        r"""
+        Some pre-processing hacks to make the model `accelerate` compatible. Check
+        https://github.com/huggingface/transformers/pull/21707 for more details.
+        """
+        hf_device_map = self.hf_device_map
+
+        if len(hf_device_map) > 1 and "language_model" not in hf_device_map and torch.cuda.device_count() > 1:
+            # warn users about unexpected behavior when using multi-GPU + BLIP-2 + `accelerate`.
+            logger.warning(
+                "The `language_model` is not in the `hf_device_map` dictionary and you are running your script"
+                " in a multi-GPU environment. this may lead to unexpected behavior when using `accelerate`."
+                " Please pass a `device_map` that contains `language_model` to remove this warning."
+                " Please refer to https://github.com/huggingface/blog/blob/main/accelerate-large-models.md for",
+                " more details on creating a `device_map` for large models.",
+            )
+
+        if hasattr(self.language_model, "_hf_hook"):
+            self.language_model._hf_hook.io_same_device = True  # For `generate` compatibility
+
+    def forward(
+        self,
+        pixel_values: torch.FloatTensor,
+        input_ids_before_image: torch.FloatTensor,
+        input_ids_after_image: torch.FloatTensor,
+        labels_after_image: torch.FloatTensor,
+        # 因为label不会出现在image之前，所以这里不需要labels_before_image， 按照input_ids_before_image补-100就可以了
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, Blip2ForConditionalGenerationModelOutput]:
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # step 1: forward the images through the vision encoder,
+        # to get image embeddings of shape (batch_size, seq_len, hidden_size)
+        vision_outputs = self.vision_model(
+            pixel_values=pixel_values,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        image_embeds = vision_outputs[0]
+
+        # step 2: forward the query tokens through the QFormer, using the image embeddings for cross-attention
+        image_attention_mask = torch.ones(
+            image_embeds.size()[:-1], dtype=torch.long, device=image_embeds.device)
+
+        query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, -1)
+        query_outputs = self.qformer(
+            query_embeds=query_tokens,
+            encoder_hidden_states=image_embeds,
+            encoder_attention_mask=image_attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        query_output = query_outputs[0]
+
+        # step 2.5 generate the lm input by prompt and output
+        language_model_inputs = self.language_projection(query_output)
+        language_model_attention_mask = torch.ones(
+            language_model_inputs.size()[:-1], dtype=torch.long, device=language_model_inputs.device
+        )
+        # 确保language_model_inputs的batch
+        assert language_model_inputs.shape[0] == input_ids_after_image.shape[0]
+        inputs_embeds_before_image = self.language_model.get_input_embeddings()(input_ids_before_image)
+        inputs_embeds_after_image = self.language_model.get_input_embeddings()(input_ids_after_image)
+        inputs_embeds = torch.cat(
+            [
+                inputs_embeds_before_image.to(language_model_inputs.device),
+                language_model_inputs,
+                inputs_embeds_after_image.to(language_model_inputs.device)
+            ], dim=1)
+
+        attention_mask_before = torch.ones_like(input_ids_before_image)
+        attention_mask_after = torch.ones_like(input_ids_after_image)
+        attention_mask = torch.cat(
+            [
+                attention_mask_before.to(language_model_attention_mask.device),
+                language_model_attention_mask,
+                attention_mask_after.to(language_model_attention_mask.device)
+            ], dim=1
+        )
+        # labels也需要对应的处理，把前面空缺的-100加进去
+        labels = torch.cat(
+            [
+                torch.tensor(
+                    [-100]).expand_as(input_ids_before_image).to(language_model_inputs.device),
+                torch.tensor([-100]).expand(query_tokens.shape[:-1]
+                                            ).to(language_model_inputs.device),
+                labels_after_image,
+            ], dim=1
+        )
+
+        # step 3: use the language model
+
+        if self.config.use_decoder_only_language_model:
+            outputs = self.language_model(
+                inputs_embeds=inputs_embeds,
+                attention_mask=attention_mask,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+                labels=labels,
+            )
+            loss = outputs.loss if return_dict else outputs[0]
+            logits = outputs.logits if return_dict else outputs[1]
+
+        else:
+            raise Exception("not impl")
+
+        if not return_dict:
+            output = (logits, vision_outputs, query_outputs, outputs)
+            return ((loss,) + output) if loss is not None else output
+
+        return Blip2ForConditionalGenerationModelOutput(
+            loss=loss,
+            logits=logits,
+            vision_outputs=vision_outputs,
+            qformer_outputs=query_outputs,
+            language_model_outputs=outputs,
+        )
+
+    def prepare_inputs_for_chat(
+        self,
+        tokenizer: PreTrainedTokenizer,
+        query: str,
+        pixel_values: torch.Tensor,
+        previous_querys: List[str],
+        previous_outputs: List[str],
+        max_length: int,
+    ):
+        # 1. process input_ids
+        assert len(previous_querys) == len(previous_outputs)
+        device = self.device
+        prefix = self.config.prompt_prefix
+        human_name = self.config.human_name
+        assistant_name = self.config.assistant_name
+        input_ids_before_image = tokenizer(
+            prefix, return_tensors="pt").input_ids.to(device)
+        inputs_ids_after_image = []
+        for (p, o) in zip(previous_querys, previous_outputs):
+            # {pormpt}\n[答]: {output}\n[问]:
+            inputs_ids_after_image += tokenizer(f"{human_name}: {p}\n", add_special_tokens=False).input_ids + \
+                tokenizer(f"{assistant_name}: {o}\n", add_special_tokens=False).input_ids
+
+        inputs_ids_after_image += tokenizer(f"{human_name}: {query}\n",
+                                            add_special_tokens=False).input_ids + tokenizer(f"{assistant_name} :",
+                                            add_special_tokens=False).input_ids
+        inputs_ids_after_image = torch.IntTensor([inputs_ids_after_image]).to(device)
+        # 2. Prepare embeddings
+        pixel_values.to(device)
+        image_embeds = self.vision_model(pixel_values, return_dict=True).last_hidden_state
+        image_attention_mask = torch.ones(
+            image_embeds.size()[:-1], dtype=torch.long, device=image_embeds.device)
+        query_tokens = self.query_tokens.expand(image_embeds.shape[0], -1, -1)
+        query_outputs = self.qformer(
+            query_embeds=query_tokens,
+            encoder_hidden_states=image_embeds,
+            encoder_attention_mask=image_attention_mask,
+            return_dict=True,
+        )
+        query_output = query_outputs.last_hidden_state
+        language_model_inputs = self.language_projection(query_output)
+
+        # concatenate query embeddings with prompt embeddings
+        prefix_inputs_embeds = self.get_input_embeddings()(input_ids_before_image)
+        prompt_inputs_embeds = self.get_input_embeddings()(inputs_ids_after_image)
+        inputs_embeds = torch.cat([
+            prefix_inputs_embeds.to(language_model_inputs.device),
+            language_model_inputs,
+            prompt_inputs_embeds.to(language_model_inputs.device)], dim=1)
+
+        if inputs_embeds.shape[1] > max_length:
+            inputs_embeds = inputs_embeds[:, -max_length:, :]
+
+        input_ids = torch.concat([
+            input_ids_before_image,
+            torch.tensor([tokenizer.eos_token_id]).expand(
+                query_tokens.shape[:-1]).to(language_model_inputs.device),
+            inputs_ids_after_image,
+        ], dim=1)
+
+        return input_ids, inputs_embeds
+
+    def chat(self,
+             tokenizer,
+             query: str,
+             pixel_values: torch.Tensor,
+             previous_querys: List[str],
+             previous_outputs: List[str],
+             **generate_kwargs,):
+        """
+        use for generate text by chat-style
+        Args:
+            tokenizer (PretrainedTokenizer): llama tokenizer
+            query (str): current input query
+            pixel_values (torch.Tensor): image after image_processor
+            prompts (List[str]): chat history
+            outputs (List[str]): chat history
+
+        Returns:
+            text: generate text
+        """
+        input_ids, inputs_embeds = self.prepare_inputs_for_chat(
+            tokenizer, query, pixel_values, previous_querys, previous_outputs, 2048
+        )
+        response = self.language_model.generate(
+            inputs_embeds=inputs_embeds,
+            attention_mask=torch.ones_like(input_ids),
+            **generate_kwargs,
+        )
+        response = tokenizer.decode(response[0], skip_special_tokens=True)
+        return response

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cc4682fb5bf8adee4967316f1421b1782d2389c3ac671c448313c925d1eddc4
+size 4380450257