Update for `transformers` changes (#5)

- update (8ecee597dccd0f8ee2e0c4e057109f85ad7b4b1e)
- add video index (c2ef27c0dcfaf3b06114e31c0f7bdeae6695e2a1)
- add pipeline (3f553d44d832eb790714955f131f7c76f2288098)


Co-authored-by: Raushan Turganbay <RaushanTurganbay@users.noreply.huggingface.co>

README.md

@@ -4,7 +4,7 @@ license: other
 tags:
 - vision
 - image-captioning
-pipeline_tag: image-to-text
+pipeline_tag: image-text-to-text
 ---
 
 # InstructBLIP model

added_tokens.json

@@ -1,3 +1,5 @@
 {
+  "<image>": 32001,
+  "<video>": 32002,
   "[PAD]": 32000
 }

config.json

@@ -1,244 +1,49 @@
 {
-  "_commit_hash": null,
   "architectures": [
     "InstructBlipForConditionalGeneration"
   ],
+  "image_token_index": 32001,
   "initializer_factor": 1.0,
   "initializer_range": 0.02,
   "model_type": "instructblip",
   "num_query_tokens": 32,
   "qformer_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
-    "architectures": null,
-    "attention_probs_dropout_prob": 0.1,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_frequency": 2,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_hidden_size": 1408,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "gelu",
-    "hidden_dropout_prob": 0.1,
-    "hidden_size": 768,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "initializer_range": 0.02,
-    "intermediate_size": 3072,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-12,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "max_position_embeddings": 512,
-    "min_length": 0,
     "model_type": "instructblip_qformer",
-    "no_repeat_ngram_size": 0,
-    "num_attention_heads": 12,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_hidden_layers": 12,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": 0,
-    "position_embedding_type": "absolute",
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": null,
-    "torchscript": false,
-    "transformers_version": "4.31.0.dev0",
-    "typical_p": 1.0,
-    "use_bfloat16": false,
     "vocab_size": 30523
   },
   "text_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
     "architectures": [
       "LLaMAForCausalLM"
     ],
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
+    "attention_bias": false,
     "bos_token_id": 0,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
     "eos_token_id": 1,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
+    "head_dim": 128,
     "hidden_act": "silu",
     "hidden_size": 5120,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
     "initializer_range": 0.02,
     "intermediate_size": 13824,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "max_position_embeddings": 2048,
     "max_sequence_length": 2048,
-    "min_length": 0,
+    "mlp_bias": false,
     "model_type": "llama",
-    "no_repeat_ngram_size": 0,
     "num_attention_heads": 40,
-    "num_beam_groups": 1,
-    "num_beams": 1,
     "num_hidden_layers": 40,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
+    "num_key_value_heads": 40,
     "pad_token_id": 32000,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
+    "pretraining_tp": 1,
     "rms_norm_eps": 1e-06,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
+    "rope_scaling": null,
+    "rope_theta": 10000.0,
     "tie_word_embeddings": false,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
     "torch_dtype": "float16",
-    "torchscript": false,
-    "transformers_version": "4.31.0.dev0",
-    "typical_p": 1.0,
-    "use_bfloat16": false,
-    "use_cache": true,
-    "vocab_size": 32001
+    "vocab_size": 32064
   },
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": null,
+  "transformers_version": "4.47.0.dev0",
   "use_decoder_only_language_model": true,
+  "video_token_index": 32002,
   "vision_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
-    "architectures": null,
-    "attention_dropout": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "gelu",
-    "hidden_size": 1408,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "image_size": 224,
-    "initializer_range": 1e-10,
-    "intermediate_size": 6144,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-06,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "min_length": 0,
-    "model_type": "instructblip_vision_model",
-    "no_repeat_ngram_size": 0,
-    "num_attention_heads": 16,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_hidden_layers": 39,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "patch_size": 14,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "qkv_bias": true,
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": null,
-    "torchscript": false,
-    "transformers_version": "4.31.0.dev0",
-    "typical_p": 1.0,
-    "use_bfloat16": false
+    "model_type": "instructblip_vision_model"
   }
 }

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 32000,
+  "transformers_version": "4.47.0.dev0"
+}

model-00001-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8166f3a131c58dda927671938c790a2d16495faf773898078f1e342c40e5849
-size 9866787200
+oid sha256:428b0d5ca487f34d774946392802451e88691722fd942e73b1b6f0938de71cba
+size 9868071712

model-00002-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08516cf3146b38bea03c9bbe94ddb82c345f824a00158170c1568285c41119bf
-size 9867440176
+oid sha256:87f01b7cee567a2b709c7d7d543fdaafcf2c9fa62a7f3483dc11d7df8cef3304
+size 9867437080

model-00003-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa8a29582cbba77a936b4720c7d776f12525093cd904dddc7b47fcf46e4f184d
-size 9940840032
+oid sha256:b0db3116a9d7772293d490edcda023fde18d5d05c23e986c7cb5b90f4c97a7ff
+size 9940837328

model-00004-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1c41db027b117f89d9523ece7e0a35c3c4618956ef0f0a76cd53c934f3bdc55
-size 9940840424
+oid sha256:1cbda65d83efac99725d0b9711ef266d56bfa71ad60dbbca4bcd3fd1aa4901dc
+size 9940837328

model-00005-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f58877f99071d92dd0041ca8991db7364614d53caed5bc18fbda3fd9cd9d8b3d
-size 9867399016
+oid sha256:9b770fa3b52491d50b123fefba46f03495e4055b93e018f1c55bac3bf06646a3
+size 9867395920

model-00006-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac626e0c7ea09681acb1ba513a629ef5250daafe5660a1d9a99172f0a0c47850
-size 7282655488
+oid sha256:78e6f106ac84f3dfb2ce0c1efc59fe8436ab9d85dcd306c8d0e46e7b8839dcda
+size 7283943792

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "num_query_tokens": 32,
+  "processor_class": "InstructBlipProcessor"
+}

qformer_tokenizer/special_tokens_map.json

@@ -1,8 +1,44 @@
 {
-  "bos_token": "[DEC]",
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
+  "bos_token": {
+    "content": "[DEC]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

qformer_tokenizer/tokenizer_config.json

@@ -1,7 +1,59 @@
 {
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30522": {
+      "content": "[DEC]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[DEC]",
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
   "do_lower_case": true,
+  "extra_special_tokens": {},
   "mask_token": "[MASK]",
   "model_max_length": 512,
   "pad_token": "[PAD]",

special_tokens_map.json

@@ -1,5 +1,11 @@
 {
-  "bos_token": "</s>",
+  "bos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
   "eos_token": {
     "content": "</s>",
     "lstrip": false,
@@ -7,6 +13,18 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "[PAD]",
-  "unk_token": "</s>"
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -1,19 +1,68 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<video>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
   "bos_token": "</s>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": {
-    "__type": "AddedToken",
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
   "model_max_length": 2048,
-  "pad_token": null,
+  "pad_token": "[PAD]",
   "processor_class": "InstructBlipProcessor",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "truncation_side": "left",
-  "unk_token": "</s>"
+  "unk_token": "</s>",
+  "use_default_system_prompt": false
 }