ravi.naik commited on
Commit
667ae00
1 Parent(s): e752318

Fixed relative import issues

Browse files
.gitignore ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py,cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # poetry
98
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102
+ #poetry.lock
103
+
104
+ # pdm
105
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106
+ #pdm.lock
107
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108
+ # in version control.
109
+ # https://pdm.fming.dev/#use-with-ide
110
+ .pdm.toml
111
+
112
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113
+ __pypackages__/
114
+
115
+ # Celery stuff
116
+ celerybeat-schedule
117
+ celerybeat.pid
118
+
119
+ # SageMath parsed files
120
+ *.sage.py
121
+
122
+ # Environments
123
+ .env
124
+ .venv
125
+ env/
126
+ venv/
127
+ ENV/
128
+ env.bak/
129
+ venv.bak/
130
+
131
+ # Spyder project settings
132
+ .spyderproject
133
+ .spyproject
134
+
135
+ # Rope project settings
136
+ .ropeproject
137
+
138
+ # mkdocs documentation
139
+ /site
140
+
141
+ # mypy
142
+ .mypy_cache/
143
+ .dmypy.json
144
+ dmypy.json
145
+
146
+ # Pyre type checker
147
+ .pyre/
148
+
149
+ # pytype static type analyzer
150
+ .pytype/
151
+
152
+ # Cython debug symbols
153
+ cython_debug/
154
+
155
+ # PyCharm
156
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
159
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160
+ #.idea/
README.md CHANGED
@@ -9,5 +9,36 @@ app_file: app.py
9
  pinned: false
10
  license: mit
11
  ---
 
 
 
 
 
 
 
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  pinned: false
10
  license: mit
11
  ---
12
+ ## Phi2 : Multimodal Finetuning
13
+ ### Details
14
+ 1. LLM Backbone: Phi2
15
+ 2. Vision Tower: clip-vit-large-patch14-336
16
+ 3. Audio Model: Whisper
17
+ 4. Pretraining Dataset: LAION-CC-SBU dataset with BLIP captions(200k samples)
18
+ 5. Finetuning Dataset: Instruct 150k dataset based on COCO
19
 
20
+ ### Design
21
+ ![image](https://github.com/RaviNaik/ERA-CAPSTONE/assets/23289802/56df24cd-2681-4e17-ab64-9652f609b15f)
22
+
23
+ ### Pretraining
24
+ #### Training Loss Curve
25
+ ![image](https://github.com/RaviNaik/ERA-CAPSTONE/assets/23289802/b6c37a95-0a56-4b52-8719-3ff56dc1b703)
26
+
27
+ #### Learing Rate
28
+ ![image](https://github.com/RaviNaik/ERA-CAPSTONE/assets/23289802/44d9a11b-b28d-47e1-ba1d-d6dc22ebe748)
29
+
30
+ #### Training Logs
31
+ ![image](https://github.com/RaviNaik/ERA-CAPSTONE/assets/23289802/76543d98-d9fe-4c1a-ac47-3d06e48053ad)
32
+
33
+ ### Finetuning
34
+ #### Training Loss Curve
35
+ ![image](https://github.com/RaviNaik/ERA-CAPSTONE/assets/23289802/45ef40bd-fae5-4cfe-a522-c0eed2833230)
36
+
37
+ #### Learing Rate
38
+ ![image](https://github.com/RaviNaik/ERA-CAPSTONE/assets/23289802/df60ee62-a537-4e36-a7f7-f7111e101162)
39
+
40
+ #### Training Logs
41
+ ![image](https://github.com/RaviNaik/ERA-CAPSTONE/assets/23289802/2747acce-bc99-4c37-a05a-d5e81cb9aa9d)
42
+
43
+ ### Results
44
+ ![image](https://github.com/RaviNaik/ERA-CAPSTONE/assets/23289802/f12a9f04-df32-413e-b957-774c30381b2b)
inference/model/builder.py CHANGED
@@ -1,105 +1,162 @@
1
  import os
2
  import warnings
3
- import shutil
4
 
5
- from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig, CLIPImageProcessor
 
 
 
 
 
 
6
  import torch
7
- from llava_phi.model import *
8
- from llava_phi.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
9
 
 
 
 
 
 
 
10
 
11
- def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="cuda", device="cuda"):
 
 
 
 
 
 
 
 
 
12
  kwargs = {"device_map": device_map}
13
  if load_8bit:
14
- kwargs['load_in_8bit'] = True
15
  elif load_4bit:
16
- kwargs['load_in_4bit'] = True
17
- kwargs['quantization_config'] = BitsAndBytesConfig(
18
  load_in_4bit=True,
19
  bnb_4bit_compute_dtype=torch.float16,
20
  bnb_4bit_use_double_quant=True,
21
- bnb_4bit_quant_type='nf4'
22
  )
23
  # else: # TODO: after fine-tuning LLava-Phi, load the model weights with fp16 will pose nan
24
  # kwargs['torch_dtype'] = torch.float16
25
 
26
- if 'phi' in model_name.lower():
27
  # Load LLaVA-Phi model
28
- if 'lora' in model_name.lower() and model_base is None:
29
- warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument.')
30
- if 'lora' in model_name.lower() and model_base is not None:
 
 
31
  lora_cfg_pretrained = AutoConfig.from_pretrained(model_path)
32
  tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
33
- print('Loading LLaVA-Phi from base model...')
34
- model = LlavaPhiForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs)
 
 
35
  token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features
36
  if model.lm_head.weight.shape[0] != token_num:
37
- model.lm_head.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
38
- model.model.embed_tokens.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
 
 
 
 
 
 
 
 
39
 
40
- print('Loading additional LLaVA-Phi weights...')
41
- if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')):
42
- non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location='cpu')
 
 
 
43
  else:
44
  # this is probably from HF Hub
45
  from huggingface_hub import hf_hub_download
 
46
  def load_from_hf(repo_id, filename, subfolder=None):
47
  cache_file = hf_hub_download(
48
- repo_id=repo_id,
49
- filename=filename,
50
- subfolder=subfolder)
51
- return torch.load(cache_file, map_location='cpu')
52
- non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin')
53
- non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in non_lora_trainables.items()}
54
- if any(k.startswith('model.model.') for k in non_lora_trainables):
55
- non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in non_lora_trainables.items()}
 
 
 
 
 
 
 
 
56
  model.load_state_dict(non_lora_trainables, strict=False)
57
 
58
  from peft import PeftModel
59
- print('Loading LoRA weights...')
 
60
  model = PeftModel.from_pretrained(model, model_path)
61
- print('Merging LoRA weights...')
62
  model = model.merge_and_unload()
63
- print('Model is loaded...')
64
  elif model_base is not None:
65
  # this may be mm projector only
66
- print('Loading LLaVA-Phi from base model...')
67
  tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
68
  cfg_pretrained = AutoConfig.from_pretrained(model_path)
69
- model = LlavaPhiForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
 
 
70
 
71
- mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu')
72
- mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()}
 
 
 
 
73
  model.load_state_dict(mm_projector_weights, strict=False)
74
  else:
75
  print("load llaVA-Phi MLLM!!!")
76
  config = LlavaPhiConfig.from_pretrained(model_path, trust_remote_code=True)
77
  tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)
78
  model = LlavaPhiForCausalLM.from_pretrained(
79
- model_path,
80
- config=config,
81
- use_safetensors=True,
82
- **kwargs).to("cuda")
83
  else:
84
  # Load language model
85
  if model_base is not None:
86
  # PEFT model
87
  from peft import PeftModel
 
88
  tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
89
- model = AutoModelForCausalLM.from_pretrained(model_base, torch_dtype=torch.float16, low_cpu_mem_usage=True, device_map="auto")
 
 
 
 
 
90
  print(f"Loading LoRA weights from {model_path}")
91
  model = PeftModel.from_pretrained(model, model_path)
92
  print(f"Merging weights")
93
  model = model.merge_and_unload()
94
- print('Convert to FP16...')
95
  model.to(torch.float16)
96
  else:
97
  tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
98
- model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
 
 
99
 
100
  image_processor = CLIPImageProcessor.from_pretrained(model_path)
101
 
102
- if 'phi' in model_name.lower():
103
  mm_use_im_start_end = getattr(model.config, "mm_use_im_start_end", False)
104
  mm_use_im_patch_token = getattr(model.config, "mm_use_im_patch_token", True)
105
 
@@ -107,7 +164,9 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
107
  if mm_use_im_patch_token:
108
  tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
109
  if mm_use_im_start_end:
110
- tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
 
 
111
  # model.resize_token_embeddings(len(tokenizer))
112
  else:
113
  raise ValueError(f"Unsupported model name: {model_name}")
 
1
  import os
2
  import warnings
 
3
 
4
+ from transformers import (
5
+ AutoTokenizer,
6
+ AutoModelForCausalLM,
7
+ AutoConfig,
8
+ BitsAndBytesConfig,
9
+ CLIPImageProcessor,
10
+ )
11
  import torch
12
+ from .language_model.llava_phi import LlavaPhiForCausalLM
13
+ from .language_model.configuration_llava_phi import LlavaPhiConfig
14
 
15
+ IGNORE_INDEX = -100
16
+ IMAGE_TOKEN_INDEX = -200
17
+ DEFAULT_IMAGE_TOKEN = "<image>"
18
+ DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
19
+ DEFAULT_IM_START_TOKEN = "<im_start>"
20
+ DEFAULT_IM_END_TOKEN = "<im_end>"
21
 
22
+
23
+ def load_pretrained_model(
24
+ model_path,
25
+ model_base,
26
+ model_name,
27
+ load_8bit=False,
28
+ load_4bit=False,
29
+ device_map="cuda",
30
+ device="cuda",
31
+ ):
32
  kwargs = {"device_map": device_map}
33
  if load_8bit:
34
+ kwargs["load_in_8bit"] = True
35
  elif load_4bit:
36
+ kwargs["load_in_4bit"] = True
37
+ kwargs["quantization_config"] = BitsAndBytesConfig(
38
  load_in_4bit=True,
39
  bnb_4bit_compute_dtype=torch.float16,
40
  bnb_4bit_use_double_quant=True,
41
+ bnb_4bit_quant_type="nf4",
42
  )
43
  # else: # TODO: after fine-tuning LLava-Phi, load the model weights with fp16 will pose nan
44
  # kwargs['torch_dtype'] = torch.float16
45
 
46
+ if "phi" in model_name.lower():
47
  # Load LLaVA-Phi model
48
+ if "lora" in model_name.lower() and model_base is None:
49
+ warnings.warn(
50
+ "There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument."
51
+ )
52
+ if "lora" in model_name.lower() and model_base is not None:
53
  lora_cfg_pretrained = AutoConfig.from_pretrained(model_path)
54
  tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
55
+ print("Loading LLaVA-Phi from base model...")
56
+ model = LlavaPhiForCausalLM.from_pretrained(
57
+ model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs
58
+ )
59
  token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features
60
  if model.lm_head.weight.shape[0] != token_num:
61
+ model.lm_head.weight = torch.nn.Parameter(
62
+ torch.empty(
63
+ token_num, tokem_dim, device=model.device, dtype=model.dtype
64
+ )
65
+ )
66
+ model.model.embed_tokens.weight = torch.nn.Parameter(
67
+ torch.empty(
68
+ token_num, tokem_dim, device=model.device, dtype=model.dtype
69
+ )
70
+ )
71
 
72
+ print("Loading additional LLaVA-Phi weights...")
73
+ if os.path.exists(os.path.join(model_path, "non_lora_trainables.bin")):
74
+ non_lora_trainables = torch.load(
75
+ os.path.join(model_path, "non_lora_trainables.bin"),
76
+ map_location="cpu",
77
+ )
78
  else:
79
  # this is probably from HF Hub
80
  from huggingface_hub import hf_hub_download
81
+
82
  def load_from_hf(repo_id, filename, subfolder=None):
83
  cache_file = hf_hub_download(
84
+ repo_id=repo_id, filename=filename, subfolder=subfolder
85
+ )
86
+ return torch.load(cache_file, map_location="cpu")
87
+
88
+ non_lora_trainables = load_from_hf(
89
+ model_path, "non_lora_trainables.bin"
90
+ )
91
+ non_lora_trainables = {
92
+ (k[11:] if k.startswith("base_model.") else k): v
93
+ for k, v in non_lora_trainables.items()
94
+ }
95
+ if any(k.startswith("model.model.") for k in non_lora_trainables):
96
+ non_lora_trainables = {
97
+ (k[6:] if k.startswith("model.") else k): v
98
+ for k, v in non_lora_trainables.items()
99
+ }
100
  model.load_state_dict(non_lora_trainables, strict=False)
101
 
102
  from peft import PeftModel
103
+
104
+ print("Loading LoRA weights...")
105
  model = PeftModel.from_pretrained(model, model_path)
106
+ print("Merging LoRA weights...")
107
  model = model.merge_and_unload()
108
+ print("Model is loaded...")
109
  elif model_base is not None:
110
  # this may be mm projector only
111
+ print("Loading LLaVA-Phi from base model...")
112
  tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
113
  cfg_pretrained = AutoConfig.from_pretrained(model_path)
114
+ model = LlavaPhiForCausalLM.from_pretrained(
115
+ model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs
116
+ )
117
 
118
+ mm_projector_weights = torch.load(
119
+ os.path.join(model_path, "mm_projector.bin"), map_location="cpu"
120
+ )
121
+ mm_projector_weights = {
122
+ k: v.to(torch.float16) for k, v in mm_projector_weights.items()
123
+ }
124
  model.load_state_dict(mm_projector_weights, strict=False)
125
  else:
126
  print("load llaVA-Phi MLLM!!!")
127
  config = LlavaPhiConfig.from_pretrained(model_path, trust_remote_code=True)
128
  tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)
129
  model = LlavaPhiForCausalLM.from_pretrained(
130
+ model_path, config=config, use_safetensors=True, **kwargs
131
+ ).to("cuda")
 
 
132
  else:
133
  # Load language model
134
  if model_base is not None:
135
  # PEFT model
136
  from peft import PeftModel
137
+
138
  tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
139
+ model = AutoModelForCausalLM.from_pretrained(
140
+ model_base,
141
+ torch_dtype=torch.float16,
142
+ low_cpu_mem_usage=True,
143
+ device_map="auto",
144
+ )
145
  print(f"Loading LoRA weights from {model_path}")
146
  model = PeftModel.from_pretrained(model, model_path)
147
  print(f"Merging weights")
148
  model = model.merge_and_unload()
149
+ print("Convert to FP16...")
150
  model.to(torch.float16)
151
  else:
152
  tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
153
+ model = AutoModelForCausalLM.from_pretrained(
154
+ model_path, low_cpu_mem_usage=True, **kwargs
155
+ )
156
 
157
  image_processor = CLIPImageProcessor.from_pretrained(model_path)
158
 
159
+ if "phi" in model_name.lower():
160
  mm_use_im_start_end = getattr(model.config, "mm_use_im_start_end", False)
161
  mm_use_im_patch_token = getattr(model.config, "mm_use_im_patch_token", True)
162
 
 
164
  if mm_use_im_patch_token:
165
  tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
166
  if mm_use_im_start_end:
167
+ tokenizer.add_tokens(
168
+ [DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True
169
+ )
170
  # model.resize_token_embeddings(len(tokenizer))
171
  else:
172
  raise ValueError(f"Unsupported model name: {model_name}")
inference/model/language_model/configuration_llava_phi.py CHANGED
@@ -68,23 +68,23 @@ class LlavaPhiVisionConfig(PretrainedConfig):
68
  model_type = "llava_phi_clip_vision_model"
69
 
70
  def __init__(
71
- self,
72
- hidden_size=768,
73
- intermediate_size=3072,
74
- projection_dim=512,
75
- num_hidden_layers=12,
76
- num_attention_heads=12,
77
- num_channels=3,
78
- image_size=224,
79
- patch_size=32,
80
- hidden_act="quick_gelu",
81
- layer_norm_eps=1e-5,
82
- attention_dropout=0.0,
83
- initializer_range=0.02,
84
- initializer_factor=1.0,
85
- mm_vision_select_feature="patch",
86
- mm_vision_select_layer=-2,
87
- **kwargs,
88
  ):
89
  super().__init__(**kwargs)
90
 
@@ -105,16 +105,24 @@ class LlavaPhiVisionConfig(PretrainedConfig):
105
  self.mm_vision_select_layer = mm_vision_select_layer
106
 
107
  @classmethod
108
- def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
 
 
109
  cls._set_token_in_kwargs(kwargs)
110
 
111
- config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
 
 
112
 
113
  # get the vision config dict if we are loading from CLIPConfig
114
  if config_dict.get("model_type") == "llava_phi-phi":
115
  config_dict = config_dict["vision_config"]
116
 
117
- if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
 
 
 
 
118
  logger.warning(
119
  f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
120
  f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
@@ -127,11 +135,7 @@ class ProjectorConfig(PretrainedConfig):
127
  model_type = "llava_phi_projector"
128
 
129
  def __init__(
130
- self,
131
- mm_projector_type="linear",
132
- mm_hidden_size=768,
133
- hidden_size=2560,
134
- **kwargs
135
  ):
136
  self.mm_projector_type = mm_projector_type
137
  self.mm_hidden_size = mm_hidden_size
@@ -139,16 +143,24 @@ class ProjectorConfig(PretrainedConfig):
139
  super().__init__(**kwargs)
140
 
141
  @classmethod
142
- def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
 
 
143
  cls._set_token_in_kwargs(kwargs)
144
 
145
- config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
 
 
146
 
147
  # get the vision config dict if we are loading from CLIPConfig
148
  if config_dict.get("model_type") == "llava_phi-phi":
149
  config_dict = config_dict["projector_config"]
150
 
151
- if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
 
 
 
 
152
  logger.warning(
153
  f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
154
  f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
@@ -159,7 +171,7 @@ class ProjectorConfig(PretrainedConfig):
159
 
160
  DEFAULT_VISUAL_CONFIG = {
161
  "vision_tower": LlavaPhiVisionConfig().to_dict(),
162
- "mm_projector": ProjectorConfig().to_dict()
163
  }
164
 
165
 
 
68
  model_type = "llava_phi_clip_vision_model"
69
 
70
  def __init__(
71
+ self,
72
+ hidden_size=768,
73
+ intermediate_size=3072,
74
+ projection_dim=512,
75
+ num_hidden_layers=12,
76
+ num_attention_heads=12,
77
+ num_channels=3,
78
+ image_size=224,
79
+ patch_size=32,
80
+ hidden_act="quick_gelu",
81
+ layer_norm_eps=1e-5,
82
+ attention_dropout=0.0,
83
+ initializer_range=0.02,
84
+ initializer_factor=1.0,
85
+ mm_vision_select_feature="patch",
86
+ mm_vision_select_layer=-2,
87
+ **kwargs,
88
  ):
89
  super().__init__(**kwargs)
90
 
 
105
  self.mm_vision_select_layer = mm_vision_select_layer
106
 
107
  @classmethod
108
+ def from_pretrained(
109
+ cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
110
+ ) -> "PretrainedConfig":
111
  cls._set_token_in_kwargs(kwargs)
112
 
113
+ config_dict, kwargs = cls.get_config_dict(
114
+ pretrained_model_name_or_path, **kwargs
115
+ )
116
 
117
  # get the vision config dict if we are loading from CLIPConfig
118
  if config_dict.get("model_type") == "llava_phi-phi":
119
  config_dict = config_dict["vision_config"]
120
 
121
+ if (
122
+ "model_type" in config_dict
123
+ and hasattr(cls, "model_type")
124
+ and config_dict["model_type"] != cls.model_type
125
+ ):
126
  logger.warning(
127
  f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
128
  f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
 
135
  model_type = "llava_phi_projector"
136
 
137
  def __init__(
138
+ self, mm_projector_type="linear", mm_hidden_size=768, hidden_size=2560, **kwargs
 
 
 
 
139
  ):
140
  self.mm_projector_type = mm_projector_type
141
  self.mm_hidden_size = mm_hidden_size
 
143
  super().__init__(**kwargs)
144
 
145
  @classmethod
146
+ def from_pretrained(
147
+ cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
148
+ ) -> "PretrainedConfig":
149
  cls._set_token_in_kwargs(kwargs)
150
 
151
+ config_dict, kwargs = cls.get_config_dict(
152
+ pretrained_model_name_or_path, **kwargs
153
+ )
154
 
155
  # get the vision config dict if we are loading from CLIPConfig
156
  if config_dict.get("model_type") == "llava_phi-phi":
157
  config_dict = config_dict["projector_config"]
158
 
159
+ if (
160
+ "model_type" in config_dict
161
+ and hasattr(cls, "model_type")
162
+ and config_dict["model_type"] != cls.model_type
163
+ ):
164
  logger.warning(
165
  f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
166
  f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
 
171
 
172
  DEFAULT_VISUAL_CONFIG = {
173
  "vision_tower": LlavaPhiVisionConfig().to_dict(),
174
+ "mm_projector": ProjectorConfig().to_dict(),
175
  }
176
 
177
 
inference/model/llava_arch.py CHANGED
@@ -19,8 +19,19 @@ import torch
19
 
20
  from .multimodal_encoder.clip_encoder import CLIPVisionTower
21
  from .multimodal_projector.builder import build_vision_projector
22
- from .language_model.configuration_llava_phi import LlavaPhiConfig, LlavaPhiVisionConfig, ProjectorConfig
23
- from llava_phi.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
 
 
 
 
 
 
 
 
 
 
 
24
 
25
 
26
  class LlavaMetaModel:
@@ -34,14 +45,13 @@ class LlavaMetaModel:
34
  )
35
 
36
  def get_vision_tower(self):
37
- vision_tower = getattr(self, 'vision_tower', None)
38
  if type(vision_tower) is list:
39
  vision_tower = vision_tower[0]
40
  return vision_tower
41
 
42
 
43
  class LlavaMetaForCausalLM(ABC):
44
-
45
  @abstractmethod
46
  def get_model(self):
47
  pass
@@ -59,8 +69,17 @@ class LlavaMetaForCausalLM(ABC):
59
  ):
60
  vision_tower = self.get_vision_tower()
61
  if vision_tower is None or images is None or input_ids.shape[1] == 1:
62
- if past_key_values is not None and vision_tower is not None and images is not None and input_ids.shape[1] == 1:
63
- attention_mask = torch.ones((attention_mask.shape[0], past_key_values[-1][-1].shape[-2] + 1), dtype=attention_mask.dtype, device=attention_mask.device)
 
 
 
 
 
 
 
 
 
64
  return input_ids, attention_mask, past_key_values, None, labels
65
 
66
  if type(images) is list or images.ndim == 5:
@@ -81,9 +100,16 @@ class LlavaMetaForCausalLM(ABC):
81
  # FIXME: this is a hacky fix, for deepspeed zero3 to work
82
  half_len = cur_input_ids.shape[0] // 2
83
  cur_image_features = image_features[cur_image_idx]
84
- cur_input_embeds_1 = self.get_model().embed_tokens(cur_input_ids[:half_len])
85
- cur_input_embeds_2 = self.get_model().embed_tokens(cur_input_ids[half_len:])
86
- cur_input_embeds = torch.cat([cur_input_embeds_1, cur_image_features[0:0], cur_input_embeds_2], dim=0)
 
 
 
 
 
 
 
87
  new_input_embeds.append(cur_input_embeds)
88
  if labels is not None:
89
  new_labels.append(labels[batch_idx])
@@ -98,37 +124,79 @@ class LlavaMetaForCausalLM(ABC):
98
  while image_token_indices.numel() > 0:
99
  cur_image_features = image_features[cur_image_idx]
100
  image_token_start = image_token_indices[0]
101
- if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
102
- cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[:image_token_start-1]).detach())
103
- cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[image_token_start-1:image_token_start]))
 
 
 
 
 
 
 
 
 
 
104
  cur_new_input_embeds.append(cur_image_features)
105
- cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[image_token_start+1:image_token_start+2]))
 
 
 
 
106
  if labels is not None:
107
  cur_new_labels.append(cur_labels[:image_token_start])
108
- cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device, dtype=labels.dtype))
109
- cur_new_labels.append(cur_labels[image_token_start:image_token_start+1])
110
- cur_labels = cur_labels[image_token_start+2:]
 
 
 
 
 
 
 
 
 
111
  else:
112
- cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[:image_token_start]))
 
 
113
  cur_new_input_embeds.append(cur_image_features)
114
  if labels is not None:
115
  cur_new_labels.append(cur_labels[:image_token_start])
116
- cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device, dtype=labels.dtype))
117
- cur_labels = cur_labels[image_token_start+1:]
 
 
 
 
 
 
 
118
  cur_image_idx += 1
119
- if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
120
- cur_input_ids = cur_input_ids[image_token_start+2:]
 
 
121
  else:
122
- cur_input_ids = cur_input_ids[image_token_start+1:]
123
  image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
124
  if cur_input_ids.numel() > 0:
125
- if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
126
- cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids).detach())
 
 
 
 
127
  else:
128
- cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids))
 
 
129
  if labels is not None:
130
  cur_new_labels.append(cur_labels)
131
- cur_new_input_embeds = [x.to(device=self.device) for x in cur_new_input_embeds]
 
 
132
  cur_new_input_embeds = torch.cat(cur_new_input_embeds, dim=0)
133
  new_input_embeds.append(cur_new_input_embeds)
134
  if labels is not None:
@@ -140,7 +208,17 @@ class LlavaMetaForCausalLM(ABC):
140
 
141
  new_input_embeds_align = []
142
  for cur_new_embed in new_input_embeds:
143
- cur_new_embed = torch.cat((cur_new_embed, torch.zeros((max_len - cur_new_embed.shape[0], cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device)), dim=0)
 
 
 
 
 
 
 
 
 
 
144
  new_input_embeds_align.append(cur_new_embed)
145
  new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
146
 
@@ -148,27 +226,67 @@ class LlavaMetaForCausalLM(ABC):
148
  new_labels_align = []
149
  _new_labels = new_labels
150
  for cur_new_label in new_labels:
151
- cur_new_label = torch.cat((cur_new_label, torch.full((max_len - cur_new_label.shape[0],), IGNORE_INDEX, dtype=cur_new_label.dtype, device=cur_new_label.device)), dim=0)
 
 
 
 
 
 
 
 
 
 
 
152
  new_labels_align.append(cur_new_label)
153
  new_labels = torch.stack(new_labels_align, dim=0)
154
 
155
  if attention_mask is not None:
156
  new_attention_mask = []
157
- for cur_attention_mask, cur_new_labels, cur_new_labels_align in zip(attention_mask, _new_labels, new_labels):
158
- new_attn_mask_pad_left = torch.full((cur_new_labels.shape[0] - labels.shape[1],), True, dtype=attention_mask.dtype, device=attention_mask.device)
159
- new_attn_mask_pad_right = torch.full((cur_new_labels_align.shape[0] - cur_new_labels.shape[0],), False, dtype=attention_mask.dtype, device=attention_mask.device)
160
- cur_new_attention_mask = torch.cat((new_attn_mask_pad_left, cur_attention_mask, new_attn_mask_pad_right), dim=0)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
161
  new_attention_mask.append(cur_new_attention_mask)
162
  attention_mask = torch.stack(new_attention_mask, dim=0)
163
  assert attention_mask.shape == new_labels.shape
164
  else:
165
  new_input_embeds = torch.stack(new_input_embeds, dim=0)
166
  if labels is not None:
167
- new_labels = torch.stack(new_labels, dim=0)
168
 
169
  if attention_mask is not None:
170
- new_attn_mask_pad_left = torch.full((attention_mask.shape[0], new_input_embeds.shape[1] - input_ids.shape[1]), True, dtype=attention_mask.dtype, device=attention_mask.device)
171
- attention_mask = torch.cat((new_attn_mask_pad_left, attention_mask), dim=1)
 
 
 
 
 
 
 
 
 
 
172
  assert attention_mask.shape == new_input_embeds.shape[:2]
173
 
174
  return None, attention_mask, past_key_values, new_input_embeds, new_labels
@@ -179,7 +297,9 @@ class LlavaMetaForCausalLM(ABC):
179
  self.resize_token_embeddings(len(tokenizer))
180
 
181
  if model_args.mm_use_im_start_end:
182
- num_new_tokens = tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
 
 
183
  self.resize_token_embeddings(len(tokenizer))
184
 
185
  if num_new_tokens > 0:
@@ -187,9 +307,11 @@ class LlavaMetaForCausalLM(ABC):
187
  output_embeddings = self.get_output_embeddings().weight.data
188
 
189
  input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
190
- dim=0, keepdim=True)
 
191
  output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
192
- dim=0, keepdim=True)
 
193
 
194
  input_embeddings[-num_new_tokens:] = input_embeddings_avg
195
  output_embeddings[-num_new_tokens:] = output_embeddings_avg
@@ -199,7 +321,7 @@ class LlavaMetaForCausalLM(ABC):
199
  p.requires_grad = True
200
  for p in self.get_output_embeddings().parameters():
201
  p.requires_grad = False
202
-
203
  elif model_args.mm_use_im_patch_token:
204
  if model_args.tune_mm_mlp_adapter:
205
  for p in self.get_input_embeddings().parameters():
 
19
 
20
  from .multimodal_encoder.clip_encoder import CLIPVisionTower
21
  from .multimodal_projector.builder import build_vision_projector
22
+ from .language_model.configuration_llava_phi import (
23
+ LlavaPhiConfig,
24
+ LlavaPhiVisionConfig,
25
+ ProjectorConfig,
26
+ )
27
+
28
+ # from llava_phi.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
29
+ IGNORE_INDEX = -100
30
+ IMAGE_TOKEN_INDEX = -200
31
+ DEFAULT_IMAGE_TOKEN = "<image>"
32
+ DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
33
+ DEFAULT_IM_START_TOKEN = "<im_start>"
34
+ DEFAULT_IM_END_TOKEN = "<im_end>"
35
 
36
 
37
  class LlavaMetaModel:
 
45
  )
46
 
47
  def get_vision_tower(self):
48
+ vision_tower = getattr(self, "vision_tower", None)
49
  if type(vision_tower) is list:
50
  vision_tower = vision_tower[0]
51
  return vision_tower
52
 
53
 
54
  class LlavaMetaForCausalLM(ABC):
 
55
  @abstractmethod
56
  def get_model(self):
57
  pass
 
69
  ):
70
  vision_tower = self.get_vision_tower()
71
  if vision_tower is None or images is None or input_ids.shape[1] == 1:
72
+ if (
73
+ past_key_values is not None
74
+ and vision_tower is not None
75
+ and images is not None
76
+ and input_ids.shape[1] == 1
77
+ ):
78
+ attention_mask = torch.ones(
79
+ (attention_mask.shape[0], past_key_values[-1][-1].shape[-2] + 1),
80
+ dtype=attention_mask.dtype,
81
+ device=attention_mask.device,
82
+ )
83
  return input_ids, attention_mask, past_key_values, None, labels
84
 
85
  if type(images) is list or images.ndim == 5:
 
100
  # FIXME: this is a hacky fix, for deepspeed zero3 to work
101
  half_len = cur_input_ids.shape[0] // 2
102
  cur_image_features = image_features[cur_image_idx]
103
+ cur_input_embeds_1 = self.get_model().embed_tokens(
104
+ cur_input_ids[:half_len]
105
+ )
106
+ cur_input_embeds_2 = self.get_model().embed_tokens(
107
+ cur_input_ids[half_len:]
108
+ )
109
+ cur_input_embeds = torch.cat(
110
+ [cur_input_embeds_1, cur_image_features[0:0], cur_input_embeds_2],
111
+ dim=0,
112
+ )
113
  new_input_embeds.append(cur_input_embeds)
114
  if labels is not None:
115
  new_labels.append(labels[batch_idx])
 
124
  while image_token_indices.numel() > 0:
125
  cur_image_features = image_features[cur_image_idx]
126
  image_token_start = image_token_indices[0]
127
+ if getattr(self.config, "tune_mm_mlp_adapter", False) and getattr(
128
+ self.config, "mm_use_im_start_end", False
129
+ ):
130
+ cur_new_input_embeds.append(
131
+ self.get_model()
132
+ .embed_tokens(cur_input_ids[: image_token_start - 1])
133
+ .detach()
134
+ )
135
+ cur_new_input_embeds.append(
136
+ self.get_model().embed_tokens(
137
+ cur_input_ids[image_token_start - 1 : image_token_start]
138
+ )
139
+ )
140
  cur_new_input_embeds.append(cur_image_features)
141
+ cur_new_input_embeds.append(
142
+ self.get_model().embed_tokens(
143
+ cur_input_ids[image_token_start + 1 : image_token_start + 2]
144
+ )
145
+ )
146
  if labels is not None:
147
  cur_new_labels.append(cur_labels[:image_token_start])
148
+ cur_new_labels.append(
149
+ torch.full(
150
+ (cur_image_features.shape[0],),
151
+ IGNORE_INDEX,
152
+ device=labels.device,
153
+ dtype=labels.dtype,
154
+ )
155
+ )
156
+ cur_new_labels.append(
157
+ cur_labels[image_token_start : image_token_start + 1]
158
+ )
159
+ cur_labels = cur_labels[image_token_start + 2 :]
160
  else:
161
+ cur_new_input_embeds.append(
162
+ self.get_model().embed_tokens(cur_input_ids[:image_token_start])
163
+ )
164
  cur_new_input_embeds.append(cur_image_features)
165
  if labels is not None:
166
  cur_new_labels.append(cur_labels[:image_token_start])
167
+ cur_new_labels.append(
168
+ torch.full(
169
+ (cur_image_features.shape[0],),
170
+ IGNORE_INDEX,
171
+ device=labels.device,
172
+ dtype=labels.dtype,
173
+ )
174
+ )
175
+ cur_labels = cur_labels[image_token_start + 1 :]
176
  cur_image_idx += 1
177
+ if getattr(self.config, "tune_mm_mlp_adapter", False) and getattr(
178
+ self.config, "mm_use_im_start_end", False
179
+ ):
180
+ cur_input_ids = cur_input_ids[image_token_start + 2 :]
181
  else:
182
+ cur_input_ids = cur_input_ids[image_token_start + 1 :]
183
  image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
184
  if cur_input_ids.numel() > 0:
185
+ if getattr(self.config, "tune_mm_mlp_adapter", False) and getattr(
186
+ self.config, "mm_use_im_start_end", False
187
+ ):
188
+ cur_new_input_embeds.append(
189
+ self.get_model().embed_tokens(cur_input_ids).detach()
190
+ )
191
  else:
192
+ cur_new_input_embeds.append(
193
+ self.get_model().embed_tokens(cur_input_ids)
194
+ )
195
  if labels is not None:
196
  cur_new_labels.append(cur_labels)
197
+ cur_new_input_embeds = [
198
+ x.to(device=self.device) for x in cur_new_input_embeds
199
+ ]
200
  cur_new_input_embeds = torch.cat(cur_new_input_embeds, dim=0)
201
  new_input_embeds.append(cur_new_input_embeds)
202
  if labels is not None:
 
208
 
209
  new_input_embeds_align = []
210
  for cur_new_embed in new_input_embeds:
211
+ cur_new_embed = torch.cat(
212
+ (
213
+ cur_new_embed,
214
+ torch.zeros(
215
+ (max_len - cur_new_embed.shape[0], cur_new_embed.shape[1]),
216
+ dtype=cur_new_embed.dtype,
217
+ device=cur_new_embed.device,
218
+ ),
219
+ ),
220
+ dim=0,
221
+ )
222
  new_input_embeds_align.append(cur_new_embed)
223
  new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
224
 
 
226
  new_labels_align = []
227
  _new_labels = new_labels
228
  for cur_new_label in new_labels:
229
+ cur_new_label = torch.cat(
230
+ (
231
+ cur_new_label,
232
+ torch.full(
233
+ (max_len - cur_new_label.shape[0],),
234
+ IGNORE_INDEX,
235
+ dtype=cur_new_label.dtype,
236
+ device=cur_new_label.device,
237
+ ),
238
+ ),
239
+ dim=0,
240
+ )
241
  new_labels_align.append(cur_new_label)
242
  new_labels = torch.stack(new_labels_align, dim=0)
243
 
244
  if attention_mask is not None:
245
  new_attention_mask = []
246
+ for cur_attention_mask, cur_new_labels, cur_new_labels_align in zip(
247
+ attention_mask, _new_labels, new_labels
248
+ ):
249
+ new_attn_mask_pad_left = torch.full(
250
+ (cur_new_labels.shape[0] - labels.shape[1],),
251
+ True,
252
+ dtype=attention_mask.dtype,
253
+ device=attention_mask.device,
254
+ )
255
+ new_attn_mask_pad_right = torch.full(
256
+ (cur_new_labels_align.shape[0] - cur_new_labels.shape[0],),
257
+ False,
258
+ dtype=attention_mask.dtype,
259
+ device=attention_mask.device,
260
+ )
261
+ cur_new_attention_mask = torch.cat(
262
+ (
263
+ new_attn_mask_pad_left,
264
+ cur_attention_mask,
265
+ new_attn_mask_pad_right,
266
+ ),
267
+ dim=0,
268
+ )
269
  new_attention_mask.append(cur_new_attention_mask)
270
  attention_mask = torch.stack(new_attention_mask, dim=0)
271
  assert attention_mask.shape == new_labels.shape
272
  else:
273
  new_input_embeds = torch.stack(new_input_embeds, dim=0)
274
  if labels is not None:
275
+ new_labels = torch.stack(new_labels, dim=0)
276
 
277
  if attention_mask is not None:
278
+ new_attn_mask_pad_left = torch.full(
279
+ (
280
+ attention_mask.shape[0],
281
+ new_input_embeds.shape[1] - input_ids.shape[1],
282
+ ),
283
+ True,
284
+ dtype=attention_mask.dtype,
285
+ device=attention_mask.device,
286
+ )
287
+ attention_mask = torch.cat(
288
+ (new_attn_mask_pad_left, attention_mask), dim=1
289
+ )
290
  assert attention_mask.shape == new_input_embeds.shape[:2]
291
 
292
  return None, attention_mask, past_key_values, new_input_embeds, new_labels
 
297
  self.resize_token_embeddings(len(tokenizer))
298
 
299
  if model_args.mm_use_im_start_end:
300
+ num_new_tokens = tokenizer.add_tokens(
301
+ [DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True
302
+ )
303
  self.resize_token_embeddings(len(tokenizer))
304
 
305
  if num_new_tokens > 0:
 
307
  output_embeddings = self.get_output_embeddings().weight.data
308
 
309
  input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
310
+ dim=0, keepdim=True
311
+ )
312
  output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
313
+ dim=0, keepdim=True
314
+ )
315
 
316
  input_embeddings[-num_new_tokens:] = input_embeddings_avg
317
  output_embeddings[-num_new_tokens:] = output_embeddings_avg
 
321
  p.requires_grad = True
322
  for p in self.get_output_embeddings().parameters():
323
  p.requires_grad = False
324
+
325
  elif model_args.mm_use_im_patch_token:
326
  if model_args.tune_mm_mlp_adapter:
327
  for p in self.get_input_embeddings().parameters():
inference/model/multimodal_encoder/clip_encoder.py CHANGED
@@ -5,7 +5,7 @@ import torch.nn as nn
5
 
6
  from transformers import CLIPPreTrainedModel, CLIPVisionConfig
7
  from transformers.models.clip.modeling_clip import CLIPVisionTransformer
8
- from llava_phi.model.language_model.configuration_llava_phi import LlavaPhiVisionConfig
9
 
10
 
11
  class CLIPVisionTower(CLIPPreTrainedModel):
 
5
 
6
  from transformers import CLIPPreTrainedModel, CLIPVisionConfig
7
  from transformers.models.clip.modeling_clip import CLIPVisionTransformer
8
+ from inference.model.language_model.configuration_llava_phi import LlavaPhiVisionConfig
9
 
10
 
11
  class CLIPVisionTower(CLIPPreTrainedModel):
requirements.txt CHANGED
@@ -6,7 +6,6 @@ gradio_client==0.2.9
6
  markdown2[all]
7
  numpy
8
  requests
9
- sentencepiece
10
  tokenizers==0.15.0
11
  torch==2.0.1
12
  shortuuid
 
6
  markdown2[all]
7
  numpy
8
  requests
 
9
  tokenizers==0.15.0
10
  torch==2.0.1
11
  shortuuid