| | from transformers import PretrainedConfig |
| | from transformers import CONFIG_MAPPING |
| | from transformers import AutoConfig |
| | from transformers.models.phi.configuration_phi import PhiConfig |
| |
|
| | IGNORE_INDEX = -100 |
| | IMAGE_TOKEN_INDEX = -200 |
| | DEFAULT_IMAGE_TOKEN = "<image>" |
| |
|
| |
|
| | class TinyLlavaConfig(PretrainedConfig): |
| |
|
| | model_type = "tinyllava" |
| | def __init__( |
| | self, |
| | llm_model_name_or_path = '', |
| | tokenizer_name_or_path = None, |
| | vision_model_name_or_path = '', |
| | vision_model_name_or_path2 = '', |
| | connector_type = None, |
| | text_config=None, |
| | hidden_size=2048, |
| | vocab_size=32000, |
| | ignore_index=-100, |
| | image_token_index=32000, |
| | pad_token = None, |
| | pad_token_id = None, |
| | tokenizer_padding_side = 'right', |
| | tokenizer_model_max_length = 2048, |
| | vision_config = None, |
| | vision_hidden_size = None, |
| | vision_feature_layer = -2, |
| | vision_feature_select_strategy = 'patch', |
| | image_aspect_ratio = 'square', |
| | resampler_hidden_size = None, |
| | num_queries = None, |
| | num_resampler_layers = None, |
| | use_cache = False, |
| | cache_dir = None, |
| | tokenizer_use_fast = False, |
| | tune_type_llm = 'frozen', |
| | tune_type_connector = 'frozen', |
| | tune_type_vision_tower = 'frozen', |
| | tune_vision_tower_from_layer = -1, |
| | |
| | **kwargs |
| | |
| | ): |
| | self.llm_model_name_or_path = llm_model_name_or_path |
| | self.tokenizer_name_or_path = tokenizer_name_or_path or self.llm_model_name_or_path |
| | self.vision_model_name_or_path = vision_model_name_or_path |
| | self.vision_model_name_or_path2 = vision_model_name_or_path2 |
| | self.connector_type = connector_type |
| | self.tune_type_llm = tune_type_llm |
| | self.tune_type_connector = tune_type_connector |
| | self.tune_type_vision_tower = tune_type_vision_tower |
| | self.tune_vision_tower_from_layer = tune_vision_tower_from_layer |
| | |
| | self.ignore_index = IGNORE_INDEX |
| | self.image_token_index = IMAGE_TOKEN_INDEX |
| | self.pad_token = pad_token |
| | self.pad_token_id = pad_token_id |
| | self.tokenizer_padding_side = tokenizer_padding_side |
| | self.tokenizer_model_max_length = tokenizer_model_max_length |
| | self.vision_feature_layer = vision_feature_layer |
| | self.vision_feature_select_strategy = vision_feature_select_strategy |
| | self.image_aspect_ratio = image_aspect_ratio |
| | self.resampler_hidden_size = resampler_hidden_size |
| | self.num_queries = num_queries |
| | self.num_resampler_layers = num_resampler_layers |
| | self.use_cache = use_cache |
| | self.cache_dir = cache_dir |
| | self.tokenizer_use_fast = tokenizer_use_fast |
| | self._load_text_config(text_config) |
| | self._load_vision_config(vision_config) |
| | |
| | super().__init__(**kwargs) |
| | |
| | |
| | def _load_text_config(self, text_config=None): |
| | if self.llm_model_name_or_path is None or self.llm_model_name_or_path == '': |
| | self.text_config = CONFIG_MAPPING['llama']() |
| | |
| | else: |
| | self.text_config = PhiConfig.from_pretrained(self.llm_model_name_or_path, trust_remote_code=True) |
| | if text_config is not None: |
| | self.text_config = self.text_config.from_dict(text_config) |
| | |
| | self.hidden_size = getattr(self.text_config, 'hidden_size', getattr(self.text_config, 'model_dim', None)) |
| | self.vocab_size = getattr(self.text_config, 'vocab_size', None) |
| | |
| | |
| | |
| | def _load_vision_config(self, vision_config=None): |
| | if self.vision_model_name_or_path is None or self.vision_model_name_or_path == '': |
| | self.vision_config = CONFIG_MAPPING['clip_vision_model']( |
| | intermediate_size=4096, |
| | hidden_size=1024, |
| | patch_size=14, |
| | image_size=336, |
| | num_hidden_layers=24, |
| | num_attention_heads=16, |
| | vocab_size=32000, |
| | projection_dim=768, |
| | ) |
| | |
| | else: |
| | self.vision_config = AutoConfig.from_pretrained(self.vision_model_name_or_path.split(':')[-1]) |
| | self.vision_config = getattr(self.vision_config, 'vision_config', self.vision_config) |
| | if vision_config is not None: |
| | self.vision_config = self.vision_config.from_dict(vision_config) |
| | |
| | self.vision_config.model_name_or_path = self.vision_model_name_or_path.split(':')[-1] |
| | self.vision_config.model_name_or_path2 = self.vision_model_name_or_path2.split(':')[-1] |
| | self.vision_hidden_size = getattr(self.vision_config, 'hidden_size', None) |
