from dataclasses import dataclass from transformers import GPT2Config, CLIPVisionConfig PREFIX_MAP = { "openai/clip-vit-base-patch32": 50, "openai/clip-vit-base-patch16": 197, "openai/clip-vit-large-patch14": 257, "openai/clip-vit-large-patch14-336": 577 } TEXT_HIDDEN_SIZE_MAP = { "gpt2": 768, "gpt2-medium": 768, "gpt2-large": 1280, "gpt2-xl": 1600 } IMAGE_HIDDEN_SIZE_MAP = { "openai/clip-vit-base-patch32": 768, "openai/clip-vit-base-patch16": 768, "openai/clip-vit-large-patch14": 768, "openai/clip-vit-large-patch14-336": 768 } @dataclass class CLIPGPT2Config: image_model: str = "openai/clip-vit-base-patch32" freeze_image_model: bool = True text_model: str = "gpt2-large" freeze_text_model: bool = True linear_mapping_type: int = "linear" add_image_token: bool = True freeze_ln: bool = False image_from_pretrained: bool = True text_from_pretrained: bool = True def __post_init__(self): self.prefix_length = PREFIX_MAP[self.image_model] self.image_hidden_size = IMAGE_HIDDEN_SIZE_MAP[self.image_model] self.text_hidden_size = TEXT_HIDDEN_SIZE_MAP[self.text_model] self.image_resize = 224 if "336" not in self.image_model else 336 self.text_config = GPT2Config.from_pretrained(self.text_model) self.image_config = CLIPVisionConfig.from_pretrained(self.image_model) self.vocab_size = self.text_config.vocab_size + self.add_image_token