Code: import torch
from torch import nn
from torch.utils.data import DataLoader
from transformers import GPT2Tokenizer, GPT2LMHeadModel, CLIPProcessor, CLIPModel, AdamW
from datasets import load_dataset
from PIL import Image

class MultimodalGPT2(nn.Module):
    def __init__(self, gpt2_model_name, clip_model_name):
        super(MultimodalGPT2, self).__init__()
        self.clip = CLIPModel.from_pretrained(clip_model_name)
        self.gpt2 = GPT2LMHeadModel.from_pretrained(gpt2_model_name)
        self.adapter = nn.Linear(self.clip.config.projection_dim, self.gpt2.config.n_embd)
    def forward(self, input_ids, attention_mask, image=None):
        if image is not None:
            image_features = self.clip.get_image_features(pixel_values=image)
            adapter_features = self.adapter(image_features)
            adapter_features = adapter_features.unsqueeze(1).expand(-1, input_ids.size(1), -1)
            inputs_embeds = self.gpt2.transformer.wte(input_ids) + adapter_features
            outputs = self.gpt2(inputs_embeds=inputs_embeds, attention_mask=attention_mask, labels=input_ids)
        else:
            outputs = self.gpt2(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
        return outputs
def preprocess_data(example, processor, tokenizer):
    images = example.get('images', [])
    if images:
        image = images[0]
        if isinstance(image, Image.Image):
            image = image.convert('RGB')
        else:
            image = Image.open(image).convert('RGB')
        image = processor(images=image, return_tensors="pt")['pixel_values'].squeeze()
    else:
        image = None
    text = (example.get('user', '') + " " + example.get('assistant', '')).strip()
    encoding = tokenizer(text, truncation=True, padding='max_length', max_length=128)
    return {
        'input_ids': encoding['input_ids'],
        'attention_mask': encoding['attention_mask'],
        'image': image
    }
def collate_fn(batch):
    input_ids = torch.tensor([item['input_ids'] for item in batch])
    attention_mask = torch.tensor([item['attention_mask'] for item in batch])
    images = torch.stack([item['image'] for item in batch if item['image'] is not None]) if any(item['image'] is not None for item in batch) else None
    return {'input_ids': input_ids, 'attention_mask': attention_mask, 'image': images}

def main():
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    dataset = load_dataset('HuggingFaceM4/the_cauldron', 'ai2d', split='train')
    tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
    tokenizer.pad_token = tokenizer.eos_token
    processor = CLIPProcessor.from_pretrained('openai/clip-vit-base-patch32')
    processed_dataset = dataset.map(lambda x: preprocess_data(x, processor, tokenizer), remove_columns=dataset.column_names)
    dataloader = DataLoader(processed_dataset, batch_size=8, shuffle=True, collate_fn=collate_fn)
    model = MultimodalGPT2('gpt2', 'openai/clip-vit-base-patch32').to(device)
    optimizer = AdamW(model.parameters(), lr=5e-5)
    model.train()
    for epoch in range(1):
        for batch in dataloader:
            input_ids = batch['input_ids'].to(device)
            attention_mask = batch['attention_mask'].to(device)
            images = batch['image'].to(device) if batch['image'] is not None else None
            outputs = model(input_ids=input_ids, attention_mask=attention_mask, image=images)
            loss = outputs.loss
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()
            print(loss.item())
    torch.save(model.state_dict(), 'multimodal_gpt2.pth')
    model.eval()
    def generate(prompt_text=None, image_path=None):
        if prompt_text is not None:
            input_ids = tokenizer.encode(prompt_text, return_tensors='pt').to(device)
            attention_mask = torch.ones_like(input_ids).to(device)
        else:
            input_ids = torch.tensor([[tokenizer.eos_token_id]]).to(device)
            attention_mask = torch.ones_like(input_ids).to(device)
        if image_path is not None:
            image = Image.open(image_path).convert('RGB')
            image = processor(images=image, return_tensors="pt")['pixel_values'].to(device)
        else:
            image = None
        with torch.no_grad():
            if image is not None:
                image_features = model.clip.get_image_features(pixel_values=image)
                adapter_features = model.adapter(image_features).unsqueeze(1)
                inputs_embeds = model.gpt2.transformer.wte(input_ids) + adapter_features
                outputs = model.gpt2.generate(inputs_embeds=inputs_embeds, attention_mask=attention_mask, max_length=50)
            else:
                outputs = model.gpt2.generate(input_ids=input_ids, attention_mask=attention_mask, max_length=50)
        return tokenizer.decode(outputs[0], skip_special_tokens=True)
    print(generate(prompt_text="Question: What do respiration and combustion give out? Choices: A. Oxygen B. Carbon dioxide C. Nitrogen D. Heat Answer with the letter."))
    print(generate(image_path='example.jpg'))
    print(generate(prompt_text="Question: What do respiration and combustion give out? Choices: A. Oxygen B. Carbon dioxide C. Nitrogen D. Heat Answer with the letter.", image_path='example.jpg'))

if __name__ == "__main__":
    main() Error: C:\Users\Dell\AI\pythonProject\.venv\Scripts\python.exe C:\Users\Dell\AI\pythonProject\!EridAI\VLM\main.py 
2024-10-06 17:14:43.910743: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-10-06 17:14:46.198925: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
Map: 100%|██████████| 2434/2434 [00:55<00:00, 44.02 examples/s]
C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\huggingface_hub\file_download.py:157: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\Users\Dell\.cache\huggingface\hub\models--openai--clip-vit-base-patch32. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.
To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development
  warnings.warn(message)
C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\transformers\optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  warnings.warn(
Traceback (most recent call last):
  File "C:\Users\Dell\AI\pythonProject\!EridAI\VLM\main.py", line 98, in <module>
    main()
  File "C:\Users\Dell\AI\pythonProject\!EridAI\VLM\main.py", line 60, in main
    for batch in dataloader:
                 ^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\torch\utils\data\dataloader.py", line 631, in __next__
    data = self._next_data()
           ^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\torch\utils\data\dataloader.py", line 675, in _next_data
    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\torch\utils\data\_utils\fetch.py", line 54, in fetch
    return self.collate_fn(data)
           ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\!EridAI\VLM\main.py", line 45, in collate_fn
    images = torch.stack([item['image'] for item in batch if item['image'] is not None]) if any(item['image'] is not None for item in batch) else None
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: expected Tensor as element 0 in argument 0, but got list

Process finished with exit code 1