Error: C:\Users\Dell\AI\pythonProject\.venv\Scripts\python.exe C:\Users\Dell\AI\pythonProject\!EridAI\VLM\main.py 
Downloading readme: 100%|██████████| 50.3k/50.3k [00:00<00:00, 443kB/s]
Downloading data: 100%|██████████| 438M/438M [00:55<00:00, 7.94MB/s]
Generating train split: 100%|██████████| 2434/2434 [00:00<00:00, 4442.11 examples/s]
2024-10-06 17:11:02.043847: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2024-10-06 17:11:02.718711: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\huggingface_hub\file_download.py:157: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\Users\Dell\.cache\huggingface\hub\models--openai--clip-vit-base-patch32. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.
To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development
  warnings.warn(message)
Map:   0%|          | 0/2434 [00:00<?, ? examples/s]
Traceback (most recent call last):
  File "C:\Users\Dell\AI\pythonProject\!EridAI\VLM\main.py", line 92, in <module>
    main()
  File "C:\Users\Dell\AI\pythonProject\!EridAI\VLM\main.py", line 48, in main
    processed_dataset = dataset.map(lambda x: preprocess_data(x, processor, tokenizer), remove_columns=dataset.column_names)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\datasets\arrow_dataset.py", line 602, in wrapper
    out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
                                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\datasets\arrow_dataset.py", line 567, in wrapper
    out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
                                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\datasets\arrow_dataset.py", line 3161, in map
    for rank, done, content in Dataset._map_single(**dataset_kwargs):
  File "C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\datasets\arrow_dataset.py", line 3522, in _map_single
    example = apply_function_on_filtered_inputs(example, i, offset=offset)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\datasets\arrow_dataset.py", line 3421, in apply_function_on_filtered_inputs
    processed_inputs = function(*fn_args, *additional_args, **fn_kwargs)
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\!EridAI\VLM\main.py", line 48, in <lambda>
    processed_dataset = dataset.map(lambda x: preprocess_data(x, processor, tokenizer), remove_columns=dataset.column_names)
                                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\!EridAI\VLM\main.py", line 28, in preprocess_data
    image = Image.open(images[0]).convert('RGB')
            ^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\Dell\AI\pythonProject\.venv\Lib\site-packages\PIL\Image.py", line 3286, in open
    prefix = fp.read(16)
             ^^^^^^^
AttributeError: 'PngImageFile' object has no attribute 'read'

Process finished with exit code 1
 Code: import torch
from torch import nn
from torch.utils.data import DataLoader
from transformers import GPT2Tokenizer, GPT2LMHeadModel, CLIPProcessor, CLIPModel, AdamW
from datasets import load_dataset
from PIL import Image

class MultimodalGPT2(nn.Module):
    def __init__(self, gpt2_model_name, clip_model_name):
        super(MultimodalGPT2, self).__init__()
        self.clip = CLIPModel.from_pretrained(clip_model_name)
        self.gpt2 = GPT2LMHeadModel.from_pretrained(gpt2_model_name)
        self.adapter = nn.Linear(self.clip.config.projection_dim, self.gpt2.config.n_embd)
    def forward(self, input_ids, attention_mask, image=None):
        if image is not None:
            image_features = self.clip.get_image_features(pixel_values=image)
            adapter_features = self.adapter(image_features)
            adapter_features = adapter_features.unsqueeze(1).expand(-1, input_ids.size(1), -1)
            inputs_embeds = self.gpt2.transformer.wte(input_ids) + adapter_features
            outputs = self.gpt2(inputs_embeds=inputs_embeds, attention_mask=attention_mask, labels=input_ids)
        else:
            outputs = self.gpt2(input_ids=input_ids, attention_mask=attention_mask, labels=input_ids)
        return outputs

def preprocess_data(example, processor, tokenizer):
    images = example.get('images', [])
    if images:
        image = Image.open(images[0]).convert('RGB')
        image = processor(images=image, return_tensors="pt")['pixel_values'].squeeze()
    else:
        image = None
    text = example['user'] + " " + example['assistant']
    encoding = tokenizer(text, truncation=True, padding='max_length', max_length=128)
    return {'input_ids': encoding['input_ids'], 'attention_mask': encoding['attention_mask'], 'image': image}

def collate_fn(batch):
    input_ids = torch.tensor([item['input_ids'] for item in batch])
    attention_mask = torch.tensor([item['attention_mask'] for item in batch])
    images = torch.stack([item['image'] for item in batch if item['image'] is not None]) if any(item['image'] is not None for item in batch) else None
    return {'input_ids': input_ids, 'attention_mask': attention_mask, 'image': images}

def main():
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    dataset = load_dataset('HuggingFaceM4/the_cauldron', 'ai2d', split='train')
    tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
    tokenizer.pad_token = tokenizer.eos_token
    processor = CLIPProcessor.from_pretrained('openai/clip-vit-base-patch32')
    processed_dataset = dataset.map(lambda x: preprocess_data(x, processor, tokenizer), remove_columns=dataset.column_names)
    dataloader = DataLoader(processed_dataset, batch_size=8, shuffle=True, collate_fn=collate_fn)
    model = MultimodalGPT2('gpt2', 'openai/clip-vit-base-patch32').to(device)
    optimizer = AdamW(model.parameters(), lr=5e-5)
    model.train()
    for epoch in range(1):
        for batch in dataloader:
            input_ids = batch['input_ids'].to(device)
            attention_mask = batch['attention_mask'].to(device)
            images = batch['image'].to(device) if batch['image'] is not None else None
            outputs = model(input_ids=input_ids, attention_mask=attention_mask, image=images)
            loss = outputs.loss
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()
            print(loss.item())
    torch.save(model.state_dict(), 'multimodal_gpt2.pth')
    model.eval()
    def generate(prompt_text=None, image_path=None):
        if prompt_text is not None:
            input_ids = tokenizer.encode(prompt_text, return_tensors='pt').to(device)
            attention_mask = torch.ones_like(input_ids).to(device)
        else:
            input_ids = torch.tensor([[tokenizer.eos_token_id]]).to(device)
            attention_mask = torch.ones_like(input_ids).to(device)
        if image_path is not None:
            image = Image.open(image_path).convert('RGB')
            image = processor(images=image, return_tensors="pt")['pixel_values'].to(device)
        else:
            image = None
        with torch.no_grad():
            if image is not None:
                image_features = model.clip.get_image_features(pixel_values=image)
                adapter_features = model.adapter(image_features).unsqueeze(1)
                inputs_embeds = model.gpt2.transformer.wte(input_ids) + adapter_features
                outputs = model.gpt2.generate(inputs_embeds=inputs_embeds, attention_mask=attention_mask, max_length=50)
            else:
                outputs = model.gpt2.generate(input_ids=input_ids, attention_mask=attention_mask, max_length=50)
        return tokenizer.decode(outputs[0], skip_special_tokens=True)
    print(generate(prompt_text="Question: What do respiration and combustion give out? Choices: A. Oxygen B. Carbon dioxide C. Nitrogen D. Heat Answer with the letter."))
    print(generate(image_path='example.jpg'))
    print(generate(prompt_text="Question: What do respiration and combustion give out? Choices: A. Oxygen B. Carbon dioxide C. Nitrogen D. Heat Answer with the letter.", image_path='example.jpg'))

if __name__ == "__main__":
    main()