from huggingface_hub import cached_download, hf_hub_url from PIL import Image import os import gradio as gr import spaces import torch from torch import nn from torch.utils.data import Dataset, DataLoader from torchvision import transforms from transformers import CLIPTextModel, CLIPTokenizer, CLIPVisionModel, CLIPModel @spaces.GPU() def train_image_generation_model(image_folder, text_folder, model_name="image_generation_model"): """Trains an image generation model on the provided dataset. Args: image_folder (str): Path to the folder containing training images. text_folder (str): Path to the folder containing text prompts for each image. model_name (str, optional): Name for the saved model file. Defaults to "image_generation_model". Returns: str: Path to the saved model file. """ class ImageTextDataset(Dataset): def __init__(self, image_folder, text_folder, transform=None): self.image_paths = [os.path.join(image_folder, f) for f in os.listdir(image_folder) if f.lower().endswith(('.png', '.jpg', '.jpeg'))] self.text_paths = [os.path.join(text_folder, f) for f in os.listdir(text_folder) if f.lower().endswith('.txt')] self.transform = transform def __len__(self): return len(self.image_paths) def __getitem__(self, idx): image = Image.open(self.image_paths[idx]).convert("RGB") if self.transform: image = self.transform(image) with open(self.text_paths[idx], 'r') as f: text = f.read().strip() return image, text # Load CLIP model clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-base-patch32") # Define image and text transformations transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711]) ]) # Create dataset and dataloader dataset = ImageTextDataset(image_folder, text_folder, transform=transform) dataloader = DataLoader(dataset, batch_size=8, shuffle=True) # Define optimizer and loss function optimizer = torch.optim.Adam(clip_model.parameters(), lr=1e-5) loss_fn = nn.CrossEntropyLoss() # Train the model for epoch in range(10): for i, (images, texts) in enumerate(dataloader): optimizer.zero_grad() image_features = clip_model.get_image_features(images) text_features = clip_model.get_text_features(tokenizer(texts, return_tensors="pt")["input_ids"]) similarity = image_features @ text_features.T loss = loss_fn(similarity, torch.arange(images.size(0), device=images.device)) loss.backward() optimizer.step() print(f"Epoch: {epoch} | Iteration: {i} | Loss: {loss.item()}") # Save the trained model model_path = os.path.join(os.getcwd(), model_name + ".pt") torch.save(clip_model.state_dict(), model_path) return model_path # Define Gradio interface iface = gr.Interface( fn=train_image_generation_model, inputs=[ gr.File(label="Image Folder", file_count="directory"), gr.File(label="Text Prompts Folder", file_count="directory"), gr.Textbox(label="Model Name"), ], outputs=gr.File(label="Model File"), title="Image Generation Model Trainer", description="Upload a folder of images and their corresponding text prompts to train a model.\n Images foler should contain image files. Prompts folder should contain .txt files. Each text file is prompt for each image in images folder.", ) iface.launch(share=True)