utils.py

import glob
import numpy as np
import pickle
import torch
from torch import nn
from torch.utils.data import DataLoader, Dataset
import torchvision.transforms as T
from torchvision.utils import make_grid


# default image directory (training only)
img_path = '/home/alan/Projects/gen_dnd_art/filtered_images/im128/*pkl'
img_files = glob.glob(img_path)

# determine class names from image directory (training only)
'''
labels = np.array([i.split('/')[-1].split('_')[:3] for i in img_files])
species = np.unique(labels[:, 0]).tolist()
classes = np.unique(labels[:, 1]).tolist()
genders = np.unique(labels[:, 2]).tolist()
'''

# hard code class labels (for application)
species = ['dragonborn', 'dwarf', 'elf', 'gnome', 'halfling', 'human', 'orc', 'tiefling']
classes = [
    'barbarian', 'bard', 'cleric', 'druid', 'fighter', 'monk',
    'paladin','ranger', 'rogue', 'sorcerer', 'warlock', 'wizard'
]
genders = ['', 'female', 'male']


class ImSet(Dataset):
    def __init__(self, img_path=img_path):
        super().__init__()
        self.img_files = glob.glob(img_path)
        self.transform = T.Compose([
            T.ToTensor(),
            T.ColorJitter(0.1, 0.1, 0.1, 0.1),
            T.RandomHorizontalFlip(),
            # add random noise and clip
            lambda x: torch.clip(torch.randn(x.shape) / 20 + x, 0, 1),
            T.Normalize(0.5, 0.5)
        ])
    
    def __len__(self):
        return len(self.img_files)
    
    def __getitem__(self, i):
        img_file = self.img_files[i]
        
        # load image
        with open(img_file, 'rb') as fid:
            img = pickle.load(fid)
        
        # apply transforms
        img = self.transform(img).float()
        
        # extract class label
        img_fname = img_file.split('/')[-1]
        species_, class_, gender_, _, _ = img_fname.split('_')
        species_ = species.index(species_)
        class_ = classes.index(class_)
        gender_ = genders.index(gender_)
        
        return (img_fname, img, species_, class_, gender_)

class VariationalEncoder(nn.Module):
    def __init__(self, input_channels=3, latent_size=2048):
        super().__init__()
        
        self.latent_size = latent_size
        
        self.net = nn.Sequential(
            # 128 -> 63
            nn.Conv2d(input_channels, 8, 4, 2),
            nn.LeakyReLU(0.2),
            
            # 63 -> 31
            nn.Conv2d(8, 16, 3, 2),
            nn.LeakyReLU(0.2),
            
            # 31 -> 15
            nn.Conv2d(16, 32, 3, 2),
            nn.LeakyReLU(0.2),

            # 15 -> 7
            nn.Conv2d(32, 64, 3, 2),
            nn.LeakyReLU(0.2),

            # 7 -> 5
            nn.Conv2d(64, 128, 3, 1),
            nn.LeakyReLU(0.2),

            # 5 -> 4
            nn.Conv2d(128, 256, 2, 1),
            nn.LeakyReLU(0.2),
            
            # 4 -> 3
            nn.Conv2d(256, 512, 2, 1),
            nn.LeakyReLU(0.2),
            
            # 3 -> 2
            nn.Conv2d(512, 1024, 2, 1),
            nn.LeakyReLU(0.2),
            
            # 2 -> 1
            nn.Conv2d(1024, latent_size, 2, 1),
            nn.LeakyReLU(0.2),
            
            nn.Flatten(),
            nn.Linear(latent_size, latent_size),
            nn.Dropout(0.4)
        )
        
        # parameters for variational autoencoder
        self.mu = nn.Linear(latent_size, latent_size)
        self.sigma = nn.Linear(latent_size, latent_size)
        self.N = torch.distributions.Normal(0, 1)
        # self.N.loc = self.N.loc.cuda()
        # self.N.scale = self.N.scale.cuda()
        self.kl = 0
    
    def forward(self, x):
        x = self.net(x)
        mu =  self.mu(x)
        sigma = torch.exp(self.sigma(x))
        x = mu + sigma * self.N.sample(mu.shape)
        self.kl = (sigma**2 + mu**2 - torch.log(sigma) - 1/2).sum()
        
        return x

class ConditionalEncoder(VariationalEncoder):
    def __init__(self, latent_size=2048):
        super().__init__(input_channels=4, latent_size=latent_size)
        
        self.emb_species = nn.Embedding(len(species), 128**2 // 3 + 128**2 % 3)
        self.emb_class = nn.Embedding(len(classes), 128**2 // 3)
        self.emb_gender = nn.Embedding(len(genders), 128**2 // 3)
        self.emb_reshape = nn.Unflatten(1, (1, 128, 128))


    def forward(self, img, species_, class_, gender_):
        x = self.emb_species(species_)
        y = self.emb_class(class_)
        z = self.emb_gender(gender_)
        
        x = torch.concat([x, y, z], dim=1)
        x = self.emb_reshape(x)
        
        x = torch.concat([img, x], dim=1)
        x = self.net(x)
        
        mu =  self.mu(x)
        sigma = torch.exp(self.sigma(x))
        x = mu + sigma * self.N.sample(mu.shape)
        self.kl = (sigma**2 + mu**2 - torch.log(sigma) - 1/2).sum()
        return x

    
class Decoder(nn.Module):
    def __init__(self, latent_size=2048):
        super().__init__()
        self.latent_size = latent_size
        self.net = nn.Sequential(
            
            nn.Linear(latent_size, latent_size),
            nn.Dropout(0.4),
            
            nn.Unflatten(1, (latent_size, 1, 1)),
            
            # 1 -> 2
            nn.ConvTranspose2d(latent_size, 1024, 2, 1),
            nn.LeakyReLU(0.2),
            
            # 2 -> 3
            nn.ConvTranspose2d(1024, 512, 2, 1),
            nn.LeakyReLU(0.2),
            
            # 3 -> 4
            nn.ConvTranspose2d(512, 256, 2, 1),
            nn.LeakyReLU(0.2),
            
            # 4 -> 5
            nn.ConvTranspose2d(256, 128, 2, 1),
            nn.LeakyReLU(0.2),
            
            # 5 -> 7
            nn.ConvTranspose2d(128, 64, 3, 1),
            nn.LeakyReLU(0.2),
            
            # 7 -> 15
            nn.ConvTranspose2d(64, 32, 3, 2),
            nn.LeakyReLU(0.2),
            
            # 15 -> 31
            nn.ConvTranspose2d(32, 16, 3, 2),
            nn.LeakyReLU(0.2),
            
            # 31 -> 63
            nn.ConvTranspose2d(16, 8, 3, 2),
            nn.LeakyReLU(0.2),

            # 63 -> 128
            nn.ConvTranspose2d(8, 3, 4, 2),
            nn.Tanh()
        )
    
    def forward(self, x):
        return self.net(x)


class ConditionalDecoder(Decoder):
    def __init__(self, latent_size=1024):
        super().__init__(latent_size)
        
        self.emb_species = nn.Embedding(len(species), latent_size // 3 + latent_size % 3)
        self.emb_class = nn.Embedding(len(classes), latent_size // 3)
        self.emb_gender = nn.Embedding(len(genders), latent_size // 3)
        self.label_net = nn.Linear(2*latent_size, latent_size)
        
    def forward(self, Z, species_, class_, gender_):
        x = self.emb_species(species_)
        y = self.emb_class(class_)
        z = self.emb_gender(gender_)
        
        x = torch.concat([Z, x, y, z], dim=1)
        x = self.label_net(x)
        x = self.net(x)
        return x

    
class VariationalAutoEncoder(nn.Module):
    def __init__(self, latent_size=1024):
        super().__init__()
        self.latent_size = latent_size
        self.enc = VariationalEncoder(latent_size)
        self.dec = Decoder(latent_size)
    
    def forward(self, x):
        return self.dec(self.enc(x))

class ConditionalVariationalAutoEncoder(nn.Module):
    def __init__(self, latent_size=1024):
        super().__init__()
        self.latent_size = latent_size
        self.enc = ConditionalEncoder(latent_size)
        self.dec = ConditionalDecoder(latent_size)
    
    def forward(self, img, species_, class_, gender_):
        Z = self.enc(img, species_, class_, gender_)
        x = self.dec(Z, species_, class_, gender_)
        return x

def show_tensor(Z, ax, **kwargs):
    if len(Z.shape) > 3:
        Z = Z[0]
    
    if Z.min() < 1:
        Z = (Z + 1) / 2
    
    Z = np.transpose(Z.detach().cpu().numpy(), (1, 2, 0))
    ax.imshow(Z, **kwargs)
    return ax