Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<train_model>
import shutil
import os
import random
from time import time
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch.nn.utils import spectral_norm
from torch.utils.data import Dataset
import torchvision
import xml.etree.ElementTree as ET
from tqdm import tqdm_notebook as tqdm
from scipy.stats import truncnorm
def image_resize(image, width=None, height=None, inter=cv2.INTER_AREA):
dim = None 
(h, w)= image.shape[:2]
if width is None and height is None:
return image
if width is None:
r = height / float(h)
 width = max(height, int(w * r))
else:
r = width / float(w)
 height = max(width, int(h * r))
dim =(width, height)
 resized = cv2.resize(image, dim, interpolation=inter)
 return resized
def random_crop(arr_image, width, height):
x = random.randint(0, arr_image.shape[1] - width)
 y = random.randint(0, arr_image.shape[0] - height)
 arr_image = arr_image[y:y + height, x:x + width]
return arr_image
def random_fliplr(arr_image):
r = random.randint(0, 2)
 if r == 1:
return np.fliplr(arr_image)
 return arr_image
class ArrayDataset(Dataset):
def __init__(self, path, size=64, margin=0, mean=None, std=None, count=None):
self.size = size
self.margin = margin
files = os.listdir(path)
 if count is not None:
files = files[:count]
self.arrange(path, files)
 if mean is None or std is None:
m, s = self.calc_norm()
if mean is None: mean = m
if std is None: std = s
self.mean = np.array(mean ).reshape(-1, 1, 1)
 self.std = np.array(std ).reshape(-1, 1, 1)
 def arrange(self, path, files):
self.images = []
self.labels = []
sizem = self.size + self.margin
for fn in files:
image = cv2.imread(os.path.join(path, fn))
if image.shape[0] > image.shape[1]:
image = image_resize(image, width=sizem)
 else:
image = image_resize(image, height=sizem)
 image = image[..., ::-1].copy()
self.images.append(image)
 lbl = fn[:3]
if str.isdigit(lbl):
self.labels.append(int(lbl))
else:
self.labels.append(0)
 def calc_norm(self):
mean = np.zeros(3)
 std = np.zeros(3)
 for x in self.images:
x = np.transpose(x,(2, 0, 1))
x = x.reshape(3, -1)
 mean += x.mean(1)
 std += x.std(1)
 m =(mean / len(self.images)) / 255.0
s =(std / len(self.images)) / 255.0
return m, s
def __len__(self):
return len(self.images)
 def __getitem__(self, idx):
x = self.images[idx]
x = random_crop(x, self.size, self.size)
 x = random_fliplr(x ).copy()
x = np.transpose(x,(2, 0, 1))
x = x / 255.0
x = x - self.mean
x = x / self.std
x = x.astype(np.float32)
 return x, self.labels[idx] 
<choose_model_class>
CHANNEL = {'RGB': 3, 'BW': 1}
IMG_CHANNEL = CHANNEL['RGB']
DATA_LOCATION = '.. /input/all-dogs/all-dogs/'
BATCH_SIZE = 32
LATENT_DIM = 128
CONV_DEPTHS_G = 32
CONV_DEPTHS_D = 48
LEARNING_RATE_G = 0.0003
LEARNING_RATE_D = 0.0001
BETA_1 = 0.5
T0_interval = 200
EPOCHS = 600
NUM_CLASSES = 10
TIME_FOR_TRAIN = 25000
%%time

 train_data = ArrayDataset(PATH_DOGS, size=SIZE, margin=0)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=BATCH, num_workers=4, drop_last=True)
 netG = Generator(NZ, FEATURES_G ).to(DEVICE)
 netD = Discriminator(FEATURES_D ).to(DEVICE)
 criterion = nn.BCELoss()
optimizerG = optim.Adam(netG.parameters() , lr=LRG, betas=(0.5, 0.999))
optimizerD = optim.Adam(netD.parameters() , lr=LRD, betas=(0.5, 0.999))
schedulerG = op.lr_scheduler.LambdaLR(optimizerG, lambda epoch: LRFACTOR_G ** epoch)
 schedulerD = op.lr_scheduler.LambdaLR(optimizerD, lambda epoch: LRFACTOR_D ** epoch)
 def noise_generator(batch_size):
return torch.randn(batch_size, NZ, 1, 1)
 def label_generator(batch_size, mode):
if mode == 'real':
label = np.random.uniform(*REAL_LABEL, batch_size ).reshape(batch_size, 1)
 return torch.tensor(label, dtype=torch.float32)
 if mode == 'fake':
label = np.random.uniform(*FAKE_LABEL, batch_size ).reshape(batch_size, 1)
 return torch.tensor(label, dtype=torch.float32)
 if mode == 'real-fake':
return torch.full(( batch_size, 1), REAL_FAKE_LABEL)
 raise Exception('Invalid parameter mode')
<set_options>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
def needs_scheduler(epoch):
if epoch > 0:
if epoch % 50 == 0:
print('Scheduler update')
 return True
return False 
<save_model>
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = torchvision.datasets.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin + w, ymin + w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
 transform = torchvision.transforms.Compose([torchvision.transforms.RandomHorizontalFlip(p=0.1),
torchvision.transforms.ToTensor() ,
torchvision.transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 data_gen = DataGenerator(DATA_LOCATION, transform=transform, n_samples=25000)
 train_loader = torch.utils.data.DataLoader(data_gen, shuffle=True, batch_size=BATCH_SIZE, num_workers=4 )
torch.save(netG.state_dict() , 'generator.pth')
 torch.save(netD.state_dict() , 'discriminator.pth')
<define_variables>
class PixelwiseNorm(torch.nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y
class Generator(torch.nn.Module):
def __init__(self, latent_dim, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(torch.nn.ConvTranspose2d(latent_dim, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(torch.nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.conv3 = spectral_norm(torch.nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.conv4 = spectral_norm(torch.nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.conv5 = spectral_norm(torch.nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.conv6 = spectral_norm(torch.nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = torch.nn.functional.leaky_relu(self.conv1(x))
x = self.pixnorm(x)
 x = torch.nn.functional.leaky_relu(self.conv2(x))
x = torch.nn.Dropout(0.05 )(x)
 x = self.pixnorm(x)
 x = torch.nn.functional.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = torch.nn.functional.leaky_relu(self.conv4(x))
x = torch.nn.Dropout(0.05 )(x)
 x = self.pixnorm(x)
 x = torch.nn.functional.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
SEEDS = [
1525, 491, 1117, 1509, 1538, 1478, 16, 1251, 1126, 81, 254, 704, 1740, 2081, 79, 980, 1770, 795, 1317, 1471, 2198, 153, 1488, 1456, 454, 573, 690, 2289, 1753, 2075, 772, 301, 1240, 83, 686, 932, 826, 27, 24, 1744, 880, 2029, 1421, 1490, 667, 1387, 1948, 2130, 15, 133, 2151, 602, 76, 2090, 1277, 717, 2175, 2168, 1313, 1562, 1526, 1092, 800, 1804, 1605, 1120, 2320, 2372, 2080, 460, 126, 2302, 1940, 983, 668, 773, 1788, 565, 1540, 2337, 140, 2173, 2010, 1784, 1248, 1106, 1722, 208, 1757, 2371, 449, 481, 1914, 2217, 437, 1161, 169, 693, 807, 1839, 217, 1520, 1102, 30, 1199, 239, 603, 730, 569, 2309, 1434, 1679, 3, 313, 532, 35, 825, 824, 388, 255, 1758, 763, 87, 751, 2073, 585, 407, 1510, 365, 739, 417, 463, 1412, 1636, 1237, 1439, 278, 227, 1916, 2261, 154, 1622, 1833, 2213, 1738, 74, 892, 1203, 2001, 1227, 815, 872, 1769, 699, 2374, 1050, 1073, 620, 202, 1420, 2211, 840, 568, 1715, 1337, 1651, 1591, 878, 1445, 1100, 1702, 551, 72, 1917, 1256, 1479, 152, 2192, 559, 2031, 1920, 23, 2304, 2153, 1848, 2032, 1976, 578, 650, 1464, 1706, 820, 2232, 762, 33, 1627, 1141, 2262, 781, 174,
] 
<load_pretrained>
class MinibatchStdDev(torch.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = torch.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size, 1, height, width)
 y = torch.cat([x, y], 1)
 return y
state_dict = torch.load('generator.pth')
 def load_model(model):
model.load_state_dict(state_dict)
 def generate_numpy_images(netG, count, batch_size, noise_gen):
netG.eval()
images_set = []
with torch.no_grad() :
for n in range(count // batch_size):
gen_z = noise_gen(batch_size ).to(DEVICE)
 gen_images = netG(gen_z)
 aimages = gen_images.detach().cpu()
aimages =(aimages + 1.0)* 0.5 * 255
images = aimages.numpy().transpose(0, 2, 3, 1)
 images = images.astype(int)
 images_set.append(images)
 images = np.concatenate(images_set)
 return images
def generate_tensors(netG, count, batch_size, noise_gen):
netG.eval()
images_set = []
with torch.no_grad() :
for n in range(count // batch_size):
gen_z = noise_gen(batch_size ).to(DEVICE)
 gen_images = netG(gen_z)
 images = gen_images.detach().cpu()
images_set.append(images)
 images = torch.cat(images_set)
 netG.train()
return images
def generate_images(count, batch, seed):
netG = Generator(NZ, FEATURES_G ).to(DEVICE)
 load_model(netG)
 random_seed(seed)
 images = generate_tensors(netG, count, batch, noise_generator)
 return images 
<train_model>
class Discriminator(torch.nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = torch.nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(torch.nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = torch.nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(torch.nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = torch.nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(torch.nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = torch.nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.conv5 = spectral_norm(torch.nn.Conv2d(nfeats * 8 + 1, 1, 2, 1, 0, bias=False))
def forward(self, x):
x = torch.nn.functional.leaky_relu(self.conv1(x), 0.2)
 x = torch.nn.functional.leaky_relu(self.bn2(self.conv2(x)) , 0.2)
 x = torch.nn.functional.leaky_relu(self.bn3(self.conv3(x)) , 0.2)
 x = torch.nn.functional.leaky_relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x = self.conv5(x)
 x = x.view(x.size(0), -1)
 return x
%%time
PATH_GENERATED = '.. /output_images'
if not os.path.exists(PATH_GENERATED):
os.mkdir(PATH_GENERATED)
 n = 0
for seed in SEEDS:
images = generate_images(50, 50, seed)
 images =(images + 1.0)* 0.5
for ix in range(50):
image = images[ix]
save_image(image, os.path.join(PATH_GENERATED, f'image_{n:05d}.png'), normalize=False)
 n += 1
print(n)
 shutil.make_archive('images', 'zip', PATH_GENERATED)
<set_options>
def mse_loss(input, target):
return torch.sum(( input - target)**2)/ input.data.nelement()
gan_loss = mse_loss
fixed_noise = torch.randn(25, LATENT_DIM, 1, 1, device=device)
%matplotlib inline
warnings.filterwarnings('ignore', category=FutureWarning)
 device = 'cuda' if torch.cuda.is_available() else 'cpu'<set_options>
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
config = {'DataLoader': {'batch_size': 128,
'shuffle': True},
'Generator': {'latent_dim': 120,
'embed_dim': 32,
'ch': 64,
'num_classes': 120,
'use_attn': True},
'Discriminator': {'ch': 64,
'num_classes': 120,
'use_attn': True},
'sample_latents': {'latent_dim': 120,
'num_classes': 120},
'num_iterations': 26000,
'decay_start_iteration': 25000,
'd_steps': 1,
'lr_G': 2e-4,
'lr_D': 4e-4,
'betas':(0.0, 0.999),
'margin': 1.0,
'gamma': 0.1,
'ema': 0.999,
'seed': 42}<set_options>
def train_module(epochs):
step = 0
start = time()
for epoch in range(epochs):
for ii,(real_images)in enumerate(train_loader):
end = time()
if(end - start)> TIME_FOR_TRAIN:
break
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)
 outputR = netD(real_images)
 D_x = outputR.mean().item()
noise = torch.randn(batch_size, LATENT_DIM,1, 1, device=device)
 fake = netG(noise)
 outputF = netD(fake.detach())
 D_G_z1 = outputF.mean().item()
errD = gan_loss(outputR, torch.mean(outputF)+ labels)+ gan_loss(outputF, torch.mean(outputR)- labels)
 errD.backward(retain_graph=True)
 optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 outputF = netD(fake)
 errG = gan_loss(outputR, torch.mean(outputF)- labels)+ gan_loss(outputF, torch.mean(outputR)+ labels)
 errG.backward(retain_graph=True)
 D_G_z2 = outputF.mean().item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
valid_image = netG(fixed_noise)
 step += 1
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)
 if epoch % 5 == 0:
show_generated_img()
print(end - start)
 if(end - start)> TIME_FOR_TRAIN:
break
<define_variables><EOS> 
netG = Generator(LATENT_DIM, CONV_DEPTHS_G, IMG_CHANNEL ).to(device)
 netD = Discriminator(3, CONV_DEPTHS_D ).to(device)
 LEARNING_RATE_G = 0.003
LEARNING_RATE_D = 0.001
optimizerD = torch.optim.Adam(netD.parameters() , lr=LEARNING_RATE_G, betas=(BETA_1, 0.999))
optimizerG = torch.optim.Adam(netG.parameters() , lr=LEARNING_RATE_D, betas=(BETA_1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=EPOCHS // T0_interval, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=EPOCHS // T0_interval, eta_min=0.00005)
 fixed_noise = torch.randn(25, LATENT_DIM, 1, 1, device=device)
 real_label = 0.5
fake_label = 0.0
batch_size = train_loader.batch_size
train_module(EPOCHS )