Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<set_options>
import os
import PIL
import torchvision
import torchvision.datasets as dset
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
import xml.etree.ElementTree as ET
import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.optim as optim
from torch.nn.init import xavier_uniform_
import time
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.optim as optim
from torch.nn.utils import spectral_norm
import torch.utils.data
import torchvision
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
import matplotlib.image as mpimg
import torch.nn.functional as F
from torch.nn import Parameter
import numpy as np
import os
import gzip, pickle
import tensorflow as tf
from scipy import linalg
import pathlib
import urllib
import warnings
from tqdm import tqdm
from PIL import Image
import zipfile
from tqdm import tqdm_notebook as tqdm
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 device<find_best_params>
kernel_start_time = time.perf_counter()
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad )<concatenate>
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y
class MinibatchStdDev(nn.Module):
def __init__(self):
super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = torch.sqrt(y.pow(2.0 ).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size, 1, height, width)
 y = torch.cat([x, y], 1)
 return y
def conv3x3(in_channel, out_channel):
return nn.Conv2d(in_channel,out_channel,
kernel_size=3,stride=1,padding=1,dilation=1,bias=False)
 def conv1x1(in_channel, out_channel):
return nn.Conv2d(in_channel,out_channel,
kernel_size=1,stride=1,padding=0,dilation=1,bias=False)
 def init_weight(m):
classname = m.__class__.__name__
if classname.find('Conv')!= -1:
nn.init.orthogonal_(m.weight, gain=1)
 if m.bias is not None:
m.bias.data.zero_()
elif classname.find('Batch')!= -1:
m.weight.data.normal_(1,0.02)
 m.bias.data.zero_()
elif classname.find('Linear')!= -1:
nn.init.orthogonal_(m.weight, gain=1)
 if m.bias is not None:
m.bias.data.zero_()
elif classname.find('Embedding')!= -1:
nn.init.orthogonal_(m.weight, gain=1)
 class Attention(nn.Module):
def __init__(self, channels):
super().__init__()
self.channels = channels
self.theta = nn.utils.spectral_norm(conv1x1(channels, channels//8)).apply(init_weight)
 self.phi = nn.utils.spectral_norm(conv1x1(channels, channels//8)).apply(init_weight)
 self.g = nn.utils.spectral_norm(conv1x1(channels, channels//2)).apply(init_weight)
 self.o = nn.utils.spectral_norm(conv1x1(channels//2, channels)).apply(init_weight)
 self.gamma = nn.Parameter(torch.tensor(0.) , requires_grad=True)
 def forward(self, inputs):
batch,c,h,w = inputs.size()
theta = self.theta(inputs)
 phi = F.max_pool2d(self.phi(inputs), [2,2])
 g = F.max_pool2d(self.g(inputs), [2,2])
 theta = theta.view(batch, self.channels//8, -1)
 phi = phi.view(batch, self.channels//8, -1)
 g = g.view(batch, self.channels//2, -1)
 beta = F.softmax(torch.bmm(theta.transpose(1,2), phi), -1)
 o = self.o(torch.bmm(g, beta.transpose(1,2)).view(batch,self.channels//2,h,w))
return self.gamma*o + inputs
class ConditionalNorm(nn.Module):
def __init__(self, in_channel, n_condition):
super().__init__()
self.bn = nn.BatchNorm2d(in_channel, affine=False)
 self.embed = nn.Linear(n_condition, in_channel* 2)
 nn.init.orthogonal_(self.embed.weight.data[:, :in_channel], gain=1)
 self.embed.weight.data[:, in_channel:].zero_()
def forward(self, inputs, label):
out = self.bn(inputs)
 embed = self.embed(label.float())
 gamma, beta = embed.chunk(2, dim=1)
 gamma = gamma.unsqueeze(2 ).unsqueeze(3)
 beta = beta.unsqueeze(2 ).unsqueeze(3)
 out = gamma * out + beta
return out<choose_model_class>
class Generator(nn.Module):
def __init__(self, nz=128, num_classes=120, channels=3, nfeats=32):
super(Generator, self ).__init__()
self.nz = nz
self.num_classes = num_classes
self.channels = channels
self.label_emb = nn.Embedding(num_classes, nz)
 self.pixnorm = PixelwiseNorm()
self.conv1 = spectral_norm(nn.ConvTranspose2d(2*nz, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, channels, 3, 1, 1, bias=False))
def forward(self, inputs):
z, labels = inputs
enc = self.label_emb(labels ).view(( -1, self.nz, 1, 1))
enc = F.normalize(enc, p=2, dim=1)
 x = torch.cat(( z, enc), 1)
 x = F.relu(self.conv1(x))
x = F.relu(self.conv2(x))
x = self.pixnorm(x)
 x = F.relu(self.conv3(x))
x = self.pixnorm(x)
 x = F.relu(self.conv4(x))
x = self.pixnorm(x)
 x = F.relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
class Discriminator(nn.Module):
def __init__(self, num_classes=120, channels=3, nfeats=64):
super(Discriminator, self ).__init__()
self.channels = channels
self.num_classes = num_classes
self.label_emb = nn.Embedding(num_classes, 64*64)
 self.conv1 = nn.Conv2d(channels+1, nfeats, 5, 2, 2, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 5, 2, 2, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 5, 2, 2, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 5, 2, 2, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.pixnorm = PixelwiseNorm()
self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))
def forward(self, inputs):
imgs, labels = inputs
enc = self.label_emb(labels ).view(( -1, 1, 64, 64))
enc = F.normalize(enc, p=2, dim=1)
 x = torch.cat(( imgs, enc), 1)
 x = F.relu(self.conv1(x), 0.2)
 x = F.relu(self.bn2(self.conv2(x)) , 0.2)
 x = F.relu(self.bn3(self.conv3(x)) , 0.2)
 x = F.relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x = torch.sigmoid(self.conv5(x))
return x.view(-1, 1 )
<define_search_model>
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf, crop_dogs=True):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples, self.labels = self.load_dogs_data(directory, crop_dogs)
 def load_dogs_data(self, directory, crop_dogs):
required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
labels = []
paths = []
for root, _, fnames in sorted(os.walk(directory)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
try: img = dset.folder.default_loader(path)
 except: continue
annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
if crop_dogs:
tree = ET.parse(os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename))
root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 object_img = required_transforms(img.crop(( xmin, ymin, xmax, ymax)))
 imgs.append(object_img)
 labels.append(annotation_dirname.split('-')[1].lower())
 else:
object_img = required_transforms(img)
 imgs.append(object_img)
 labels.append(annotation_dirname.split('-')[1].lower())
 return imgs, labels
def __getitem__(self, index):
sample = self.samples[index]
label = self.labels[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample), label
def __len__(self):
return len(self.samples )
class ResBlock_G(nn.Module):
def __init__(self, in_channel, out_channel, condition_dim, upsample=True):
super().__init__()
self.cbn1 = ConditionalNorm(in_channel, condition_dim)
 self.upsample = nn.Sequential()
if upsample:
self.upsample.add_module('upsample',nn.Upsample(scale_factor=2, mode='nearest'))
self.conv3x3_1 = nn.utils.spectral_norm(conv3x3(in_channel, out_channel)).apply(init_weight)
 self.cbn2 = ConditionalNorm(out_channel, condition_dim)
 self.conv3x3_2 = nn.utils.spectral_norm(conv3x3(out_channel, out_channel)).apply(init_weight)
 self.conv1x1 = nn.utils.spectral_norm(conv1x1(in_channel, out_channel)).apply(init_weight)
 def forward(self, inputs, condition):
x = F.leaky_relu(self.cbn1(inputs, condition))
x = self.upsample(x)
 x = self.conv3x3_1(x)
 x = self.conv3x3_2(F.leaky_relu(self.cbn2(x, condition)))
 x += self.conv1x1(self.upsample(inputs))
return x
class Generator(nn.Module):
def __init__(self, n_feat, codes_dim=24, n_classes=n_classes):
super().__init__()
self.fc = nn.Sequential(
 nn.utils.spectral_norm(nn.Linear(codes_dim, 16*n_feat*4*4)).apply(init_weight)
)
 self.res1 = ResBlock_G(16*n_feat, 16*n_feat, codes_dim+n_classes, upsample=True)
 self.res2 = ResBlock_G(16*n_feat, 8*n_feat, codes_dim+n_classes, upsample=True)
 self.res3 = ResBlock_G(8*n_feat, 4*n_feat, codes_dim+n_classes, upsample=True)
 self.attn = Attention(4*n_feat)
 self.res4 = ResBlock_G(4*n_feat, 2*n_feat, codes_dim+n_classes, upsample=True)
 self.conv = nn.Sequential(
 nn.LeakyReLU() ,
nn.utils.spectral_norm(conv3x3(2*n_feat, 3)).apply(init_weight), 
)
 def forward(self, z, label_ohe, codes_dim=24):

 batch = z.size(0)
 z = z.squeeze()
label_ohe = label_ohe.squeeze()
codes = torch.split(z, codes_dim, dim=1)
 x = self.fc(codes[0])
 x = x.view(batch,-1,4,4)
 condition = torch.cat([codes[1], label_ohe], dim=1)
 x = self.res1(x, condition)
 condition = torch.cat([codes[2], label_ohe], dim=1)
 x = self.res2(x, condition)
 condition = torch.cat([codes[3], label_ohe], dim=1)
 x = self.res3(x, condition)
 x = self.attn(x)
 condition = torch.cat([codes[4], label_ohe], dim=1)
 x = self.res4(x, condition)
 x = self.conv(x)
 x = torch.tanh(x)
 return x
class ResBlock_D(nn.Module):
def __init__(self, in_channel, out_channel, downsample=True):
super().__init__()
self.layer = nn.Sequential(
 nn.LeakyReLU(0.2),
nn.utils.spectral_norm(conv3x3(in_channel, out_channel)).apply(init_weight),
nn.LeakyReLU(0.2),
nn.utils.spectral_norm(conv3x3(out_channel, out_channel)).apply(init_weight), 
)
 self.shortcut = nn.Sequential(
 nn.utils.spectral_norm(conv1x1(in_channel,out_channel)).apply(init_weight), 
)
 if downsample:
self.layer.add_module('avgpool',nn.AvgPool2d(kernel_size=2,stride=2))
self.shortcut.add_module('avgpool',nn.AvgPool2d(kernel_size=2,stride=2))
def forward(self, inputs):
x = self.layer(inputs)
 x += self.shortcut(inputs)
 return x
class Discriminator(nn.Module):
def __init__(self, n_feat, n_classes=n_classes):
super().__init__()
self.res1 = ResBlock_D(3, n_feat, downsample=True)
 self.attn = Attention(n_feat)
 self.res2 = ResBlock_D(n_feat, 2*n_feat, downsample=True)
 self.res3 = ResBlock_D(2*n_feat, 4*n_feat, downsample=True)
 self.res4 = ResBlock_D(4*n_feat, 8*n_feat, downsample=True)
 self.res5 = ResBlock_D(8*n_feat,16*n_feat, downsample=False)
 self.fc = nn.utils.spectral_norm(nn.Linear(16*n_feat,1)).apply(init_weight)
 self.embedding = nn.Embedding(num_embeddings=n_classes, embedding_dim=16*n_feat ).apply(init_weight)
 def forward(self, inputs, label):
batch = inputs.size(0)
 h = self.res1(inputs)
 h = self.attn(h)
 h = self.res2(h)
 h = self.res3(h)
 h = self.res4(h)
 h = self.res5(h)
 h = torch.sum(( F.leaky_relu(h,0.2)).view(batch,-1,4*4), dim=2)
 outputs = self.fc(h)
 if label is not None:
embed = self.embedding(label)
 outputs += torch.sum(embed*h,dim=1,keepdim=True)
 outputs = torch.sigmoid(outputs)
 return outputs<categorify>
database = '.. /input/all-dogs/all-dogs/'
crop_dogs = True
n_samples = np.inf
BATCH_SIZE = 32
epochs = 1600
criterion = nn.BCELoss()
nz = 128
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 transform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.5),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform, n_samples=n_samples, crop_dogs=crop_dogs)
 decoded_dog_labels = {i:breed for i, breed in enumerate(sorted(set(train_data.labels)))}
encoded_dog_labels = {breed:i for i, breed in enumerate(sorted(set(train_data.labels)))}
train_data.labels = [encoded_dog_labels[l] for l in train_data.labels]
train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,
batch_size=BATCH_SIZE, num_workers=4)
 print("Dog breeds loaded: ", len(encoded_dog_labels))
print("Data samples loaded:", len(train_data))
def generate_img(netG,fixed_noise,fixed_aux_labels=None):
if fixed_aux_labels is not None:
gen_image = netG(fixed_noise,fixed_aux_labels ).to('cpu' ).clone().detach().squeeze(0)
 else:
gen_image = netG(fixed_noise ).to('cpu' ).clone().detach().squeeze(0)
 gen_image = gen_image*0.5 + 0.5
gen_image_numpy = gen_image.numpy().transpose(0,2,3,1)
 return gen_image_numpy
def show_generate_imgs(netG,fixed_noise,fixed_aux_labels=None):
gen_images_numpy = generate_img(netG,fixed_noise,fixed_aux_labels)
 fig = plt.figure(figsize=(25, 16))
for i, img in enumerate(gen_images_numpy):
ax = fig.add_subplot(4, 8, i + 1, xticks=[], yticks=[])
 plt.imshow(img)
 plt.show()
plt.close()<choose_model_class>
netG = Generator(nz, num_classes=len(encoded_dog_labels), nfeats=32 ).to(device)
 netD = Discriminator(num_classes=len(encoded_dog_labels), nfeats=32 ).to(device)
 print("Generator parameters: ", sum(p.numel() for p in netG.parameters() if p.requires_grad))
print("Discriminator parameters:", sum(p.numel() for p in netD.parameters() if p.requires_grad))
optimizerG = optim.Adam(netG.parameters() , lr=0.0002, betas=(0.5, 0.999))
optimizerD = optim.Adam(netD.parameters() , lr=0.0002, betas=(0.5, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG, T_0=epochs//200, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD, T_0=epochs//200, eta_min=0.00005 )
def run(lr_G=3e-4,lr_D=3e-4, beta1=0.0, beta2=0.999, nz=120, epochs=2,
n_ite_D=1, ema_decay_rate=0.999, show_epoch_list=None, output_freq=10):
netG = Generator(n_feat=36, codes_dim=24, n_classes=n_classes ).to(device)
 netD = Discriminator(n_feat=42, n_classes=n_classes ).to(device)
 if EMA:
netG_EMA = Generator(n_feat=42, codes_dim=24, n_classes=n_classes ).to(device)
 netG_EMA.load_state_dict(netG.state_dict())
 for p in netG_EMA.parameters() :
p.requires_grad = False
print(count_parameters(netG))
print(count_parameters(netD))
real_label = 0.9
fake_label = 0
D_loss_list = []
G_loss_list = []
dis_criterion = nn.BCELoss().to(device)
 optimizerD = optim.Adam(netD.parameters() , lr=lr_D, betas=(beta1, beta2))
optimizerG = optim.Adam(netG.parameters() , lr=lr_G, betas=(beta1, beta2))
fixed_noise = torch.randn(32, nz, 1, 1, device=device)
 fixed_aux_labels = np.random.randint(0,n_classes, 32)
 fixed_aux_labels_ohe = np.eye(n_classes)[fixed_aux_labels]
fixed_aux_labels_ohe = torch.from_numpy(fixed_aux_labels_ohe[:,:,np.newaxis,np.newaxis])
 fixed_aux_labels_ohe = fixed_aux_labels_ohe.float().to(device, non_blocking=True)
 netG.train()
netD.train()
for epoch in range(1,epochs+1):
if elapsed_time(start_time)> TIME_LIMIT:
print(f'elapsed_time go beyond {TIME_LIMIT} sec')
 break
D_running_loss = 0
G_running_loss = 0
for ii, data in enumerate(train_loader):
for _ in range(n_ite_D):
if LABEL_NOISE:
real_label = 0.9
fake_label = 0
if np.random.random() < LABEL_NOISE_PROB:
real_label = 0
fake_label = 0.9
netD.zero_grad()
real_images = data['img'].to(device, non_blocking=True)
 batch_size = real_images.size(0)
 dis_labels = torch.full(( batch_size, 1), real_label, device=device)
 aux_labels = data['label'].long().to(device, non_blocking=True)
 dis_output = netD(real_images, aux_labels)
 errD_real = dis_criterion(dis_output, dis_labels)
 errD_real.backward(retain_graph=True)
 noise = torch.randn(batch_size, nz, 1, 1, device=device)
 aux_labels = np.random.randint(0,n_classes, batch_size)
 aux_labels_ohe = np.eye(n_classes)[aux_labels]
aux_labels_ohe = torch.from_numpy(aux_labels_ohe[:,:,np.newaxis,np.newaxis])
 aux_labels_ohe = aux_labels_ohe.float().to(device, non_blocking=True)
 aux_labels = torch.from_numpy(aux_labels ).long().to(device, non_blocking=True)
 fake = netG(noise, aux_labels_ohe)
 dis_labels.fill_(fake_label)
 dis_output = netD(fake.detach() ,aux_labels)
 errD_fake = dis_criterion(dis_output, dis_labels)
 errD_fake.backward(retain_graph=True)
 D_running_loss +=(errD_real.item() + errD_fake.item())/len(train_loader)
 optimizerD.step()
netG.zero_grad()
dis_labels.fill_(real_label)
 noise = torch.randn(batch_size, nz, 1, 1, device=device)
 aux_labels = np.random.randint(0,n_classes, batch_size)
 aux_labels_ohe = np.eye(n_classes)[aux_labels]
aux_labels_ohe = torch.from_numpy(aux_labels_ohe[:,:,np.newaxis,np.newaxis])
 aux_labels_ohe = aux_labels_ohe.float().to(device, non_blocking=True)
 aux_labels = torch.from_numpy(aux_labels ).long().to(device, non_blocking=True)
 fake = netG(noise, aux_labels_ohe)
 dis_output = netD(fake, aux_labels)
 errG = dis_criterion(dis_output, dis_labels)
 errG.backward(retain_graph=True)
 G_running_loss += errG.item() /len(train_loader)
 optimizerG.step()
if EMA:
param_itr = cycle(netG.parameters())
 for i,p_EMA in enumerate(netG_EMA.parameters()):
p = next(param_itr)
 p_EMA.data =(1-ema_decay_rate)*p_EMA.data + ema_decay_rate*p.data
p_EMA.requires_grad = False
D_loss_list.append(D_running_loss)
 G_loss_list.append(G_running_loss)
 if epoch % output_freq == 0:
print('[{:d}/{:d}] D_loss = {:.3f}, G_loss = {:.3f}, elapsed_time = {:.1f} min'.format(epoch,epochs,D_running_loss,G_running_loss,elapsed_time(start_time)/60))
if epoch in show_epoch_list:
print('epoch = {}'.format(epoch))
if not EMA:
show_generate_imgs(netG,fixed_noise,fixed_aux_labels_ohe)
 elif EMA:
show_generate_imgs(netG_EMA,fixed_noise,fixed_aux_labels_ohe)
 if epoch % 100 == 0:
if not EMA:
torch.save(netG.state_dict() , f'generator_epoch{epoch}.pth')
 elif EMA:
torch.save(netG_EMA.state_dict() , f'generator_epoch{epoch}.pth')
 if not EMA:
torch.save(netG.state_dict() , 'generator.pth')
 elif EMA:
torch.save(netG_EMA.state_dict() , 'generator.pth')
 torch.save(netD.state_dict() , 'discriminator.pth')
 res = {'netG':netG,
'netD':netD,
'nz':nz,
'fixed_noise':fixed_noise,
'fixed_aux_labels_ohe':fixed_aux_labels_ohe,
'D_loss_list':D_loss_list,
'G_loss_list':G_loss_list,
}
if EMA:
res['netG_EMA'] = netG_EMA
return res<define_search_space>
for epoch in range(epochs):
epoch_time = time.perf_counter()
if time.perf_counter() - kernel_start_time > 30000:
print("Time limit reached! Stopping kernel!"); break
for ii,(real_images, dog_labels)in tqdm(enumerate(train_loader),total=len(train_loader)) :
if real_images.shape[0]!= BATCH_SIZE: continue
real_labels = torch.squeeze(torch.empty(( BATCH_SIZE, 1), device=device ).uniform_(0.60, 0.8))
fake_labels = torch.squeeze(torch.empty(( BATCH_SIZE, 1), device=device ).uniform_(0.00, 0.2))
netD.zero_grad()
dog_labels = torch.tensor(dog_labels, device=device)
 real_images = real_images.to(device)
 output = netD(( real_images, dog_labels))
errD_real = criterion(output, real_labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(BATCH_SIZE, nz, 1, 1, device=device)
 fake_images = netG(( noise, dog_labels))
output = netD(( fake_images.detach() , dog_labels))
errD_fake = criterion(output, fake_labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
output = netD(( fake_images, dog_labels))
errG = criterion(output, real_labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)
 print('%.2fs [%d/%d] Loss_D: %.4f Loss_G: %.4f' %(
 time.perf_counter() -epoch_time, epoch+1, epochs, errD.item() , errG.item()))
print('D(x): ',D_x,', D(G(z)) : ',D_G_z1)
 show_generated_img()
%%time
show_epoch_list = np.arange(0,500+10,10)
 res = run(lr_G=3e-4,lr_D=3e-4, beta1=0.0, beta2=0.999, nz=120, epochs=500,
n_ite_D=1, ema_decay_rate=None, show_epoch_list=show_epoch_list, output_freq=10)
<define_variables>
def mse(imageA, imageB):
err = np.sum(( imageA.astype("float")- imageB.astype("float")) ** 2)
 err /= float(imageA.shape[0] * imageA.shape[1])
 return err
def analyse_generated_by_class(n_images=5):
good_breeds = []
for l in range(len(decoded_dog_labels)) :
sample = []
for _ in range(n_images):
noise = torch.randn(1, nz, 1, 1, device=device)
 dog_label = torch.full(( 1,), l, device=device, dtype=torch.long)
 gen_image = netG(( noise, dog_label)).to("cpu" ).clone().detach().squeeze(0)
 gen_image = gen_image.numpy().transpose(1, 2, 0)
 sample.append(gen_image)
 d = np.round(np.sum([mse(sample[k], sample[k+1])for k in range(len(sample)-1)])/n_images, 1)
 if d < 1.0: continue
print(f"Generated breed({d}): ", decoded_dog_labels[l])
 figure, axes = plt.subplots(1, len(sample), figsize=(64, 64))
for index, axis in enumerate(axes):
axis.axis('off')
 image_array =(sample[index] + 1.) / 2.
 axis.imshow(image_array)
 plt.show()
good_breeds.append(l)
 return good_breeds
def submission_generate_images(res, truncated=None):
im_batch_size=50
n_images=10000
if not EMA:
netG = res['netG']
elif EMA:
netG = res['netG_EMA']
nz = res['nz']
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 for i_batch in range(0, n_images, im_batch_size):
if truncated is not None:
flag = True
while flag:
z = np.random.randn(100*im_batch_size*nz)
 z = z[np.where(abs(z)<truncated)]
if len(z)>=im_batch_size*nz:
flag=False
gen_z = torch.from_numpy(z[:im_batch_size*nz] ).view(im_batch_size,nz,1,1)
 gen_z = gen_z.float().to(device)
 else:
gen_z = torch.randn(im_batch_size, nz, 1, 1, device=device)
 aux_labels = np.random.randint(0,n_classes, im_batch_size)
 aux_labels_ohe = np.eye(n_classes)[aux_labels]
aux_labels_ohe = torch.from_numpy(aux_labels_ohe[:,:,np.newaxis,np.newaxis])
 aux_labels_ohe = aux_labels_ohe.float().to(device)
 gen_images = netG(gen_z,aux_labels_ohe)
 gen_images = gen_images.to("cpu" ).clone().detach()
gen_images = gen_images*0.5 + 0.5
for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :],
os.path.join(f'.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive(f'images', 'zip', f'.. /output_images' )<set_options>
def create_submit(good_breeds):
print("Creating submit")
 os.makedirs('.. /output_images', exist_ok=True)
 im_batch_size = 100
n_images = 10000
all_dog_labels = np.random.choice(good_breeds, size=n_images, replace=True)
 for i_batch in range(0, n_images, im_batch_size):
noise = torch.randn(im_batch_size, nz, 1, 1, device=device)
 dog_labels = torch.from_numpy(all_dog_labels[i_batch:(i_batch+im_batch_size)] ).to(device)
 gen_images = netG(( noise, dog_labels))
gen_images =(gen_images.to("cpu" ).clone().detach() + 1)/ 2
for ii, img in enumerate(gen_images):
save_image(gen_images[ii, :, :, :], os.path.join('.. /output_images', f'image_{i_batch + ii:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )
<set_options><EOS> 
good_breeds = analyse_generated_by_class(6)
 create_submit(good_breeds )