Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<define_variables>
batch_size = 64
all_files = os.listdir(root_images )<define_variables>
start = time()
breeds = glob.glob(root_annots+'*')
 annotations = []
for breed in breeds:
annotations += glob.glob(breed+'/*' )<categorify>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
breed_map = {}
for annotation in annotations:
breed = annotation.split('/')[-2]
index = breed.split('-')[0]
breed_map.setdefault(index, breed )<categorify>
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(64),
torchvision.transforms.CenterCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples )
all_labels = [breed_map[file.split('_')[0]] for file in all_files]
le = LabelEncoder()
all_labels = le.fit_transform(all_labels )<define_variables>
%%time
database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomHorizontalFlip(p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform,n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True, batch_size=batch_size, num_workers = 4 )
all_bboxes = [load_bbox(file)for file in all_files]<normalization>
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y
all_images = [load_bboxcrop_resized_image(f, b)for f, b in zip(all_files, resized_bboxes)]
all_images = np.array(all_images )<categorify>
class MinibatchStdDev(th.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y
class GeneralizedDropOut(nn.Module):
def __init__(self, mode='mul', strength=0.4, axes=(0,1), normalize=False):
super(GeneralizedDropOut, self ).__init__()
self.mode = mode.lower()
assert self.mode in ['mul', 'drop', 'prop'], 'Invalid GDropLayer mode'%mode
self.strength = strength
self.axes = [axes] if isinstance(axes, int)else list(axes)
 self.normalize = normalize
self.gain = None
def forward(self, x, deterministic=False):
if deterministic or not self.strength:
return x
rnd_shape = [s if axis in self.axes else 1 for axis, s in enumerate(x.size())]
if self.mode == 'drop':
p = 1 - self.strength
rnd = np.random.binomial(1, p=p, size=rnd_shape)/ p
elif self.mode == 'mul':
rnd =(1 + self.strength)** np.random.normal(size=rnd_shape)
 else:
coef = self.strength * x.size(1)** 0.5
rnd = np.random.normal(size=rnd_shape)* coef + 1
if self.normalize:
rnd = rnd / np.linalg.norm(rnd, keepdims=True)
 rnd = Variable(torch.from_numpy(rnd ).type(x.data.type()))
if x.is_cuda:
rnd = rnd.cuda()
return x * rnd
def __repr__(self):
param_str = '(mode = %s, strength = %s, axes = %s, normalize = %s)' %(self.mode, self.strength, self.axes, self.normalize)
 return self.__class__.__name__ + param_str
class DogDataset(Dataset):
def __init__(self, images, labels):
super().__init__()
self.images = images
self.labels = labels
self.transform = A.Compose([A.HorizontalFlip(p=0.5), ToTensor() ])
 def __len__(self):
return len(self.images)
 def __getitem__(self, idx):
img, label = self.images[idx], self.labels[idx]
img = self.transform(image=img)['image']
label = torch.as_tensor(label, dtype=torch.long)
 return img, label<create_dataframe>
def l2normalize(v, eps=1e-12):
return v /(v.norm() + eps)
 class SpectralNorm(nn.Module):
def __init__(self, module, name='weight', power_iterations=1):
super(SpectralNorm, self ).__init__()
self.module = module
self.name = name
self.power_iterations = power_iterations
def _update_u_v(self):
if not self._made_params() :
self._make_params()
w = getattr(self.module, self.name)
 u = getattr(self.module, self.name + "_u")
 height = w.data.shape[0]
for _ in range(self.power_iterations):
v = l2normalize(torch.mv(torch.t(w.view(height,-1 ).data), u))
u = l2normalize(torch.mv(w.view(height,-1 ).data, v))
setattr(self.module, self.name + "_u", u)
 w.data = w.data / torch.dot(u, torch.mv(w.view(height,-1 ).data, v))
def _made_params(self):
try:
u = getattr(self.module, self.name + "_u")
 return True
except AttributeError:
return False
def _make_params(self):
w = getattr(self.module, self.name)
 height = w.data.shape[0]
width = w.view(height, -1 ).data.shape[1]
u = l2normalize(w.data.new(height ).normal_(0, 1))
self.module.register_buffer(self.name + "_u", u)
 def forward(self, *args):
self._update_u_v()
return self.module.forward(*args )
<define_variables><EOS> 
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.bn1 = nn.BatchNorm2d(nfeats * 8)
 self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 8)
 self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.bn4 = nn.BatchNorm2d(nfeats * 2)
 self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.bn5 = nn.BatchNorm2d(nfeats)
 self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = self.conv1(x)
 x = self.bn1(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv2(x)
 x = self.bn2(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv3(x)
 x = self.bn3(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv4(x)
 x = self.bn4(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv5(x)
 x = self.bn5(x)
 x = F.leaky_relu(x, 0.05)
 x = self.pixnorm(x)
 x = self.conv6(x)
 x = torch.tanh(x)
 return x
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.gdo3 = GeneralizedDropOut(mode='prop', strength=0.0)
 self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))
self.avg_pool2d = nn.AvgPool2d(kernel_size=2)
 self.pixnorm = PixelwiseNorm()
self.batch_discriminator = MinibatchStdDev()
def forward(self, x):
x = self.conv1(x)
 x = F.leaky_relu(x, 0.2)
 x = self.conv2(x)
 x = self.bn2(x)
 x = F.leaky_relu(x, 0.2)
 x = self.pixnorm(x)
 x = self.conv3(x)
 x = self.gdo3(x)
 x = self.bn3(x)
 x = F.leaky_relu(x, 0.2)
 x = self.pixnorm(x)
 x = self.conv4(x)
 x = self.bn4(x)
 x = F.leaky_relu(x, 0.2)
 x = self.pixnorm(x)
 x = self.batch_discriminator(x)
 x = self.conv5(x)
 x= self.avg_pool2d(x)
 x = torch.sigmoid(x)
 return x.view(-1, 1)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 lr = 0.0003
lr_d = 0.0001
beta1 = 0.5
epochs = 1400
nz = 256
netG = Generator(nz, 64, 3 ).to(device)
 netD = Discriminator(3, 64 ).to(device)
 criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr=lr, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr=lr_d, betas=(beta1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=epochs//200, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=epochs//200, eta_min=0.00005)
 fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 real_label = 0.99
fake_label = 0.01
batch_size = train_loader.batch_size
step = 0
for epoch in range(epochs):
for ii,(real_images)in tqdm(enumerate(train_loader), total=len(train_loader)) :
end = time()
if(end - start)> 32000 :
break
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)+ np.random.uniform(-0.1, 0.1)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)+ np.random.uniform(0, 0.2)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
if step % 500 == 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
valid_image = netG(fixed_noise)
 step += 1
lr_schedulerG.step(errG.item())
 lr_schedulerD.step(errD.item())
 if epoch % 5 == 0:
show_generated_img()
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 50
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, nz, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images = netG(gen_z)
 images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(( gen_images[i_image, :, :, :] +1.0)/2.0, os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )