Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<find_best_model_class>
import gzip, pickle
import os
import numpy as np
import pandas as pd
import random
import shutil
import numpy as np
import os
import torch
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from torch.utils.data import Dataset, DataLoader
from scipy import linalg
import pathlib
import urllib
import warnings
from PIL import Image
from tqdm import tqdm_notebook as tqdm
model_linreg = LinearRegression()
model_linreg.fit(x_train, y_train)
 model_linreg_ypredict = model_linreg.predict(x_validate)
 model_linreg_rocaucscore = roc_auc_score(y_validate, model_linreg_ypredict)
 model_linreg_cvscores = cross_val_score(model_linreg, x, y, cv=20, scoring='roc_auc')
 print('linear regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_linreg_rocaucscore, model_linreg_cvscores.mean() , 2 * model_linreg_cvscores.std()))<compute_train_metric>
ComputeLB = True
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()
model_lassoreg = Lasso(alpha=0.01)
 model_lassoreg.fit(x_train, y_train)
 model_lassoreg_ypredict = model_lassoreg.predict(x_validate)
 model_lassoreg_rocaucscore = roc_auc_score(y_validate, model_lassoreg_ypredict)
 model_lassoreg_cvscores = cross_val_score(model_lassoreg, x, y, cv=20, scoring='roc_auc')
 print('lasso regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_lassoreg_rocaucscore, model_lassoreg_cvscores.mean() , 2 * model_lassoreg_cvscores.std()))<train_on_grid>
class dog_dataset(Dataset):
def __init__(self, train_y, train_X, zeros, device):
self.train_y = torch.Tensor(train_y ).to(device)
 self.train_X = torch.Tensor(train_X ).to(device)
 self.zeros = torch.Tensor(zeros ).to(device)
 def __len__(self):
return len(train_y)
 def __getitem__(self, idx):
return self.train_y[idx], self.train_X[idx], self.zeros[idx]
params = {
'alpha': np.logspace(-4, -2, base=10, num=50),
}
model_lassoreg_cv = GridSearchCV(model_lassoreg, params, iid=False, cv=5)
 model_lassoreg_cv.fit(x_train, y_train)
 model_lassoreg_cv_ypredict = model_lassoreg_cv.predict(x_validate)
 model_lassoreg_cv_rocaucscore = roc_auc_score(y_validate, model_lassoreg_cv_ypredict)
 model_lassoreg_cv_cvscores = cross_val_score(model_lassoreg_cv, x, y, cv=20, scoring='roc_auc')
 print('lasso regression grid search
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_lassoreg_cv_rocaucscore, model_lassoreg_cv_cvscores.mean() , 2 * model_lassoreg_cv_cvscores.std()))
print(' best parameters: %s' %model_lassoreg_cv.best_params_ )<find_best_model_class>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 bs = 256
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
data_set = dog_dataset(train_y, train_X, zeros, device)
 data_loader = DataLoader(data_set, bs )
model_ridgereg = Ridge(alpha=35)
 model_ridgereg.fit(x_train, y_train)
 model_ridgereg_ypredict = model_ridgereg.predict(x_validate)
 model_ridgereg_rocaucscore = roc_auc_score(y_validate, model_ridgereg_ypredict)
 model_ridgereg_cvscores = cross_val_score(model_ridgereg, x, y, cv=20, scoring='roc_auc')
 print('ridge regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_ridgereg_rocaucscore, model_ridgereg_cvscores.mean() , 2 * model_ridgereg_cvscores.std()))<find_best_model_class>
class Discriminator(nn.Module):
def __init__(self):
super(Discriminator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 self.conv1 = nn.Conv2d(1, 1,(2, 1), bias=False)
 def forward(self, imgs, imgnames):
x = self.fc1(imgnames)
 x = torch.sigmoid(x)
 x = torch.cat(( imgs, x), dim=1 ).view(-1, 1, 2, 12288)
 x = self.conv1(x)
 return x.view(-1, 12288 )
params = {'alpha': np.logspace(-4, 4, base=10, num=50)}
model_ridgereg_cv = GridSearchCV(model_ridgereg, params, iid=False, cv=5)
 model_ridgereg_cv.fit(x_train, y_train)
 model_ridgereg_cv_ypredict = model_ridgereg_cv.predict(x_validate)
 model_ridgereg_cv_rocaucscore = roc_auc_score(y_validate, model_ridgereg_cv_ypredict)
 model_ridgereg_cv_cvscores = cross_val_score(model_ridgereg_cv, x, y, cv=20, scoring='roc_auc')
 print('ridge regression grid search
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_ridgereg_cv_rocaucscore, model_ridgereg_cv_cvscores.mean() , 2 * model_ridgereg_cv_cvscores.std()))
print(' best parameters: %s' %model_ridgereg_cv.best_params_ )<find_best_model_class>
lr = 0.005
epochs = 360
netD = Discriminator().to(device)
 optimizerD = optim.Adam(netD.parameters() , lr=lr)
 criteria = nn.BCELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.0],
[1.0]]]] ).to(device))
for param in netD.conv1.parameters() :
param.requires_grad = False
model_elasticnetreg = ElasticNet(alpha=0.01, l1_ratio=0.9)
 model_elasticnetreg.fit(x_train, y_train)
 model_elasticnetreg_ypredict = model_elasticnetreg.predict(x_validate)
 model_elasticnetreg_rocaucscore = roc_auc_score(y_validate, model_elasticnetreg_ypredict)
 model_elasticnetreg_cvscores = cross_val_score(model_elasticnetreg, x, y, cv=20, scoring='roc_auc')
 print('elastic net regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_elasticnetreg_rocaucscore, model_elasticnetreg_cvscores.mean() , 2 * model_elasticnetreg_cvscores.std()))<train_on_grid>
for k in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netD.zero_grad()
y_pred = netD(Zeros, X)
 loss = criteria(y_pred, y)
 loss.backward()
optimizerD.step()
if(k + 1)% 2 == 0:
print(f"Epoch: {k+1}/{epochs} | Loss: {loss}" )
params = {'alpha': np.logspace(-4, -2, base=10, num=10),
'l1_ratio': np.linspace(0.1, 0.9, num=5),
}
model_elasticnetreg_cv = GridSearchCV(model_elasticnetreg, params, iid=False, cv=5)
 model_elasticnetreg_cv.fit(x_train, y_train)
 model_elasticnetreg_cv_ypredict = model_elasticnetreg_cv.predict(x_validate)
 model_elasticnetreg_cv_rocaucscore = roc_auc_score(y_validate, model_elasticnetreg_cv_ypredict)
 model_elasticnetreg_cv_cvscores = cross_val_score(model_elasticnetreg_cv, x, y, cv=20, scoring='roc_auc')
 print('elastic net regression grid search
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_elasticnetreg_cv_rocaucscore, model_elasticnetreg_cv_cvscores.mean() , 2 * model_elasticnetreg_cv_cvscores.std()))
print(' best parameters: %s' %model_elasticnetreg_cv.best_params_ )<find_best_model_class>
class Generator(nn.Module):
def __init__(self):
super(Generator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 def forward(self, imgnames):
x = self.fc1(imgnames)
 return x, imgnames.view(-1, 10000 )
model_kernelridgereg = KernelRidge(alpha=0.0001, kernel='polynomial', degree=4)
 model_kernelridgereg.fit(x_train, y_train)
 model_kernelridgereg_ypredict = model_kernelridgereg.predict(x_validate)
 model_kernelridgereg_rocaucscore = roc_auc_score(y_validate, model_kernelridgereg_ypredict)
 model_kernelridgereg_cvscores = cross_val_score(model_kernelridgereg, x, y, cv=20, scoring='roc_auc')
 print('kernel ridge regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_kernelridgereg_rocaucscore, model_kernelridgereg_cvscores.mean() , 2 * model_kernelridgereg_cvscores.std()))<train_on_grid>
def show_image() :
plt.figure(figsize=(15,3))
for j in range(5):
with torch.no_grad() :
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 inp = torch.Tensor(xx.reshape(( -1,10000)) ).to(device)
 img = netG(inp)[0].reshape(( -1,64,64,3)).to("cpu" ).clone().detach().numpy()
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
def show_d_images(imgs, title):
with torch.no_grad() :
imgs = imgs.reshape(( -1,64,64,3))
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = imgs[j].detach().cpu().numpy()
img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.title(title)
 plt.imshow(img)
 plt.show()
params = {'alpha': np.logspace(-4, -2, base=10, num=10),
'degree': [1, 2, 3, 4, 5],
}
model_kernelridgereg_cv = GridSearchCV(model_kernelridgereg, params, iid=False, cv=5)
 model_kernelridgereg_cv.fit(x_train, y_train)
 model_kernelridgereg_cv_ypredict = model_kernelridgereg_cv.predict(x_validate)
 model_kernelridgereg_cv_rocaucscore = roc_auc_score(y_validate, model_kernelridgereg_cv_ypredict)
 model_kernelridgereg_cv_cvscores = cross_val_score(model_kernelridgereg_cv, x, y, cv=20, scoring='roc_auc')
 print('kernel ridge regression grid search
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_kernelridgereg_cv_rocaucscore, model_kernelridgereg_cv_cvscores.mean() , 2 * model_kernelridgereg_cv_cvscores.std()))
print(' best parameters: %s' %model_kernelridgereg_cv.best_params_ )<find_best_model_class>
lr = 0.01
beta1 = 0.5
netG = Generator().to(device)
 optimizerG = optim.Adam(netG.parameters() , lr=lr)
 criterion = nn.MSELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.],
[1.]]]] ).to(device))
for param in netD.parameters() :
param.requires_grad = False
model_treereg = DecisionTreeRegressor(splitter='best', min_samples_split=5)
 model_treereg.fit(x_train, y_train)
 model_treereg_ypredict = model_treereg.predict(x_validate)
 model_treereg_rocaucscore = roc_auc_score(y_validate, model_treereg_ypredict)
 model_treereg_cvscores = cross_val_score(model_treereg, x, y, cv=20, scoring='roc_auc')
 print('decision tree regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_treereg_rocaucscore, model_treereg_cvscores.mean() , 2 * model_treereg_cvscores.std()))<compute_train_metric>
epochs = 50
for epoch in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netG.zero_grad()
fake, seed = netG(X)
 y_pred = netD(Zeros, seed)
 errG = criterion(fake, y_pred)
 errG.backward()
optimizerG.step()
if(epoch+1)% 5 == 0:
print(f"Epoch: {epoch+1}: G_Loss: {errG}")
 show_d_images(fake, "Generator output" )
model_forestreg = RandomForestRegressor(n_estimators=100, min_samples_split=3, random_state=58)
 model_forestreg.fit(x_train, y_train)
 model_forestreg_ypredict = model_forestreg.predict(x_validate)
 model_forestreg_rocaucscore = roc_auc_score(y_validate, model_forestreg_ypredict)
 model_forestreg_cvscores = cross_val_score(model_forestreg, x, y, cv=20, scoring='roc_auc')
 print('random forest regression
roc auc score: %0.4f, cross validation score: %0.4f(+/- %0.4f)' %(model_forestreg_rocaucscore, model_forestreg_cvscores.mean() , 2 * model_forestreg_cvscores.std()))<train_on_grid>
class DogGenerator() :
index = 0
t = [
transforms.RandomCrop(( 48, 48), padding=None, pad_if_needed=True, fill=0, padding_mode='symmetric'),
transforms.Scale(( 64,64))
]
tfms = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.5),
transforms.ColorJitter(brightness=(1,1.3), contrast=(1,1.3), saturation=0, hue=0)
 ])
 def getDog(self,seed):
xx = torch.Tensor(np.zeros(( 10000)) ).to(device)
 xx[self.index] = 0.999999
xx[np.random.randint(10000)] = 0.000001
img = netG(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3)).detach().cpu().numpy() * 255
self.index =(self.index+1)%10000
return self.tfms(Image.fromarray(img.astype('uint8')) )