Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<categorify>
import gzip, pickle
import os
import numpy as np
import pandas as pd
import random
import shutil
import numpy as np
import os
import torch
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from torch.utils.data import Dataset, DataLoader
from scipy import linalg
import pathlib
import urllib
import warnings
from PIL import Image
from tqdm import tqdm_notebook as tqdm
def model(train_data, test_data, n_folds = 10):
train_ids = train_data[['idhogar']]
test_ids = test_data[['idhogar']]
labels = train_data[['Target']].astype(int)
 train_data = train_data.drop(['idhogar','Target'],axis = 1)
 test_data = test_data.drop(['idhogar'], axis = 1)
 feature_names = list(train_data.columns)
 k_fold = KFold(n_splits = n_folds, shuffle = True, random_state = 2018)
 test_predictions = list()
out_of_fold = np.zeros(train_data.shape[0])
 feature_importance_values = np.zeros(len(feature_names))
Valid_F1 = []
Train_F1 = []
Valid_Score = []
Train_Score = []
for train_indices, valid_indices in k_fold.split(train_data):
train_features = train_data.loc[train_indices, :]
train_labels = labels.loc[train_indices, :]
valid_features = train_data.loc[valid_indices, :]
valid_labels = labels.loc[valid_indices, :]
model = lgb.LGBMClassifier(boosting_type='gbdt',n_estimators=2000,
objective = 'multiclass', class_weight = 'balanced',
learning_rate = 0.03, num_leaves = 31,
reg_alpha = 0.1, reg_lambda = 0.3, num_class = 4,
subsample = 0.8, n_jobs = -1, random_state = 2018)
 model.fit(train_features, train_labels, eval_metric = 'multi_error',
eval_set = [(valid_features, valid_labels),(train_features, train_labels)],
eval_names = ['valid', 'train'], categorical_feature = 'auto',
early_stopping_rounds = 100, verbose = 200)
 best_iteration = model.best_iteration_
test_predictions.append(model.predict(test_data, num_iteration = best_iteration))
feature_importance_values += model.feature_importances_ /n_folds
valid_score = model.best_score_['valid']['multi_error']
train_score = model.best_score_['train']['multi_error']
Valid_Score.append(valid_score)
 Train_Score.append(train_score)
 pred_valid = model.predict(valid_features, num_iteration = best_iteration)
 pred_train = model.predict(train_features, num_iteration = best_iteration)
 valid_f1 = f1_score(valid_labels, pred_valid, average='macro')
 train_f1 = f1_score(train_labels, pred_train, average='macro')
 Valid_F1.append(valid_f1)
 Train_F1.append(train_f1)
 out_of_fold[valid_indices] = pred_valid
gc.enable()
del model, train_features, valid_features
gc.collect()
print('................................................ ')
 feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})
 Valid_F1.append(f1_score(labels, out_of_fold, average='macro'))
Train_F1.append(np.mean(Train_F1))
Valid_Score.append(np.mean(Valid_Score))
Train_Score.append(np.mean(Train_Score))
fold_names = list(range(n_folds))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names,
'train error': Train_Score,
'valid error': Valid_Score,
'train f1' : Train_F1,
'valid f1' : Valid_F1})
 predict_df = pd.DataFrame(np.array(test_predictions ).T)
 voting_result = [predict_df.iloc[x,:].value_counts().argmax() for x in range(predict_df.shape[0])]
submission = test_ids.copy()
submission['Target'] = voting_result
return metrics, feature_importances,submission<compute_test_metric>
ComputeLB = False
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()
metric, feature_importance, submission = model(train_data, test_data, 10 )<train_model>
class dog_dataset(Dataset):
def __init__(self, train_y, train_X, zeros, device):
self.train_y = torch.Tensor(train_y ).to(device)
 self.train_X = torch.Tensor(train_X ).to(device)
 self.zeros = torch.Tensor(zeros ).to(device)
 def __len__(self):
return len(train_y)
 def __getitem__(self, idx):
return self.train_y[idx], self.train_X[idx], self.zeros[idx]
metric<save_to_csv>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 bs = 256
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
data_set = dog_dataset(train_y, train_X, zeros, device)
 data_loader = DataLoader(data_set, bs )
submit = test[['Id','idhogar']]
submit = submit.merge(submission, on = 'idhogar')
 submit = submit.drop(['idhogar'],axis = 1)
 submit.to_csv('submit.csv',index = False )<set_options>
class Discriminator(nn.Module):
def __init__(self):
super(Discriminator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 self.conv1 = nn.Conv2d(1, 1,(2, 1), bias=False)
 def forward(self, imgs, imgnames):
x = self.fc1(imgnames)
 x = torch.sigmoid(x)
 x = torch.cat(( imgs, x), dim=1 ).view(-1, 1, 2, 12288)
 x = self.conv1(x)
 return x.view(-1, 12288 )
%matplotlib inline<import_modules>
lr = 0.005
epochs = 360
netD = Discriminator().to(device)
 optimizerD = optim.Adam(netD.parameters() , lr=lr)
 criteria = nn.BCELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.0],
[1.0]]]] ).to(device))
for param in netD.conv1.parameters() :
param.requires_grad = False
from IPython.display import clear_output as clear
import sklearn<load_from_csv>
for k in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netD.zero_grad()
y_pred = netD(Zeros, X)
 loss = criteria(y_pred, y)
 loss.backward()
optimizerD.step()
if(k + 1)% 2 == 0:
print(f"Epoch: {k+1}/{epochs} | Loss: {loss}" )
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 train_df.head()<data_type_conversions>
class Generator(nn.Module):
def __init__(self):
super(Generator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 def forward(self, imgnames):
x = self.fc1(imgnames)
 return x, imgnames.view(-1, 10000 )
train_df = train_df.fillna(0)
 test_df = test_df.fillna(0 )<define_variables>
def show_image() :
plt.figure(figsize=(15,3))
for j in range(5):
with torch.no_grad() :
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 inp = torch.Tensor(xx.reshape(( -1,10000)) ).to(device)
 img = netG(inp)[0].reshape(( -1,64,64,3)).to("cpu" ).clone().detach().numpy()
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
def show_d_images(imgs, title):
with torch.no_grad() :
imgs = imgs.reshape(( -1,64,64,3))
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = imgs[j].detach().cpu().numpy()
img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.title(title)
 plt.imshow(img)
 plt.show()
train_label = np.array(train_df_sorted.Target.tolist() )<feature_engineering>
lr = 0.01
beta1 = 0.5
netG = Generator().to(device)
 optimizerG = optim.Adam(netG.parameters() , lr=lr)
 criterion = nn.MSELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.],
[1.]]]] ).to(device))
for param in netD.parameters() :
param.requires_grad = False
train_data = []
for i in range(len(train_df_sorted)) :
li = train_df_sorted.iloc[i]
li = li.drop('Target')
 li = li.drop('Id' ).tolist()
for _ in range(len(li)) :
try:
float(li[_])
 except ValueError:
if li[_] == 'yes':
li[_] = np.exp(1)
 else:
li[_] = 0
li = np.array(li, dtype='float64')
 li = np.log1p(np.abs(li))
train_data.append(li)
 train_data = np.array(train_data)
 print('done' )<feature_engineering>
epochs = 50
for epoch in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netG.zero_grad()
fake, seed = netG(X)
 y_pred = netD(Zeros, seed)
 errG = criterion(fake, y_pred)
 errG.backward()
optimizerG.step()
if(epoch+1)% 5 == 0:
print(f"Epoch: {epoch+1}: G_Loss: {errG}")
 show_d_images(fake, "Generator output" )
<set_options><EOS> 
class DogGenerator() :
index = 0
t = [
transforms.RandomCrop(( 48, 48), padding=None, pad_if_needed=True, fill=0, padding_mode='symmetric'),
transforms.Scale(( 64,64))
]
tfms = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.5),
transforms.ColorJitter(brightness=(1,1.3), contrast=(1,1.3), saturation=0, hue=0)
 ])
 def getDog(self,seed):
xx = torch.Tensor(np.zeros(( 10000)) ).to(device)
 xx[self.index] = 0.999999
xx[np.random.randint(10000)] = 0.000001
img = netG(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3)).detach().cpu().numpy() * 255
self.index =(self.index+1)%10000
return self.tfms(Image.fromarray(img.astype('uint8')) )