Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<save_to_csv>
import numpy as np
import pandas as pd
import os
import cv2
import random
import matplotlib.pyplot as plt
from PIL import Image
from skimage.util import random_noise
import xml.etree.ElementTree as ET
import torch
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from torch.autograd import Variable
from collections import OrderedDict
from torch.utils.data import TensorDataset, DataLoader,Dataset
from tqdm import tqdm_notebook as tqdm
sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] = preds.as_data_frame().values.flatten()
sample_submission.to_csv('h2o_submission_8.csv', index=False )<set_options>
def seed_everything(seed=19960720):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
random_state = 6
np.random.seed(random_state)
 warnings.filterwarnings('ignore' )<set_options>
def summary(model, input_size, batch_size=-1, device="cuda"):
def register_hook(module):
def hook(module, input, output):
class_name = str(module.__class__ ).split(".")[-1].split("'")[0]
module_idx = len(summary)
 m_key = "%s-%i" %(class_name, module_idx + 1)
 summary[m_key] = OrderedDict()
summary[m_key]["input_shape"] = list(input[0].size())
 summary[m_key]["input_shape"][0] = batch_size
if isinstance(output,(list, tuple)) :
summary[m_key]["output_shape"] = [
[-1] + list(o.size())[1:] for o in output
]
else:
summary[m_key]["output_shape"] = list(output.size())
 summary[m_key]["output_shape"][0] = batch_size
params = 0
if hasattr(module, "weight")and hasattr(module.weight, "size"):
params += torch.prod(torch.LongTensor(list(module.weight.size())))
 summary[m_key]["trainable"] = module.weight.requires_grad
if hasattr(module, "bias")and hasattr(module.bias, "size"):
params += torch.prod(torch.LongTensor(list(module.bias.size())))
 summary[m_key]["nb_params"] = params
if(
 not isinstance(module, nn.Sequential)
 and not isinstance(module, nn.ModuleList)
 and not(module == model)
):
hooks.append(module.register_forward_hook(hook))
device = device.lower()
assert device in [
"cuda",
"cpu",
], "Input device is not valid, please specify 'cuda' or 'cpu'"
if device == "cuda" and torch.cuda.is_available() :
dtype = torch.cuda.FloatTensor
else:
dtype = torch.FloatTensor
if isinstance(input_size, tuple):
input_size = [input_size]
x = [torch.rand(2, *in_size ).type(dtype)for in_size in input_size]
summary = OrderedDict()
hooks = []
model.apply(register_hook)
 model(*x)
 for h in hooks:
h.remove()
print("----------------------------------------------------------------")
 line_new = "{:>20} {:>25} {:>15}".format("Layer(type)", "Output Shape", "Param
print(line_new)
 print("================================================================")
 total_params = 0
total_output = 0
trainable_params = 0
for layer in summary:
line_new = "{:>20} {:>25} {:>15}".format(
 layer,
str(summary[layer]["output_shape"]),
"{0:,}".format(summary[layer]["nb_params"]), 
)
 total_params += summary[layer]["nb_params"]
total_output += np.prod(summary[layer]["output_shape"])
 if "trainable" in summary[layer]:
if summary[layer]["trainable"] == True:
trainable_params += summary[layer]["nb_params"]
print(line_new)
 total_input_size = abs(np.prod(input_size)* batch_size * 4./(1024 ** 2.))
 total_output_size = abs(2.* total_output * 4./(1024 ** 2.))
 total_params_size = abs(total_params.numpy() * 4./(1024 ** 2.))
 total_size = total_params_size + total_output_size + total_input_size
print("================================================================")
 print("Total params: {0:,}".format(total_params))
print("Trainable params: {0:,}".format(trainable_params))
print("Non-trainable params: {0:,}".format(total_params - trainable_params))
print("----------------------------------------------------------------")
 print("Input size(MB): %0.2f" % total_input_size)
 print("Forward/backward pass size(MB): %0.2f" % total_output_size)
 print("Params size(MB): %0.2f" % total_params_size)
 print("Estimated Total Size(MB): %0.2f" % total_size)
 print("----------------------------------------------------------------")
%matplotlib inline
get_ipython().run_line_magic('matplotlib', 'inline')
 font = {
'family': 'serif',
'serif': ['Computer Modern Roman'],
'weight' : 'regular',
'size' : 18
}
plt.rc('font', **font)
 plt.rc('text', usetex=False)
 color_map = 'viridis'<load_from_csv>
class Generator(nn.Module):
def __init__(self, nz=128, channels=3):
super(Generator, self ).__init__()
self.nz = nz
self.channels = channels
def convlayer(n_input, n_output, k_size=4, stride=2, padding=0):
block = [
nn.ConvTranspose2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False),
nn.BatchNorm2d(n_output),
nn.ReLU(inplace=True),
]
return block
self.model = nn.Sequential(
 *convlayer(self.nz, 1024, 4, 1, 0),
*convlayer(1024, 512, 4, 2, 1),
*convlayer(512, 256, 4, 2, 1),
*convlayer(256, 128, 4, 2, 1),
*convlayer(128, 64, 4, 2, 1),
nn.ConvTranspose2d(64, self.channels, 3, 1, 1),
nn.Tanh() 
)
 def forward(self, z):
z = z.view(-1, self.nz, 1, 1)
 img = self.model(z)
 return img
df_train = pd.read_csv('/kaggle/input/train.csv')
 df_test = pd.read_csv('/kaggle/input/test.csv' )<train_model>
class Discriminator(nn.Module):
def __init__(self, channels=3):
super(Discriminator, self ).__init__()
self.channels = channels
def convlayer(n_input, n_output, k_size=4, stride=2, padding=0, bn=False):
block = [nn.Conv2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False)]
if bn:
block.append(nn.BatchNorm2d(n_output))
block.append(nn.LeakyReLU(0.2, inplace=True))
return block
self.model = nn.Sequential(
 *convlayer(self.channels, 64, 3, 1, 1),
*convlayer(64, 128, 4, 2, 1),
*convlayer(128, 256, 4, 2, 1, bn=True),
*convlayer(256, 512, 4, 2, 1, bn=True),
*convlayer(512, 1024, 4, 2, 1, bn=True),
nn.Conv2d(1024, 1, 4, 1, 0, bias=False), 
)
 def forward(self, imgs):
logits = self.model(imgs)
 out = torch.sigmoid(logits)
 return out.view(-1, 1 )
print('Training data shape: {}'.format(df_train.shape))
print('Training data shape: {}'.format(df_test.shape))<count_missing_values>
image = glob.glob('.. /input/all-dogs/all-dogs/*')
 breed = glob.glob('.. /input/annotation/Annotation/*')
 annot = glob.glob('.. /input/annotation/Annotation/*/*')
 print(len(image), len(breed), len(annot))
print('Is null on train: {}'.format(df_train.isnull().any().any()))
print('Is null on test: {}'.format(df_test.isnull().any().any()))<drop_column>
def get_image(annot):

 img_path = '.. /input/all-dogs/all-dogs/'
file = annot.split('/')
 img_class = file[-2]
img_filename = img_path+file[-1]+'.jpg'
return img_filename, img_class
col_to_drop = list(df_train.columns[df_train.columns.str.startswith('ps_calc_')])
 df_train = df_train.drop(col_to_drop, axis=1)
 df_test = df_test.drop(col_to_drop, axis=1 )<create_dataframe>
select_dogs = []
select_bbox = []
def get_missing_features(df):
missings = pd.DataFrame([], columns=['feature', 'no_recoreds', 'percentage'])
 total_rows = df.shape[0]
index = 0
for feature in list(df):
total_nulls = df[feature].isnull().sum()
if total_nulls > 0:
missings_perc = total_nulls / total_rows
missings.loc[index] = [feature, total_nulls, missings_perc]
index += 1
missings = missings.sort_values('no_recoreds', ascending=False)
 return missings<feature_engineering>
def get_class_dict(breed):
class_dict = {}
for i in range(len(breed)) :
class_ = breed[i].split('/')
 class_dict[class_[-1]] = i
return class_dict
class_dict = get_class_dict(breed )
df_missings = get_missing_features(df_train)
 print(df_missings )<data_type_conversions>
def get_cleaned_data(image, annot, class_dict):
select_dogs = []
select_bbox = []
select_labels = []
for i in range(len(image)) :
bbox = get_bbox(annot[i])
 bbox = bbox[0]
dog, dog_class = get_image(annot[i])
 if dog == '.. /input/all-dogs/all-dogs/n02105855_2933.jpg':
continue
im = Image.open(dog)
 xdiff = abs(bbox[2] - bbox[0])
 ydiff = abs(bbox[3] - bbox[1])
 if(( abs(ydiff - xdiff)/min(ydiff, xdiff)<=0.25)and(( xdiff>=64)and(ydiff>=64))):
select_dogs.append(dog)
 select_bbox.append(bbox)
 select_labels.append(class_dict[dog_class])
 return select_dogs, select_bbox, select_labels
select_dogs, select_bbox, select_labels = get_cleaned_data(image, annot, class_dict )
for i, feature in enumerate(list(df_train.drop(['id'], axis=1))):
if df_train[feature].isnull().sum() > 0:
df_train[feature].fillna(df_train[feature].mode() [0],inplace=True)
 for i, feature in enumerate(list(df_test.drop(['id'], axis=1))):
if df_test[feature].isnull().sum() > 0:
df_test[feature].fillna(df_test[feature].mode() [0],inplace=True )<feature_engineering>
def prepare_labels(y):
values = np.array(y)
 onehot_encoder = OneHotEncoder(sparse=False, categories='auto')
 onehot_encoded = onehot_encoder.fit_transform(values.reshape(values.shape[0], 1))
y = onehot_encoded
return y
labels_encoded = prepare_labels(select_labels )
get_missing_features(df_train)
 get_missing_features(df_test )<count_unique_values>
class dogs_Dataset(Dataset):
def __init__(self, datafolder, datatype='train', index=[], \
transform = transforms.Compose([transforms.RandomResizedCrop(128),transforms.ToTensor() ])) :
self.datafolder = datafolder
self.datatype = datatype
self.image_files_list = [select_dogs[i] for i in index]
self.bbox_list = [select_bbox[i] for i in index]
self.label_list = [labels_encoded[i] for i in index]
self.transform = transform
self.imgs = []
for idx in range(len(self.image_files_list)) :
img_name = self.image_files_list[idx]
bbox = self.bbox_list[idx]
img = Image.open(img_name)
 img = img.crop(bbox)
 self.imgs.append(img)
 def __len__(self):
return len(self.imgs)
 def __getitem__(self, idx):
image = self.imgs[idx]
image = self.transform(image)
 if self.datatype == 'train':
label = self.label_list[idx]
else:
label = torch.zeros(1)
 return image, label
cat_cols = [col for col in df_train.columns if '_cat' in col]
dummed_cols = []
for cat_col in cat_cols:
unique_values = len(np.unique(df_train[cat_col]))
if unique_values < 50:
dummed_cols.append(cat_col)
 print('{} has {} unique values'.format(cat_col, unique_values))<prepare_x_and_y>
imgs, label = next(iter(train_loader))
imgs = imgs.numpy().transpose(0, 2, 3, 1 )
id_test = df_test['id'].values
y = df_train['target'].values
df_train = df_train.drop(['target','id'], axis = 1)
 df_test = df_test.drop(['id'], axis = 1)
 cat_features = [a for a in df_train.columns if a.endswith('cat')]
for column in cat_features:
temp = pd.get_dummies(pd.Series(df_train[column]))
df_train = pd.concat([df_train,temp],axis=1)
 df_train = df_train.drop([column],axis=1)
 for column in cat_features:
temp = pd.get_dummies(pd.Series(df_test[column]))
df_test = pd.concat([df_test,temp],axis=1)
 df_test = df_test.drop([column],axis=1)
 print(df_train.values.shape, df_test.values.shape )<statistical_test>
LR_G = 0.0005
LR_D = 0.0005
beta1 = 0.5
real_label = 0.8
fake_label = 0.0
nz = 128
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini
def gini_xgb(preds, dtrain):
labels = dtrain.get_label()
gini_score = -eval_gini(labels, preds)
 return [('gini', gini_score)]
def gini_normalized(a, p):
return gini(a, p)/ gini(a, a )<predict_on_test>
netG = Generator(nz ).to(device)
 netD = Discriminator().to(device)
 criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr=LR_D, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr=LR_G, betas=(beta1, 0.999))
fixed_noise = torch.randn(25, nz, 1, 1, device=device )
class Create_ensemble(object):
def __init__(self, n_splits, base_models):
self.n_splits = n_splits
self.base_models = base_models
def predict(self, X, y, T):
X = np.array(X)
 y = np.array(y)
 T = np.array(T)
 folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=2016 ).split(X, y))
S_train = np.zeros(( X.shape[0], len(self.base_models)))
 S_test = np.zeros(( T.shape[0], len(self.base_models)))
 for i, clf in enumerate(self.base_models):
S_test_i = np.zeros(( T.shape[0], self.n_splits))
for j,(train_idx, valid_idx)in enumerate(folds):
X_train = X[train_idx]
y_train = y[train_idx]
X_valid = X[valid_idx]
y_valid = y[valid_idx]
clf.fit(X_train, y_train)
 valid_pred = clf.predict_proba(X_valid)[:,1]
S_train[valid_idx, i] = valid_pred
S_test_i[:, j] = clf.predict_proba(T)[:,1]
print("
Training Gini for model {} : {}".format(i, eval_gini(y, S_train[:,i])))
 S_test[:, i] = S_test_i.mean(axis=1)
 return S_train, S_test<init_hyperparams>
epochs = 780
lgb_params = {}
lgb_params['learning_rate'] = 0.02
lgb_params['n_estimators'] = 700
lgb_params['max_bin'] = 15
lgb_params['subsample'] = 0.8
lgb_params['subsample_freq'] = 10
lgb_params['colsample_bytree'] = 0.8
lgb_params['min_child_samples'] = 800
lgb_params['random_state'] = 99
lgb_params['scale_pos_weight'] = 3
lgb_params2 = {}
lgb_params2['learning_rate'] = 0.02
lgb_params2['n_estimators'] = 900
lgb_params2['max_bin'] = 20
lgb_params2['subsample'] = 0.8
lgb_params2['subsample_freq'] = 10
lgb_params2['colsample_bytree'] = 0.8
lgb_params2['min_child_samples'] = 600
lgb_params2['random_state'] = 99
lgb_params2['scale_pos_weight'] = 3
lgb_model = LGBMClassifier(**lgb_params)
 lgb_model2 = LGBMClassifier(**lgb_params2 )<predict_on_test>
for epoch in range(epochs):
for ii,(real_images, train_labels)in enumerate(train_loader):
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
if(ii+1)%(len(train_loader)//2)== 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii+1, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2))
lgb_stack = Create_ensemble(n_splits = 5, base_models = [lgb_model, lgb_model2])
 X = df_train
Y = y
T = df_test
lgb_train_pred, lgb_test_pred = lgb_stack.predict(X, Y, T )<save_to_csv>
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = lgb_test_pred.mean(axis=1)
 sub.to_csv('submission.csv', float_format='%.6f', index=False )<install_modules>
gen_z = torch.randn(32, nz, 1, 1, device=device)
 gen_images = netG(gen_z ).to("cpu" ).clone().detach()
gen_images = gen_images.numpy().transpose(0, 2, 3, 1 )
<install_modules><EOS> 
if not os.path.exists('.. /output_images'):
os.mkdir('.. /output_images')
 im_batch_size = 50
n_images=10000
for i_batch in range(0, n_images, im_batch_size):
z = truncated_normal(( im_batch_size, nz, 1, 1), threshold=1)
 gen_z = torch.from_numpy(z ).float().to(device)
 gen_images =(netG(gen_z)+1.) /2.
 images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :], os.path.join('.. /output_images', f'image_{i_batch+i_image:05d}.png'))
shutil.make_archive('images', 'zip', '.. /output_images' )