Kaggle competition: Generative Dog Images
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )<categorify>
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
import os
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from torch.utils.data import Dataset, DataLoader
import os
import cv2
from tqdm import tqdm_notebook as tqdm
import time
from PIL import Image
import torch.nn.parallel
import torch.optim as optim
import torch.utils.data
import torchvision
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
import matplotlib.image as mpimg
import torchvision
import torchvision.datasets as dset
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms
import torchvision.utils as vutils
from torch.autograd import Variable
import xml.etree.ElementTree as ET
import random
from torch.nn.utils import spectral_norm
from scipy.stats import truncnorm
import torch as th
train['num_over_18'] = 0
train['num_over_18'] = train[train.age >= 18].groupby('idhogar' ).transform("count")
 train['num_over_18'] = train.groupby("idhogar")["num_over_18"].transform("max")
 train['num_over_18'] = train['num_over_18'].fillna(0)
 test['num_over_18'] = 0
test['num_over_18'] = test[test.age >= 18].groupby('idhogar' ).transform("count")
 test['num_over_18'] = test.groupby("idhogar")["num_over_18"].transform("max")
 test['num_over_18'] = test['num_over_18'].fillna(0)
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['v2a1_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['rent_to_over_18'] = df['v2a1']/df['num_over_18']
df.loc[df.num_over_18 == 0, "rent_to_over_18"] = df[df.num_over_18 == 0].v2a1
df['dependency_yes'] = df['dependency'].apply(lambda x: 1 if x == 'yes' else 0)
 df['dependency_no'] = df['dependency'].apply(lambda x: 1 if x == 'no' else 0)
 extract_features(train)
 extract_features(test )<drop_column>
kernel_start_time = time.perf_counter()
batch_size = 32
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )<split>
__print__ = print
def print(string):
os.system(f'echo "{string}"')
 __print__(string )
def split_data(train, y, households, test_percentage=0.20, seed=None):
np.random.seed(seed=seed)
 train2 = train.copy()
cv_hhs = np.random.choice(households, size=int(len(households)* test_percentage), replace=False)
 cv_idx = np.isin(households, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
return X_train, y_train, X_test, y_test<split>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=1977)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.1), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar<define_variables>
class DataGenerator(Dataset):
def __init__(self, directory, transform=None, n_samples=np.inf):
self.directory = directory
self.transform = transform
self.n_samples = n_samples
self.samples = self._load_subfolders_images(directory)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(directory))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =(
 '.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 required_transforms = torchvision.transforms.Compose([
torchvision.transforms.Resize(68),
torchvision.transforms.RandomCrop(64),
])
 imgs = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames)[:min(self.n_samples, 999999999999999)]:
path = os.path.join(root, fname)
 paths.append(path)
 for path in paths:
if is_valid_file(path):
img = dset.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(
 dirname for dirname in os.listdir('.. /input/annotation/Annotation/')if
dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join('.. /input/annotation/Annotation/',
annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
bbox =(xmin, ymin, xmin+w, ymin+w)
 object_img = required_transforms(img.crop(bbox))
imgs.append(object_img)
 return imgs
def __getitem__(self, index):
sample = self.samples[index]
if self.transform is not None:
sample = self.transform(sample)
 return np.asarray(sample)
 def __len__(self):
return len(self.samples)
extra_drop_features = ['geo_manual_elec_LE_2',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil7_COUNT',
'geo_pared_LE_5',
'agg18_estadocivil6_COUNT',
'geo_pared_LE_4',
'agg18_parentesco10_COUNT',
'num_over_18',
'agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'r4t3_to_tamhog',
'agg18_estadocivil3_COUNT',
'geo_pared_LE_3',
'geo_pared_LE_6',
'geo_pared_LE_0',
'geo_pared_LE_7',
'agg18_parentesco7_COUNT',
'agg18_parentesco10_MEAN',
'agg18_parentesco8_MEAN',
'parentesco1',
'rez_esc',
'dependency_yes',
'hacdor',
'hacapo',
'fe_people_weird_stat',
'geo_hogar_mayor',
'geo_eviv_LE_0',
'geo_epared_LE_0',
'geo_epared_LE_2',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_0',
'agg18_parentesco9_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco11_COUNT',
'parentesco_LE',
'agg18_parentesco6_MEAN',
'agg18_parentesco6_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco4_MEAN',
'agg18_parentesco4_COUNT',
'geo_energcocinar_LE_1',
'agg18_parentesco3_COUNT',
'geo_energcocinar_LE_2',
'agg18_parentesco2_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco12_MEAN',
'agg18_parentesco12_COUNT',
'dependency_no'] + ['geo_hogar_nin', 'geo_manual_elec_LE_3']<drop_column>
print('Started Creating Dataset')
 database = '.. /input/all-dogs/all-dogs/'
transform = transforms.Compose([transforms.RandomHorizontalFlip() ,
transforms.RandomRotation(5),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DataGenerator(database, transform=transform,n_samples=25000)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,batch_size=batch_size, num_workers = 4)
 print('Finished Creating Dataset' )
lgb_drop_cols = extra_drop_features + ["idhogar"]
try:
X_train.drop(lgb_drop_cols, axis=1, inplace=True)
 X_test.drop(lgb_drop_cols, axis=1, inplace=True)
 except:
print("Error dropping columns" )<init_hyperparams>
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
def forward(self, x, alpha=1e-8):

 y = x.pow(2.).mean(dim=1, keepdim=True ).add(alpha ).sqrt()
y = x / y
return y
opt_parameters = {'colsample_bytree': 0.89, 'min_child_samples': 90, 'num_leaves': 14, 'subsample': 0.96}
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]<train_model>
class MinibatchStdDev(th.nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
def forward(self, x, alpha=1e-8):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y
def _parallel_fit_estimator(estimator1, X, y, sample_weight=None, threshold=True, **fit_params):
estimator = clone(estimator1)
 X_train, y_train, X_test, y_test = split_data(X, y, households=train_households, seed=None)
 fit_params["eval_set"] = [(X_train,y_train),(X_test,y_test)]
if sample_weight is not None:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
estimator.fit(X_train, y_train, sample_weight=sample_weight, **fit_params)
 else:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
estimator.fit(X_train, y_train, **fit_params)
 if not isinstance(estimator1, ExtraTreesClassifier)and not isinstance(estimator1, RandomForestClassifier):
best_cv_round = np.argmax(estimator.evals_result_['valid']['macroF1'])
 best_cv = np.max(estimator.evals_result_['valid']['macroF1'])
 best_train = estimator.evals_result_['train']['macroF1'][best_cv_round]
else:
best_train = f1_score(y_train, estimator.predict(X_train), average="macro")
 best_cv = f1_score(y_test, estimator.predict(X_test), average="macro")
 print("Train F1:", best_train)
 print("Test F1:", best_cv)
 if threshold:
if(( best_cv > 0.39)and(best_train > 0.75)) or(( best_cv > 0.44)and(best_train > 0.65)) :
return estimator
else:
print("Unacceptable!!! Trying again...")
 return _parallel_fit_estimator(estimator1, X, y, sample_weight=sample_weight, **fit_params)
 else:
return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, threshold=True, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, threshold=threshold, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels):
super(Generator, self ).__init__()
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.prelu1 = nn.PReLU(init=0.01)
 self.conv2 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 8, 4, 2, 1, bias=False))
self.prelu2 = nn.PReLU(init=0.01)
 self.conv3 = spectral_norm(nn.ConvTranspose2d(nfeats * 8, nfeats * 4, 4, 2, 1, bias=False))
self.prelu3 = nn.PReLU(init=0.01)
 self.conv4 = spectral_norm(nn.ConvTranspose2d(nfeats * 4, nfeats * 2, 4, 2, 1, bias=False))
self.prelu4 = nn.PReLU(init=0.01)
 self.conv5 = spectral_norm(nn.ConvTranspose2d(nfeats * 2, nfeats, 4, 2, 1, bias=False))
self.prelu5 = nn.PReLU(init=0.01)
 self.conv6 = spectral_norm(nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x):
x = self.prelu1(self.conv1(x))
x = self.prelu2(self.conv2(x))
x = self.pixnorm(x)
 x = self.prelu3(self.conv3(x))
x = self.pixnorm(x)
 x = self.prelu4(self.conv4(x))
x = self.pixnorm(x)
 x = self.prelu5(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
self.conv1 = nn.Conv2d(nchannels, nfeats, 4, 2, 1, bias=False)
 self.conv2 = spectral_norm(nn.Conv2d(nfeats, nfeats * 2, 4, 2, 1, bias=False))
self.bn2 = nn.BatchNorm2d(nfeats * 2)
 self.conv3 = spectral_norm(nn.Conv2d(nfeats * 2, nfeats * 4, 4, 2, 1, bias=False))
self.bn3 = nn.BatchNorm2d(nfeats * 4)
 self.conv4 = spectral_norm(nn.Conv2d(nfeats * 4, nfeats * 8, 4, 2, 1, bias=False))
self.bn4 = nn.MaxPool2d(2)
 self.batch_discriminator = MinibatchStdDev()
self.pixnorm = PixelwiseNorm()
self.conv5 = spectral_norm(nn.Conv2d(nfeats * 8 +1, 1, 2, 1, 0, bias=False))
def forward(self, x):
x = F.leaky_relu(self.conv1(x), 0.2)
 x = F.leaky_relu(self.bn2(self.conv2(x)) , 0.2)
 x = F.leaky_relu(self.bn3(self.conv3(x)) , 0.2)
 x = F.leaky_relu(self.bn4(self.conv4(x)) , 0.2)
 x = self.batch_discriminator(x)
 x = torch.sigmoid(self.conv5(x))
return x.view(-1, 1)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 batch_size = 32
lr = 1e-3
lr_d = 5e-4
beta1 = 0.5
epochs = 2000
num_iters_d = 2
nfg = 48
nfd = 48
nc = 3
criterion = nn.BCELoss()
nz = 128
fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 real_label = 0.8
fake_label = 0.0
batch_size = train_loader.batch_size
netG = Generator(nz, nfg, nc ).to(device)
 netD = Discriminator(nc, nfd ).to(device)
 optimizerD = optim.Adam(netD.parameters() , lr=lr, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr=lr_d, betas=(beta1, 0.999))
lr_schedulerG = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerG,
T_0=epochs//200, eta_min=0.00005)
 lr_schedulerD = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizerD,
T_0=epochs//200, eta_min=0.00005 )
<compute_train_metric><EOS> 
step = 0
for epoch in range(epochs):
print(f'Epoch: {epoch}')
 if time.perf_counter() - kernel_start_time > 32000:
print("Time limit reached! Stopping kernel!")
 break
for ii,(real_images)in enumerate(train_loader):
for __ in range(num_iters_d):
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)+ torch.zeros(( batch_size, 1)).uniform_(-0.1, 0.2 ).to(device)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)+ torch.zeros(( batch_size, 1)).uniform_(0.0, 0.2 ).to(device)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
step += 1
lr_schedulerG.step(epoch)
 lr_schedulerD.step(epoch)