Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<data_type_conversions>
import numpy as np
import pandas as pd
import keras
from keras.layers import Input, Dense, Reshape, Flatten, Dropout,Concatenate
from keras.backend import random_normal,ones_like,zeros_like,mean
from keras.backend import get_session
from keras.layers import BatchNormalization, Activation, ZeroPadding2D
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.convolutional import UpSampling2D, Conv2D
from keras.models import Sequential, Model
from keras.optimizers import Adam
from keras.layers import concatenate
from keras.initializers import TruncatedNormal
from keras.callbacks import LearningRateScheduler, EarlyStopping, History
from PIL import Image
import warnings
import os
import time
from glob import glob
import datetime
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np, pandas as pd, os
import xml.etree.ElementTree as ET
import matplotlib.pyplot as plt, zipfile
from PIL import Image
from glob import glob
dprint('Clean features...')
 cols = ['dependency']
for c in tqdm(cols):
x = df_all[c].values
strs = []
for i, v in enumerate(x):
try:
val = float(v)
 except:
strs.append(v)
 val = np.nan
x[i] = val
strs = np.unique(strs)
 for s in strs:
df_all[c + '_' + s] = df_all[c].apply(lambda x: 1 if x == s else 0)
 df_all[c] = x
df_all[c] = df_all[c].astype(float)
 dprint("Done." )<feature_engineering>
ComputeLB = False
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()
dprint("Extracting features...")
 def extract_features(df):
df['bedrooms_to_rooms'] = df['bedrooms']/df['rooms']
df['rent_to_rooms'] = df['v2a1']/df['rooms']
df['rent_to_bedrooms'] = df['v2a1']/df['bedrooms']
df['tamhog_to_rooms'] = df['tamhog']/df['rooms']
df['tamhog_to_bedrooms'] = df['tamhog']/df['bedrooms']
df['r4t3_to_tamhog'] = df['r4t3']/df['tamhog']
df['r4t3_to_rooms'] = df['r4t3']/df['rooms']
df['r4t3_to_bedrooms'] = df['r4t3']/df['bedrooms']
df['rent_to_r4t3'] = df['v2a1']/df['r4t3']
df['v2a1_to_r4t3'] = df['v2a1']/(df['r4t3'] - df['r4t1'])
 df['hhsize_to_rooms'] = df['hhsize']/df['rooms']
df['hhsize_to_bedrooms'] = df['hhsize']/df['bedrooms']
df['rent_to_hhsize'] = df['v2a1']/df['hhsize']
df['qmobilephone_to_r4t3'] = df['qmobilephone']/df['r4t3']
df['qmobilephone_to_v18q1'] = df['qmobilephone']/df['v18q1']
extract_features(train)
 extract_features(test)
 dprint("Done." )<data_type_conversions>
IMG_SIZE = Input(( 12288,))
IMG_SIZE_2 = Input(( 10000,))
NOISE_SIZE = 10000
BATCH_SIZE = 64
def encode_data(df):
yes_no_map = {'no': 0, 'yes': 1}
df['dependency'] = df['dependency'].replace(yes_no_map ).astype(np.float32)
 df['edjefe'] = df['edjefe'].replace(yes_no_map ).astype(np.float32)
 df['edjefa'] = df['edjefa'].replace(yes_no_map ).astype(np.float32)
 df['idhogar'] = LabelEncoder().fit_transform(df['idhogar'])
<categorify>
def discriminatorFunction() :
input_layer = Dense(12288, activation='sigmoid' )(IMG_SIZE_2)
 input_layer = Reshape(( 2,12288,1))(concatenate([IMG_SIZE,input_layer]))
discriminator = Conv2D(filters = 1, kernel_size=[2,1],use_bias=False, name = 'layer_1' )(input_layer)
 out = Flatten()(discriminator)
 return out
dprint("Encoding Data.... ")
 encode_data(train)
 encode_data(test)
 dprint("Done..." )<feature_engineering>
print("Discriminator")
 model = discriminatorFunction()
model_discriminator = Model([IMG_SIZE,IMG_SIZE_2], model)
 model_discriminator.get_layer('layer_1' ).trainable = False
model_discriminator.get_layer('layer_1' ).set_weights([np.array([[[[-1.0 ]]],[[[1.0]]]])])
 model_discriminator.summary()
model_discriminator.compile(optimizer='adam', loss='binary_crossentropy' )
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['Id', 'idhogar'], axis=1, inplace=True)
 df.drop(['hhsize', 'female', 'area2'], axis=1, inplace=True)
 return df 
<feature_engineering>
def GeneratorFunction(noise_shape=(NOISE_SIZE,)) :
input_layer = Input(noise_shape)
 generated = Dense(12288, activation='linear' )(input_layer)
 model = Model(inputs=input_layer,outputs = [generated,Reshape(( 10000,))(input_layer)])
 model.summary()
return model
dprint("Do_feature Engineering.... ")
 train = do_features(train)
 test = do_features(test)
 dprint("Done.... " )<categorify>
print("Generator")
 model_generator = GeneratorFunction(noise_shape=(NOISE_SIZE,))
dprint("Fill Na value.... ")
 train = train.fillna(0)
 test = test.fillna(0)
 dprint("Done.... " )<prepare_x_and_y>
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 0.5
for k in range(5):
LR_Scheduler = LearningRateScheduler(lambda x: lr)
 h = model_discriminator.fit([zeros,train_X], train_y, epochs = 100,batch_size = BATCH_SIZE, callbacks=[LR_Scheduler], verbose=0)
 print('Epoch',(k+1)*10,'/50 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.533: lr = 0.1
cols_to_drop = [
id_name,
target_name,
]
X = train.drop(cols_to_drop, axis=1, errors='ignore')
 y = train[target_name].values 
<train_model>
del train_X, train_y, imagesIn
<init_hyperparams>
model_discriminator.trainable = False
z = Input(shape=(NOISE_SIZE,))
img = model_generator(z)
 real = model_discriminator(img)
 gan = Model(z, real)
 gan.get_layer('model_1' ).get_layer('layer_1' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
 print("Model created based on Discriminator and Generator")
 gan.summary()
params = {'bagging_fraction': 0.9957236684465528,
'colsample_bytree': 0.7953949538181928,
'feature_fraction': 0.7333800304661316,
'lambda_l1': 1.79753950286893,
'lambda_l2': 1.710590311253639,
'max_depth': 6,
'min_child_samples': 48,
'min_child_weight': 48.94067592560281,
'min_split_gain': 0.016737988780906453,
'num_leaves': 34,
'subsample': 0.9033449610388691}<choose_model_class>
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 1
zeros = np.zeros(( 10000,12288))
Steps_per_epoch = 50
print("Training begins...Total steps per epoch: {}".format(Steps_per_epoch))
lgb_model = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.2, objective='multiclass',
silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree= 0.93,
min_child_samples= 56, num_leaves= 19,
subsample= 0.84)
 lgb_model.set_params(**params )<train_model>
warnings.filterwarnings("ignore")
 lr = 5.
 for step in range(Steps_per_epoch):
LR_Scheduler = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = 1, batch_size=256, callbacks=[LR_Scheduler], verbose=0)
 if(step<10)|(step%5==4):
print("Step: {}/{} [G loss: {:.4f}]".format(
(step+1)*10, Steps_per_epoch*10, h.history['loss'][-1]))
if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.01
if step<10:
plt.figure(figsize=(15,3))
for j in range(5):
zz = np.zeros(( 10000))
zz[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = model_generator.predict(zz.reshape(( -1,10000)))[0].reshape(( -1,64,64,3))
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
dprint("Prepared Model.....")
 lgb_model.fit(X, y)
 dprint("Done Model.....")
 dprint("Predict Test Value.....")
 target_lgb = lgb_model.predict(test)
 dprint("Done Prediction....." )<compute_test_metric>
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx[self.index] = 0.999
xx[np.random.randint(10000)] = 0.001
img = model_generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3))
self.index =(self.index+1)%10000
return Image.fromarray(img.astype('uint8'))