Kaggle competition: Generative Dog Images
<SOS> metric: PostProcessorKernel Kaggle data source: generative-dog-images<compute_train_metric>
ComputeLB = False
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
'eval_names': ['train', 'valid'],
'callbacks': [lgb.reset_parameter(learning_rate=learning_rate_power_0997)],
'verbose': False,
'categorical_feature': 'auto'}
fit_params['verbose'] = 200<train_model>
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, Reshape, Flatten, concatenate
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD, Adam
<init_hyperparams>
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 0.5
for k in range(5):
annealer = LearningRateScheduler(lambda x: lr)
 h = discriminator.fit([zeros,train_X], train_y, epochs = 10, batch_size=256, callbacks=[annealer], verbose=0)
 print('Epoch',(k+1)*10,'/30 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.533: lr = 0.1
opt_parameters = {
'bagging_fraction': 1.0,
'colsample_bytree': 0.75,
'feature_fraction': 0.1,
'lambda_l1': 5.0,
'lambda_l2': 3.0,
'max_depth': 5,
'min_child_samples': 90,
'min_child_weight': 5.0,
'min_split_gain': 0.001,
'num_leaves': 19,
'subsample': 0.7,
'min_sum_hessian_in_leaf': 1,
'importance_type': 'gain'
}<train_model>
del train_X, train_y, imagesIn
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):

 if sample_weight is not None:
estimator.fit(X, y, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X, y, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self<categorify>
BadMemory = False
if BadMemory:
seed = Input(( 10000,))
x = Dense(2048, activation='elu' )(seed)
 x = Reshape(( 8,8,32))(x)
 x = Conv2D(128,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(64,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(32,(3, 3), activation='elu', padding='same' )(x)
 x = UpSampling2D(( 2, 2))(x)
 x = Conv2D(3,(3, 3), activation='linear', padding='same' )(x)
 generated = Flatten()(x)
 else:
seed = Input(( 10000,))
generated = Dense(12288, activation='linear' )(seed)
 generator = Model(seed, [generated,Reshape(( 10000,))(seed)])
 generator.summary()
class VotingPrefitClassifier(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
self.estimators_ = [x[1] for x in self.estimators]
self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
return self 
<train_model>
discriminator.trainable=False
gan_input = Input(shape=(10000,))
x = generator(gan_input)
 gan_output = discriminator(x)
 gan = Model(gan_input, gan_output)
 gan.get_layer('model_1' ).get_layer('conv' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
 gan.summary()
def train_lgbm_model(X_, y_, random_state_=None, opt_parameters_={}, fit_params_={}):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=random_state_, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters_)
 return clf.fit(X_, y_, **fit_params_)
 clfs = []
inner_seed = 31416
inner_n = 10
outer_seed = 314
outer_n = 10
perf_eval = {'f1_oof': [],
'f1_ave': [],
'f1_std': []}
outer_cv = StratifiedKFold(outer_n, shuffle=True, random_state=outer_seed)
 for n_outer_fold,(outer_trn_idx, outer_val_idx)in enumerate(outer_cv.split(X,y)) :
print('--- Outer loop iteration: {} ---'.format(n_outer_fold))
X_out, y_out = X.iloc[outer_trn_idx], y.iloc[outer_trn_idx]
X_stp, y_stp = X.iloc[outer_val_idx], y.iloc[outer_val_idx]
inner_cv = StratifiedKFold(inner_n, shuffle=True, random_state=inner_seed+n_outer_fold)
 y_oof = pd.Series(np.zeros(shape=(X_out.shape[0],)) ,
index=X_out.index)
 f1_scores_inner = []
for n_inner_fold,(inner_trn_idx, inner_val_idx)in enumerate(inner_cv.split(X_out,y_out)) :
X_trn, y_trn = X_out.iloc[inner_trn_idx], y_out.iloc[inner_trn_idx]
X_val, y_val = X_out.iloc[inner_val_idx], y_out.iloc[inner_val_idx]
fit_params["eval_set"] = [(X_trn,y_trn),(X_stp,y_stp)]
clf = train_lgbm_model(X_trn, y_trn, 314+n_inner_fold, opt_parameters, fit_params)
 y_oof.iloc[inner_val_idx] = clf.predict(X_val)
 f1_scores_inner.append(f1_score(y_val, y_oof.iloc[inner_val_idx], average='macro'))
del clf, X_trn, y_trn, X_val, y_val
perf_eval['f1_oof'].append(f1_score(y_out, y_oof, average='macro'))
perf_eval['f1_ave'].append(np.array(f1_scores_inner ).mean())
 perf_eval['f1_std'].append(np.array(f1_scores_inner ).std())
 fit_params["eval_set"] = [(X_out,y_out),(X_stp,y_stp)]
print('Fit the final model on the outer loop iteration: ')
 clf = train_lgbm_model(X_out, y_out, 314+n_outer_fold, opt_parameters, fit_params)
 clfs.append(( 'lgbm{}'.format(n_outer_fold), clf))
del inner_cv, X_out, y_out, X_stp, y_stp<train_model>
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 0.924
zeros = np.zeros(( 10000,12288))
ep = 1; it = 50
if BadMemory: lr = 0.005
else: lr = 5.
 for k in range(it):
annealer = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = ep, batch_size=256, callbacks=[annealer], verbose=0)
 if(k==it)|(k%10==9):
print('Epoch',(k+1),'/'+str(it)+' - loss =',h.history['loss'][-1])
 plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( -1,64,64,3))
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
if BadMemory:
ep *= 2
if ep>=32: lr = 0.001
if ep>256: ep = 256
else:
if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.5
vc = VotingPrefitClassifier(clfs)
 vc = vc.fit(X,y)
 clf_final = vc.estimators_[0]<compute_test_metric>
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx[self.index] = 0.902
xx[np.random.randint(10000)] = 0.098
img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3))
self.index =(self.index+1)%10000
return Image.fromarray(img.astype('uint8'))