Kaggle competition: APTOS 2019 Blindness Detection
<save_to_csv>
KAGGLE_DIR = '.. /input/aptos2019-blindness-detection/'
TRAIN_DF_PATH = KAGGLE_DIR + "train.csv"
TEST_DF_PATH = KAGGLE_DIR + 'test.csv'
TRAIN_IMG_PATH = KAGGLE_DIR + "train_images/"
TEST_IMG_PATH = KAGGLE_DIR + 'test_images/'
SAVED_MODEL_NAME = 'effnet_modelB5.h5'
seed = 1234
rn.seed(seed)
 np.random.seed(seed)
 tf.set_random_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 t_start = time.time()
feat_importance, clf, sub_preds, oof_preds = kfold_lightgbm(train_df, test_df, num_folds= 5, lr=0.01,
stratified= True, params=params, 
)
 feat_importance.to_csv('fimp_all.csv' )<save_to_csv>
print("Image IDs and Labels(TRAIN)")
 train_df = pd.read_csv(TRAIN_DF_PATH)
 train_df['id_code'] = train_df['id_code'] + ".png"
print(f"Training images: {train_df.shape[0]}")
 display(train_df.head())
 print("Image IDs(TEST)")
 test_df = pd.read_csv(TEST_DF_PATH)
 test_df['id_code'] = test_df['id_code'] + ".png"
print(f"Testing Images: {test_df.shape[0]}")
 display(test_df.head() )
train_df['target'] = train_df['target'].sample(frac=1,random_state=1976 ).values
feat_importance_random, clf, sub_preds, oof_preds = kfold_lightgbm(train_df, test_df, num_folds= 5, lr=0.01,
stratified= True, params=params, 
)
 feat_importance_random.to_csv('fimp_all_random.csv' )<compute_test_metric>
IMG_WIDTH = 456
IMG_HEIGHT = 456
CHANNELS = 3
<find_best_params>
def get_preds_and_labels(model, generator):

 preds = []
labels = []
for _ in range(int(np.ceil(generator.samples / BATCH_SIZE))):
x, y = next(generator)
 preds.append(model.predict(x))
labels.append(y)
 return np.concatenate(preds ).ravel() , np.concatenate(labels ).ravel()
feat_importance.set_index('f',inplace=True)
 feat_importance_random.set_index('f',inplace=True )<feature_engineering>
class Metrics(Callback):

 def on_train_begin(self, logs={}):

 self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):

 y_pred, labels = get_preds_and_labels(model, val_generator)
 y_pred = np.rint(y_pred ).astype(np.uint8 ).clip(0, 4)
 _val_kappa = cohen_kappa_score(labels, y_pred, weights='quadratic')
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {round(_val_kappa, 4)}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save(SAVED_MODEL_NAME)
 return
importance = pd.DataFrame()
statistics = []
pvalues = []
for f in tqdm(feat_importance.index.values):
statistic, pvalue = stats.ks_2samp(feat_importance.loc[f], feat_importance_random.loc[f])
 statistics.append(statistic)
 pvalues.append(pvalue)
 importance['sum']=feat_importance.sum(axis=1 ).values
importance['random_sum']=feat_importance_random.sum(axis=1 ).values
importance['f']=feat_importance.index.values
importance.set_index('f',inplace=True)
 importance['sum']=importance['sum']+importance['random_sum'].min()
importance['random_sum']=importance['random_sum']+importance['random_sum'].min()
importance['fc']=(importance['sum']+1)/(importance['random_sum']+1)
 importance['pval']=pvalues
importance['ks_stat']=statistics
padj = multipletests(importance['pval'], method='bonferroni')
 importance['padj']=padj[1]
importance.sort_values('pval' ).head()<filter>
y_labels = train_df['diagnosis'].values
importance.loc['num_ingredients']<filter>
BATCH_SIZE = 4
train_datagen = ImageDataGenerator(rotation_range=360,
horizontal_flip=True,
vertical_flip=True,
validation_split=0.15,
preprocessing_function=preprocess_image,
rescale=1 / 128.)
train_generator = train_datagen.flow_from_dataframe(train_df,
x_col='id_code',
y_col='diagnosis',
directory = TRAIN_IMG_PATH,
target_size=(IMG_WIDTH, IMG_HEIGHT),
batch_size=BATCH_SIZE,
class_mode='other',
subset='training')
 val_generator = train_datagen.flow_from_dataframe(train_df,
x_col='id_code',
y_col='diagnosis',
directory = TRAIN_IMG_PATH,
target_size=(IMG_WIDTH, IMG_HEIGHT),
batch_size=BATCH_SIZE,
class_mode='other',
subset='validation' )
feature_identified = list(importance[importance['fc']>1].index.values)
 print(len(feature_identified))<train_model>
class RAdam(keras.optimizers.Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999,
epsilon=None, decay=0., weight_decay=0., amsgrad=False,
total_steps=0, warmup_proportion=0.1, min_lr=0., **kwargs):
super(RAdam, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.weight_decay = K.variable(weight_decay, name='weight_decay')
 self.total_steps = K.variable(total_steps, name='total_steps')
 self.warmup_proportion = K.variable(warmup_proportion, name='warmup_proportion')
 self.min_lr = K.variable(lr, name='min_lr')
 if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.initial_weight_decay = weight_decay
self.initial_total_steps = total_steps
self.amsgrad = amsgrad
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
lr = self.lr
if self.initial_decay > 0:
lr = lr *(1./(1.+ self.decay * K.cast(self.iterations, K.dtype(self.decay))))
t = K.cast(self.iterations, K.floatx())+ 1
if self.initial_total_steps > 0:
warmup_steps = self.total_steps * self.warmup_proportion
decay_steps = self.total_steps - warmup_steps
lr = K.switch(
 t <= warmup_steps,
lr *(t / warmup_steps),
lr *(1.0 - K.minimum(t, decay_steps)/ decay_steps), 
)
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='m_' + str(i)) for(i, p)in enumerate(params)]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='v_' + str(i)) for(i, p)in enumerate(params)]
if self.amsgrad:
vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p), name='vhat_' + str(i)) for(i, p)in enumerate(params)]
else:
vhats = [K.zeros(1, name='vhat_' + str(i)) for i in range(len(params)) ]
self.weights = [self.iterations] + ms + vs + vhats
beta_1_t = K.pow(self.beta_1, t)
 beta_2_t = K.pow(self.beta_2, t)
 sma_inf = 2.0 /(1.0 - self.beta_2)- 1.0
sma_t = sma_inf - 2.0 * t * beta_2_t /(1.0 - beta_2_t)
 for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 m_corr_t = m_t /(1.0 - beta_1_t)
 if self.amsgrad:
vhat_t = K.maximum(vhat, v_t)
 v_corr_t = K.sqrt(vhat_t /(1.0 - beta_2_t)+ self.epsilon)
 self.updates.append(K.update(vhat, vhat_t))
else:
v_corr_t = K.sqrt(v_t /(1.0 - beta_2_t)+ self.epsilon)
 r_t = K.sqrt(( sma_t - 4.0)/(sma_inf - 4.0)* 
(sma_t - 2.0)/(sma_inf - 2.0)*
sma_inf / sma_t)
 p_t = K.switch(sma_t > 5, r_t * m_corr_t / v_corr_t, m_corr_t)
 if self.initial_weight_decay > 0:
p_t += self.weight_decay * p
p_t = p - lr * p_t
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {
'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.weight_decay)) ,
'epsilon': self.epsilon,
'amsgrad': self.amsgrad,
'total_steps': float(K.get_value(self.total_steps)) ,
'warmup_proportion': float(K.get_value(self.warmup_proportion)) ,
'min_lr': float(K.get_value(self.min_lr)) ,
}
base_config = super(RAdam, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
train_set = lgb.Dataset(train_df[feature_identified], y)
 print(train_set)
 min_value = 100000
iteration = 0
def objective(params, n_folds = 5):
global iteration
iteration+=1

 params['num_leaves']=int(params['num_leaves'])
 params['min_data_in_leaf']=int(params['min_data_in_leaf'])
 params['max_depth']=int(params['max_depth'])
 params['min_child_weight']=int(params['min_child_weight'])
 params['colsample_bytree']=round(params['colsample_bytree'],5)
 params['reg_alpha']=round(params['reg_alpha'],5)
 params['reg_lambda']=round(params['reg_lambda'],5)
 params['subsample']=round(params['subsample'],5)
 params['learning_rate']=round(params['learning_rate'],5)
 params['learning_rate']=round(params['learning_rate'],5)
 cv_results = lgb.cv(params,
train_set,
nfold = n_folds,
num_boost_round = 1000000,
early_stopping_rounds = 200,
metrics='multi_logloss',
seed = 50,
stratified=False)
 best_score = min(cv_results['multi_logloss-mean'])
 optimal_rounds =len(cv_results['multi_logloss-mean'])
 global min_value
if str(best_score)== '-inf':
best_score = min_value+1
if best_score < 0:
best_score = min_value+1
if best_score < min_value:
min_value = best_score
print(f'Round: {iteration}
the score is: {best_score:.3f}
the best score so far {min_value:.3f}')
 loss= best_score
of_connection = open(out_file, 'a')
 writer = csv.writer(of_connection)
 writer.writerow([loss, params, best_score, optimal_rounds])
 of_connection.close()
return {'loss': loss, 'params': params, 'status': STATUS_OK}
space = {
'subsample': hp.uniform('subsample', 0.2, 1),
'num_leaves': hp.quniform('num_leaves', 5, 150, 1),
'learning_rate': hp.loguniform('learning_rate', np.log(0.001), np.log(0.9)) ,
'min_child_samples': hp.quniform('min_child_samples', 1, 100, 1),
'min_data_in_leaf': hp.quniform('min_data_in_leaf', 1, 100, 1),
'reg_alpha': hp.uniform('reg_alpha', 0.0, 1.0),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree': hp.uniform('colsample_bytree', 0.2, 1.0),
'max_depth':hp.uniform('max_depth', 2, 15),
'min_child_weight':hp.quniform('min_child_weight', 1, 100, 1),
}
out_file = 'gbm_trials_essential.csv'
of_connection = open(out_file, 'w')
 writer = csv.writer(of_connection)
 writer.writerow(['loss', 'params', 'best_score', 'optimal_rounds'])
 of_connection.close()
MAX_EVALS = 150
best = fmin(fn = objective,
space = space,
algo = tpe.suggest,
max_evals = MAX_EVALS,
trials = bayes_trials)
 print(best )<init_hyperparams>
class GroupNormalization(Layer):

 def __init__(self,
groups=32,
axis=-1,
epsilon=1e-5,
center=True,
scale=True,
beta_initializer='zeros',
gamma_initializer='ones',
beta_regularizer=None,
gamma_regularizer=None,
beta_constraint=None,
gamma_constraint=None,
**kwargs):
super(GroupNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.groups = groups
self.axis = axis
self.epsilon = epsilon
self.center = center
self.scale = scale
self.beta_initializer = initializers.get(beta_initializer)
 self.gamma_initializer = initializers.get(gamma_initializer)
 self.beta_regularizer = regularizers.get(beta_regularizer)
 self.gamma_regularizer = regularizers.get(gamma_regularizer)
 self.beta_constraint = constraints.get(beta_constraint)
 self.gamma_constraint = constraints.get(gamma_constraint)
 def build(self, input_shape):
dim = input_shape[self.axis]
if dim is None:
raise ValueError('Axis ' + str(self.axis)+ ' of '
'input tensor should have a defined dimension '
'but the layer received an input with shape ' +
str(input_shape)+ '.')
 if dim < self.groups:
raise ValueError('Number of groups(' + str(self.groups)+ ')cannot be '
'more than the number of channels(' +
str(dim)+ ' ).')
 if dim % self.groups != 0:
raise ValueError('Number of groups(' + str(self.groups)+ ')must be a '
'multiple of the number of channels(' +
str(dim)+ ' ).')
 self.input_spec = InputSpec(ndim=len(input_shape),
axes={self.axis: dim})
 shape =(dim,)
 if self.scale:
self.gamma = self.add_weight(shape=shape,
name='gamma',
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint)
 else:
self.gamma = None
if self.center:
self.beta = self.add_weight(shape=shape,
name='beta',
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint)
 else:
self.beta = None
self.built = True
def call(self, inputs, **kwargs):
input_shape = K.int_shape(inputs)
 tensor_input_shape = K.shape(inputs)
 reduction_axes = list(range(len(input_shape)))
 del reduction_axes[self.axis]
broadcast_shape = [1] * len(input_shape)
 broadcast_shape[self.axis] = input_shape[self.axis] // self.groups
broadcast_shape.insert(1, self.groups)
 reshape_group_shape = K.shape(inputs)
 group_axes = [reshape_group_shape[i] for i in range(len(input_shape)) ]
group_axes[self.axis] = input_shape[self.axis] // self.groups
group_axes.insert(1, self.groups)
 group_shape = [group_axes[0], self.groups] + group_axes[2:]
group_shape = K.stack(group_shape)
 inputs = K.reshape(inputs, group_shape)
 group_reduction_axes = list(range(len(group_axes)))
 group_reduction_axes = group_reduction_axes[2:]
mean = K.mean(inputs, axis=group_reduction_axes, keepdims=True)
 variance = K.var(inputs, axis=group_reduction_axes, keepdims=True)
 inputs =(inputs - mean)/(K.sqrt(variance + self.epsilon))
inputs = K.reshape(inputs, group_shape)
 outputs = inputs
if self.scale:
broadcast_gamma = K.reshape(self.gamma, broadcast_shape)
 outputs = outputs * broadcast_gamma
if self.center:
broadcast_beta = K.reshape(self.beta, broadcast_shape)
 outputs = outputs + broadcast_beta
outputs = K.reshape(outputs, tensor_input_shape)
 return outputs
def get_config(self):
config = {
'groups': self.groups,
'axis': self.axis,
'epsilon': self.epsilon,
'center': self.center,
'scale': self.scale,
'beta_initializer': initializers.serialize(self.beta_initializer),
'gamma_initializer': initializers.serialize(self.gamma_initializer),
'beta_regularizer': regularizers.serialize(self.beta_regularizer),
'gamma_regularizer': regularizers.serialize(self.gamma_regularizer),
'beta_constraint': constraints.serialize(self.beta_constraint),
'gamma_constraint': constraints.serialize(self.gamma_constraint)
 }
base_config = super(GroupNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
params = {}
params['num_leaves']=int(best['num_leaves'])
 params['min_data_in_leaf']=int(best['min_data_in_leaf'])
 params['max_depth']=int(best['max_depth'])
 params['min_child_weight']=int(best['min_child_weight'])
 params['colsample_bytree']=round(best['colsample_bytree'],5)
 params['reg_alpha']=round(best['reg_alpha'],5)
 params['reg_lambda']=round(best['reg_lambda'],5)
 params['subsample']=round(best['subsample'],5)
 params['learning_rate']=round(best['learning_rate'],5 )<compute_train_metric>
effnet = EfficientNetB5(weights=None,
include_top=False,
input_shape=(IMG_WIDTH, IMG_HEIGHT, CHANNELS))
effnet.load_weights('.. /input/efficientnet-keras-weights-b0b5/efficientnet-b5_imagenet_1000_notop.h5' )
train_df['target']=y_bk
feat_importance, clf, sub_preds, oof_preds = kfold_lightgbm(train_df[feature_identified+['target']],
test_df[feature_identified+['target']],
num_folds= 5, lr=0.01, stratified= True,params=params, 
)
 feat_importance.to_csv('fimp_final.csv' )<find_best_score>
for i, layer in enumerate(effnet.layers):
if "batch_normalization" in layer.name:
effnet.layers[i] = GroupNormalization(groups=32, axis=-1, epsilon=0.00001 )
print(clf.classes_ )<prepare_output>
def build_model() :

 model = Sequential()
model.add(effnet)
 model.add(GlobalAveragePooling2D())
 model.add(Dropout(0.5))
model.add(Dense(5, activation=elu))
model.add(Dense(1, activation="linear"))
model.compile(loss='mse',
optimizer=RAdam(lr=0.00005),
metrics=['mse', 'acc'])
 print(model.summary())
 return model
model = build_model()
sub = pd.DataFrame()
for index,c in enumerate(clf.classes_):
sub[c]=sub_preds[:,index]
cuisine = []
for item in sub.index.values:
cuisine.append(np.argmax(sub.loc[item]))
sub['cuisine1']=cuisine
sub['id']=test_ids_for_sub<load_from_csv>
kappa_metrics = Metrics()
es = EarlyStopping(monitor='val_loss', mode='auto', verbose=1, patience=12)
 rlr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=4,
verbose=1,
mode='auto',
epsilon=0.0001)
 model.fit_generator(train_generator,
steps_per_epoch=train_generator.samples // BATCH_SIZE,
epochs=35,
validation_data=val_generator,
validation_steps = val_generator.samples // BATCH_SIZE,
callbacks=[kappa_metrics, es, rlr] )
temp_1 = pd.read_csv('.. /input/svm-12jul18/svm_output_None.csv')
 temp_2 = pd.read_csv('.. /input/let-s-cook-model/submission.csv')
 sub['cuisine2']=temp_1['cuisine']
sub['cuisine3']=temp_2['cuisine']<count_values>
model.load_weights(SAVED_MODEL_NAME )
sub['diff'] = [1 if a != b else 0 for a,b in zip(sub['cuisine2'], sub['cuisine3'])]
sub['diff'].value_counts()
sub[sub['diff']==1][['cuisine1','cuisine2','cuisine3']].head(50 )<count_values>
y_train_preds, train_labels = get_preds_and_labels(model, train_generator)
 y_train_preds = np.rint(y_train_preds ).astype(np.uint8 ).clip(0, 4)
 train_score = cohen_kappa_score(train_labels, y_train_preds, weights="quadratic")
 y_val_preds, val_labels = get_preds_and_labels(model, val_generator)
 y_val_preds = np.rint(y_val_preds ).astype(np.uint8 ).clip(0, 4)
 val_score = cohen_kappa_score(val_labels, y_val_preds, weights="quadratic" )
cuisine = []
for index in sub.index.values:
temp = sub[['cuisine1', 'cuisine2', 'cuisine3']].iloc[index].value_counts()
temp_score = temp[0]
temp_res = temp.index.values[0]
if temp_score ==1:
temp_res = sub[['cuisine3']].iloc[index].values[0]
cuisine.append(temp_res)
 sub['cuisine']=cuisine
sub[['id','cuisine']].head()<save_to_csv>
print(f"The Training Cohen Kappa Score is: {round(train_score, 5)}")
 print(f"The Validation Cohen Kappa Score is: {round(val_score, 5)}" )
sub[['id','cuisine']].to_csv('finalsub.csv',index=False )<import_modules>
class OptimizedRounder(object):

 def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):

 X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):

 loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):

 X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):

 return self.coef_['x']
import numpy as np
import pandas as pd<load_from_disk>
y_val_preds, val_labels = get_preds_and_labels(model, val_generator)
 optR = OptimizedRounder()
optR.fit(y_val_preds, val_labels)
 coefficients = optR.coefficients()
opt_val_predictions = optR.predict(y_val_preds, coefficients)
 new_val_score = cohen_kappa_score(val_labels, opt_val_predictions, weights="quadratic" )
df1 = pd.read_json('.. /input/train.json')
 df2 = pd.read_json('.. /input/test.json' )<string_transform>
print(f"Optimized Thresholds:
{coefficients}
")
 print(f"The Validation Quadratic Weighted Kappa(QWK)
 \
with optimized rounding thresholds is: {round(new_val_score, 5)}
")
 print(f"This is an improvement of {round(new_val_score - val_score, 5)}
\
over the unoptimized rounding" )
df1['ingredients1'] = df1['ingredients'].apply(','.join)
 df2['ingredients2'] = df2['ingredients'].apply(','.join )<data_type_conversions>
test_df['diagnosis'] = np.zeros(test_df.shape[0])
 test_generator = ImageDataGenerator(preprocessing_function=preprocess_image,
rescale=1 / 128.).flow_from_dataframe(test_df,
x_col='id_code',
y_col='diagnosis',
directory=TEST_IMG_PATH,
target_size=(IMG_WIDTH, IMG_HEIGHT),
batch_size=BATCH_SIZE,
class_mode='other',
shuffle=False )
<categorify><EOS> 
y_test, _ = get_preds_and_labels(model, test_generator)
 y_test = optR.predict(y_test, coefficients ).astype(np.uint8)
 test_df['diagnosis'] = y_test
test_df['id_code'] = test_df['id_code'].str.replace(r'.png$', '')
 test_df.to_csv('submission.csv', index=False )