Kaggle competition: APTOS 2019 Blindness Detection
concat_sub['target'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['target_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['target_min'],
concat_sub['target_median']))
concat_sub[['image_name', 'target']].to_csv('submission_minmax_median.csv',
index=False, float_format='%.6f' )<import_modules>
IMSIZE = 300
aptos2019_path = ".. /input/aptos2019-blindness-detection/"
aptos2019_train_path = ".. /input/aptos2019-blindness-detection/train_images/"
aptos2019_test_path = ".. /input/aptos2019-blindness-detection/test_images/"
aptos2019_extension = ".png"
aptos2015_path = ".. /input/aptos2015blindnessdetection/"
aptos2015_train_path = ".. /input/aptos2015blindnessdetection/train_images/"
aptos2015_test_path = ".. /input/aptos2015blindnessdetection/test_images/"
aptos2015_extension = ".jpg"
def plot_diag_hist(dataframe, title='NoTitle'):
f, ax = plt.subplots(figsize=(7, 4))
ax = sns.countplot(x="diagnosis", data=dataframe, palette="GnBu_d")
 sns.despine()
plt.title(title)
 plt.show()
trainbase_2019_df = pd.read_csv(aptos2019_path+"train.csv")
 test_2019_df = pd.read_csv(aptos2019_path+"test.csv")
 print("Shape of 2019 Training Data: {}".format(trainbase_2019_df.shape))
print("Shape of 2019 Test Data: {}
".format(test_2019_df.shape))
{}
".format(trainbase_2019_df.head()))
{}
".format(trainbase_2019_df['diagnosis'].value_counts() /trainbase_2019_df.shape[0]))
trainbase_2015_df = pd.read_csv(aptos2015_path+"train.csv")
 test_2015_df = pd.read_csv(aptos2015_path+"test.csv")
 print("Shape of 2015 Training Data: {}".format(trainbase_2015_df.shape))
print("Shape of 2015 Test Data: {}
".format(test_2015_df.shape))
trainbase_2015_df.rename(columns={"image": "id_code", "level": "diagnosis"}, inplace=True)
 test_2015_df.rename(columns={"image": "id_code", "level": "diagnosis"}, inplace=True)
 test_2015_df.drop(columns=["Usage"], inplace=True)
 plot_diag_hist(trainbase_2019_df, title="Predictions of 2019 Training Data")
 plot_diag_hist(trainbase_2015_df, title="Predictions of 2015 Training Data")
 full_path_train_2019 = lambda id_code : aptos2019_train_path+id_code+aptos2019_extension
full_path_test_2019 = lambda id_code : aptos2019_test_path+id_code+aptos2019_extension
full_path_train_2015 = lambda id_code : aptos2015_train_path+id_code+aptos2015_extension
full_path_test_2015 = lambda id_code : aptos2015_test_path+id_code+aptos2015_extension
trainbase_2019_df["path"] = trainbase_2019_df["id_code"].apply(full_path_train_2019)
 test_2019_df["path"] = test_2019_df["id_code"].apply(full_path_test_2019)
 trainbase_2015_df["path"] = trainbase_2015_df["id_code"].apply(full_path_train_2015)
 test_2015_df["path"] = test_2015_df["id_code"].apply(full_path_test_2015)
import numpy as np
import pandas as pd
import gc
import riiideducation
from sklearn.metrics import roc_auc_score
from sklearn.feature_selection import RFE
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import KFold
from lightgbm import LGBMClassifier
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression<find_best_params>
train_raw_df, validation_df = train_test_split(trainbase_2019_df, test_size=0.2, random_state=42,
stratify=trainbase_2019_df[['diagnosis']])
 print("Shape of Training Data: {}".format(train_raw_df.shape))
print("Diagnosis Distribution in Training Set:
{}
".format(train_raw_df['diagnosis'].value_counts() /train_raw_df.shape[0]))
print("Shape of Validation Data: {}".format(validation_df.shape))
print("Diagnosis Distribution in Validation Set:
{}
".format(validation_df['diagnosis'].value_counts() /validation_df.shape[0]))
train_raw_2015_df, validation_2015_df = train_test_split(trainbase_2015_df, test_size=0.2, random_state=42,
stratify=trainbase_2015_df[['diagnosis']])
 print("Shape of additional 2015 Training Data: {}".format(train_raw_2015_df.shape))
print("Diagnosis Distribution in 2015 Training Set:
{}
".format(train_raw_2015_df['diagnosis'].value_counts() /train_raw_2015_df.shape[0]))
print("Shape of additional 2015 Validation Data: {}".format(validation_2015_df.shape))
print("Diagnosis Distribution in 2015 Validation Set:
{}
".format(validation_2015_df['diagnosis'].value_counts() /validation_2015_df.shape[0]))
train_df = train_raw_df
train_2015_df = train_raw_2015_df
print("Shape of Oversampled Training Data: {}".format(train_df.shape))
print("Shape of Merged Validation Data: {}".format(validation_df.shape))
print("Appearance of Oversampled Training Data:
{}
".format(train_df.head()))
print("Diagnosis Distribution in Oversampled Training Set:
{}
".format(train_df['diagnosis'].value_counts() /train_df.shape[0]))
plot_diag_hist(train_df, title="Predictions of Stratified and Oversampled Training Data")
 train_df = train_df.sample(frac=1, random_state=42 ).reset_index(drop=True)
 validation_df = validation_df.sample(frac=1, random_state=42 ).reset_index(drop=True)
 train_2015_df = train_2015_df.sample(frac=1, random_state=42 ).reset_index(drop=True)
 validation_2015_df = validation_2015_df.sample(frac=1, random_state=42 ).reset_index(drop=True )
env = riiideducation.make_env()<load_from_csv>
imagepath = train_df.loc[4,"path"]
img = load_image_preprocess(imagepath, augmentation=False)
 plt.imshow(img)
 print(img.shape)
 print(( img[np.newaxis] ).shape)
 display_samples_preprocess_steps(train_df)
 img1 = augment_aptos(aug_rotate,img)
 img2 = augment_aptos(aug_rotate,img)
 show_Nimages([img,img1,img2],scale=2)
 img1 = augment_aptos(aug_flip,img)
 img2 = augment_aptos(aug_flip,img)
 img3 = augment_aptos(aug_flip,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(aug_bright,img)
 img2 = augment_aptos(aug_bright,img)
 img3 = augment_aptos(aug_bright,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(aug_crop,img)
 img2 = augment_aptos(aug_crop,img)
 img3 = augment_aptos(aug_crop,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(aug_hole,img)
 img2 = augment_aptos(aug_hole,img)
 img3 = augment_aptos(aug_hole,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(aug_flare,img)
 img2 = augment_aptos(aug_flare,img)
 img3 = augment_aptos(aug_flare,img)
 show_Nimages([img3,img1,img2],scale=2)
 img1 = augment_aptos(full_augmentation,img)
 img2 = augment_aptos(full_augmentation,img)
 img3 = augment_aptos(full_augmentation,img)
 show_Nimages([img3,img1,img2],scale=2 )
train = pd.read_csv(
 '/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'
}
)<load_from_csv>
IS_REGRESSION = False
def extract_target_vector_multi(dataframe_X):
y_targets = pd.get_dummies(dataframe_X['diagnosis'] ).values
y_targets_multi = np.empty(y_targets.shape, dtype=y_targets.dtype)
 y_targets_multi[:, 4] = y_targets[:, 4]
for i in range(3, -1, -1):
y_targets_multi[:, i] = np.logical_or(y_targets[:, i], y_targets_multi[:, i+1])
 return y_targets_multi
def extract_target_vector_regression(dataframe_X):
y_targets = dataframe_X['diagnosis'].values
return y_targets
def result_from_prediction_regression(y_pred):
return(np.clip(np.round(y_pred),0,4)).astype("uint" ).flatten()
def result_from_prediction_multi(y_pred):
return(y_pred > 0.5 ).astype(int ).sum(axis=1)- 1
if(IS_REGRESSION):
y_train_multi = extract_target_vector_regression(train_df)
 y_validate_multi = extract_target_vector_regression(validation_df)
 y_trainbase_2019_multi = extract_target_vector_regression(trainbase_2019_df)
 y_trainbase_2015_multi = extract_target_vector_regression(trainbase_2015_df)
 y_test_2015_multi = extract_target_vector_regression(test_2015_df)
 result_from_prediction = result_from_prediction_regression
else:
y_train_multi = extract_target_vector_multi(train_df)
 y_validate_multi = extract_target_vector_multi(validation_df)
 y_trainbase_2019_multi = extract_target_vector_multi(trainbase_2019_df)
 y_trainbase_2015_multi = extract_target_vector_multi(trainbase_2015_df)
 y_test_2015_multi = extract_target_vector_multi(test_2015_df)
 result_from_prediction = result_from_prediction_multi
print("Training data:
Xshape: {}
yshape: {}
".format(train_df.shape,y_train_multi.shape))
print("Validation data:
Xshape: {}
yshape: {}
".format(validation_df.shape,y_validate_multi.shape))
print("Test data:
Xshape: {}
".format(test_2019_df.shape))
print("Training data multilabel diagnosis vector:", y_train_multi.sum(axis=0))
print("Validation data multilabel diagnosis vector:", y_validate_multi.sum(axis=0))
print("Training data 2015 multilabel diagnosis vector:", y_trainbase_2015_multi.sum(axis=0))
print("Training data 2015 multilabel diagnosis vector:", y_test_2015_multi.sum(axis=0))
questions_df = pd.read_csv(
 '/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={
'question_id': 'int16',
'part': 'int8'}
)<load_from_csv>
BATCH_SIZE = 32
class My_Image_Generator(Sequence):
def __init__(self, image_filenames, labels, batch_size, shuffle=True, augment=False, train=False):
self.image_filenames, self.labels = image_filenames, labels
self.is_train = train
self.batch_size = batch_size
self.is_shuffle = shuffle
self.is_augment = augment
if(self.is_shuffle):
self.on_epoch_end()
def __len__(self):
return int(np.ceil(len(self.image_filenames)/ float(self.batch_size)))
 def __getitem__(self, idx):
batch_x = self.image_filenames[idx * self.batch_size:(idx + 1)* self.batch_size]
if(self.is_train):
batch_y = self.labels[idx * self.batch_size:(idx + 1)* self.batch_size]
else:
batch_y = np.zeros(self.batch_size)
 return self.batch_generate(batch_x, batch_y)
 def on_epoch_end(self):
if(self.is_shuffle):
self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)
 else:
pass
def batch_generate(self, batch_x, batch_y):
batch_images = []
for(sample, label)in zip(batch_x, batch_y):
if(self.is_augment):
img = load_image_preprocess(sample, augmentation=True)
 else:
img = load_image_preprocess(sample, augmentation=False)
 batch_images.append(img)
 batch_images = np.array(batch_images, np.float32)/ 255
batch_y = np.array(batch_y, np.float32)
 return batch_images, batch_y
train_data_generator_aug = My_Image_Generator(train_df["path"], labels=y_train_multi, shuffle=True, train=True,
batch_size=BATCH_SIZE, augment=True)
 train_data_generator_no_aug = My_Image_Generator(train_df["path"], labels=y_train_multi, shuffle=False, train=True,
batch_size=BATCH_SIZE, augment=False)
 valid_data_generator_aug = My_Image_Generator(validation_df["path"], y_validate_multi, shuffle=False, train=True,
batch_size=BATCH_SIZE, augment=True)
 valid_data_generator_no_aug = My_Image_Generator(validation_df["path"], y_validate_multi, shuffle=False, train=True,
batch_size=BATCH_SIZE, augment=False)
 trainbase_2019_data_generator_aug = My_Image_Generator(trainbase_2019_df["path"], labels=y_trainbase_2019_multi,
shuffle=True, train=True, batch_size=BATCH_SIZE, augment=True)
 trainbase_2019_data_generator_no_aug = My_Image_Generator(trainbase_2019_df["path"], labels=y_trainbase_2019_multi,
shuffle=False, train=True, batch_size=BATCH_SIZE, augment=False)
 trainbase_2015_data_generator_aug = My_Image_Generator(trainbase_2015_df["path"], labels=y_trainbase_2015_multi,
shuffle=True, train=True, batch_size=BATCH_SIZE, augment=True)
 trainbase_2015_data_generator_no_aug = My_Image_Generator(trainbase_2015_df["path"], labels=y_trainbase_2015_multi,
shuffle=False, train=True, batch_size=BATCH_SIZE, augment=False)
 test_data_generator_aug = My_Image_Generator(test_2019_df["path"], labels=None, shuffle=False, train=False,
batch_size=BATCH_SIZE, augment=True)
 test_data_generator_no_aug = My_Image_Generator(test_2019_df["path"], labels=None, shuffle=False, train=False,
batch_size=BATCH_SIZE, augment=False)
 test_data_2015_generator_no_aug = My_Image_Generator(test_2015_df["path"], labels=y_test_2015_multi, shuffle=False, train=True,
batch_size=BATCH_SIZE, augment=False )
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
class QWK_Metrics(Callback):
def __init__(self, model, validation_generator_X, validation_y):
self.model_to_save = model
self.valid_generator = validation_generator_X
self.valid_y = validation_y
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
y_val_pred = model.predict_generator(generator=self.valid_generator,
steps=-(self.valid_y.shape[0] // -BATCH_SIZE),
verbose=1)
 y_val_pred = result_from_prediction(y_val_pred)
 y_val_true = result_from_prediction(self.valid_y)
 _val_kappa = cohen_kappa_score(y_val_pred, y_val_true, weights='quadratic')
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model_to_save.save('model_kappa.h5')
 return
reduceLR = ReduceLROnPlateau(
 monitor='val_loss',
factor=0.1,
patience=5,
min_lr=1e-6,
verbose=1,
mode='min' 
)
 earlyStopping = EarlyStopping(
 monitor='val_loss',
patience=10,
verbose=1,
mode='min',
restore_best_weights=True 
)
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
def get_available_gpus() :
local_device_protos = device_lib.list_local_devices()
return [x.name for x in local_device_protos if x.device_type == "GPU"]
num_gpu = len(get_available_gpus())
 print("Number of available GPUs: {}".format(num_gpu))
train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
used_init = "lecun_uniform"
reg_value_kernel = 5e-5
reg_value_bias = 5e-5
def build_model_plainCNN() :
model = Sequential()
model.add(Conv2D(32,(5, 5), input_shape=[224,224,3], activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(64,(3, 3), activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(128,(3, 3), activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(256,(3, 3), activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(512,(3, 3), activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.5))
model.add(Flatten())
 model.add(Dense(512, activation='relu', kernel_initializer=used_init,
kernel_regularizer=regularizers.l1(reg_value_kernel), bias_regularizer=regularizers.l1(reg_value_bias)))
 model.add(Dropout(0.5))
model.add(Dense(5, activation='sigmoid', kernel_initializer=used_init))
return model
def build_model_flat() :
model = Sequential()
model.add(Conv2D(32,(9, 9), input_shape=[224,224,3], activation='relu', kernel_initializer=used_init))
model.add(MaxPooling2D(pool_size=(4, 4)))
 model.add(Conv2D(64,(3, 3), activation='relu', kernel_initializer=used_init))
model.add(MaxPooling2D(pool_size=(4, 4)))
 model.add(Conv2D(128,(3, 3), activation='relu', kernel_initializer=used_init))
model.add(MaxPooling2D(pool_size=(4, 4)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation='relu', kernel_initializer=used_init))
model.add(Dropout(0.2))
model.add(Dense(5, activation='sigmoid', kernel_initializer=used_init))
return model
def build_model_densenet121() :
densenet = DenseNet121(weights='.. /input/densenetpretrained/DenseNet-BC-121-32-no-top.h5', include_top=False,
input_shape=(IMSIZE,IMSIZE,3))
ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(densenet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB0() :
effnet = efn.EfficientNetB0(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnetpretrained/efficientnet-b0_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB1() :
effnet = efn.EfficientNetB1(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnetpretrained/efficientnet-b1_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB2() :
effnet = efn.EfficientNetB2(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnet/pretrainedweights/efficientnet-b2_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB3() :
effnet = efn.EfficientNetB3(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnet/pretrainedweights/pretrainedweights/efficientnet-b3_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 if(IS_REGRESSION):
ly_dense_2 = layers.Dense(1, activation='linear')
 else:
ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
def build_model_efficientnetB4() :
effnet = efn.EfficientNetB4(weights=None,
include_top=False,
input_shape=(IMSIZE, IMSIZE, 3))
effnet.load_weights(".. /input/efficientnet/pretrainedweights/efficientnet-b4_imagenet_1000_notop.h5")
 ly_gap = layers.GlobalAveragePooling2D()
ly_dropout_1 = layers.Dropout(0.5)
 ly_dense_1 = layers.Dense(256,activation="relu")
 ly_dropout_2 = layers.Dropout(0.5)
 ly_dense_2 = layers.Dense(5, activation='sigmoid')
 model = Sequential()
model.add(effnet)
 model.add(ly_gap)
 model.add(ly_dropout_1)
 model.add(ly_dense_1)
 model.add(ly_dropout_2)
 model.add(ly_dense_2)
 return model
if(num_gpu == 0):
base_model = build_model_efficientnetB3()
model = base_model
print("Training on single CPU...no GPU avaliable")
 elif(num_gpu == 1):
base_model = build_model_efficientnetB3()
model = base_model
print("Training on single GPU...")
 else:
with tf.device('/cpu:0'):
base_model = build_model_efficientnetB3()
try:
model = multi_gpu_model(base_model, gpus=4)
 print("Training on multiple GPUs.. ")
 except ValueError:
model = base_model
print("Training on single GPU or CPU...error when compiling multi_gpu_model...")
 if(IS_REGRESSION):
model.compile(loss='mse', optimizer=Adam(lr=5e-5), metrics=['accuracy'])
 else:
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-4), metrics=['accuracy'])
 print("Model Summary:")
 model.summary()
user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
for layer in model.layers:
layer.trainable = True
for layer in model.layers[:-5]:
layer.trainable = False
if(IS_REGRESSION):
model.compile(loss='mse', optimizer=Adam(lr=1e-4), metrics=['accuracy'])
 else:
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=5e-4), metrics=['accuracy'])
 print("Model Summary:")
 model.summary()
kappa_metrics_1 = QWK_Metrics(base_model, trainbase_2019_data_generator_no_aug, y_trainbase_2019_multi)
 history_1 = model.fit_generator(
 generator=trainbase_2015_data_generator_aug,
steps_per_epoch=-(y_trainbase_2015_multi.shape[0] // -BATCH_SIZE),
epochs=5,
validation_data=trainbase_2019_data_generator_no_aug,
validation_steps=-(y_trainbase_2019_multi.shape[0] // -BATCH_SIZE),
callbacks=[kappa_metrics_1, reduceLR, earlyStopping] 
)
 for layer in model.layers:
layer.trainable = True
if(IS_REGRESSION):
model.compile(loss='mse', optimizer=Adam(lr=5e-5), metrics=['accuracy'])
 else:
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=5e-5), metrics=['accuracy'])
 print("Model Summary:")
 model.summary()
kappa_metrics_2 = QWK_Metrics(base_model, trainbase_2019_data_generator_no_aug, y_trainbase_2019_multi)
 history_2 = model.fit_generator(
 generator=trainbase_2015_data_generator_aug,
steps_per_epoch=-(y_trainbase_2015_multi.shape[0] // -BATCH_SIZE),
epochs=5,
validation_data=trainbase_2019_data_generator_no_aug,
validation_steps=-(y_trainbase_2019_multi.shape[0] // -BATCH_SIZE),
callbacks=[kappa_metrics_2, reduceLR, earlyStopping] 
)
 base_model.save('model_phase_1.h5')
for column in user_lecture_stats_part.columns:
bool_column = column + '_boolean'
user_lecture_stats_part[bool_column] =(user_lecture_stats_part[column] > 0 ).astype(int )<set_options>
kappa_metrics_3 = QWK_Metrics(base_model, valid_data_generator_no_aug, y_validate_multi)
 history_3 = model.fit_generator(
 generator=train_data_generator_aug,
steps_per_epoch=-(y_train_multi.shape[0] // -BATCH_SIZE),
epochs=5,
validation_data=valid_data_generator_no_aug,
validation_steps=-(y_validate_multi.shape[0] // -BATCH_SIZE),
callbacks=[kappa_metrics_3, reduceLR, earlyStopping] 
)
 base_model.save('model_phase_2.h5' )
del train_lectures
gc.collect()<sort_values>
kappa_metrics = kappa_metrics_3
history_df = pd.DataFrame(history_3.history)
 print("Max Kappa: {}".format(max(kappa_metrics.val_kappas)))
 best_epoch = np.argmax(kappa_metrics.val_kappas)+1
print("Max Kappa at epoch {}.".format(best_epoch))
print("Epoch {}: Acc: {:.4f}, ValAcc: {:.4f}, Loss: {:.4f}, ValLoss: {:.4f}".format(best_epoch,
history_df["acc"][best_epoch-1],
history_df["val_acc"][best_epoch-1],
history_df["loss"][best_epoch-1],
history_df["val_loss"][best_epoch-1] 
))
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )<groupby>
base_model.load_weights('model_kappa.h5')
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2<groupby>
tta_cycles = 5
y_test_tta = model.predict_generator(generator=test_data_generator_no_aug,
steps=-(test_2019_df.shape[0] // -BATCH_SIZE),
verbose=1)
 y_test_tta = result_from_prediction(y_test_tta)
 test_2019_df['diagnosis'] = y_test_tta
test_2019_df.drop(["path"], inplace=True, axis=1)
 test_2019_df.to_csv("submission.csv",index=False)
 print("Submission complete!")