Kaggle competition: APTOS 2019 Blindness Detection
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )<categorify>
%matplotlib inline
np.random.seed(2019)
 tf.set_random_seed(2019 )
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]<merge>
train_df = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 test_df = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 print(train_df.shape)
 print(test_df.shape)
 train_df.head()
train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )<groupby>
N = train_df.shape[0]
x_train = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(train_df['id_code'])) :
x_train[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/train_images/{image_id}.png'
)
user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()<data_type_conversions>
N = test_df.shape[0]
x_test = np.empty(( N, 224, 224, 3), dtype=np.uint8)
 for i, image_id in enumerate(tqdm(test_df['id_code'])) :
x_test[i, :, :, :] = preprocess_image(
 f'.. /input/aptos2019-blindness-detection/test_images/{image_id}.png'
)
for column in user_lecture_stats_part.columns:
bool_column = column + '_boolean'
user_lecture_stats_part[bool_column] =(user_lecture_stats_part[column] > 0 ).astype(int )<drop_column>
y_train = pd.get_dummies(train_df['diagnosis'] ).values
print(x_train.shape)
 print(y_train.shape)
 print(x_test.shape )
train_lectures = None
del(train_lectures )<set_options>
y_train_multi = np.empty(y_train.shape, dtype=y_train.dtype)
 y_train_multi[:, 4] = y_train[:, 4]
for i in range(3, -1, -1):
y_train_multi[:, i] = np.logical_or(y_train[:, i], y_train_multi[:, i+1])
 print("Original y_train:", y_train.sum(axis=0))
print("Multilabel version:", y_train_multi.sum(axis=0))
gc.collect()<prepare_x_and_y>
x_train, x_val, y_train, y_val = train_test_split(
 x_train, y_train_multi,
test_size=0.50,
random_state=2019
)
def split_X_Y(_train):
_train = _train[_train.content_type_id == False]
_train = _train.sort_values(['timestamp'], ascending=True)
 _train.drop(['timestamp','content_type_id'], axis=1, inplace=True)
 X = _train.iloc[train_part_len:,:]
X = X[X.answered_correctly!= -1 ]
X = X.sort_values(['user_id'])
 Y = X[["answered_correctly"]]
X = X.drop(["answered_correctly"], axis=1)
 return X, Y<categorify>
BATCH_SIZE = 13
def create_datagen() :
return ImageDataGenerator(
 zoom_range=0.15,
fill_mode='constant',
cval=0.,
horizontal_flip=True,
vertical_flip=True, 
)
 data_generator = create_datagen().flow(x_train, y_train, batch_size=BATCH_SIZE, seed=2019 )
count_en = ce.CountEncoder()<groupby>
densenet = DenseNet121(
 weights='.. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5',
include_top=False,
input_shape=(224,224,3)
 )
def addFeatures(_train, _questions):
train_questions_only_df = _train[_train['answered_correctly']!=-1]
grouped_by_user_df = train_questions_only_df.groupby('user_id')
 user_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count', 'sum']} ).copy()
user_answers_df.columns = ['mean_answered_correctly_user', 'questions_answered', 'sum_answered_correctly_user']
grouped_by_content_df = train_questions_only_df.groupby('content_id')
 content_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count'] } ).copy()
content_answers_df.columns = ['mean_answered_correctly_content', 'question_asked']
questions_df = _questions.merge(content_answers_df, left_on = 'question_id', right_on = 'content_id', how = 'left')
 bundle_dict = questions_df['bundle_id'].value_counts().to_dict()
questions_df['right_answers'] = questions_df['mean_answered_correctly_content'] * questions_df['question_asked']
questions_df['bundle_size'] = questions_df['bundle_id'].apply(lambda x: bundle_dict[x])
 grouped_by_bundle_df = questions_df.groupby('bundle_id')
 bundle_answers_df = grouped_by_bundle_df.agg({'right_answers': 'sum', 'question_asked': 'sum'} ).copy()
bundle_answers_df.columns = ['bundle_rignt_answers', 'bundle_questions_asked']
bundle_answers_df['bundle_accuracy'] = bundle_answers_df['bundle_rignt_answers'] / bundle_answers_df['bundle_questions_asked']
grouped_by_part_df = questions_df.groupby('part')
 part_answers_df = grouped_by_part_df.agg({'right_answers': 'sum', 'question_asked': 'sum'} ).copy()
part_answers_df.columns = ['part_rignt_answers', 'part_questions_asked']
part_answers_df['part_accuracy'] = part_answers_df['part_rignt_answers'] / part_answers_df['part_questions_asked']
return user_answers_df, questions_df, bundle_answers_df, part_answers_df<merge>
def build_model() :
model = Sequential()
model.add(densenet)
 model.add(layers.GlobalAveragePooling2D())
 model.add(layers.Dropout(0.80))
model.add(layers.Dense(5, activation='sigmoid'))
model.compile(
 loss='binary_crossentropy',
optimizer=Adam(lr=0.00010509613402110064),
metrics=['accuracy'] 
)
 return model
def handleMissing(X, Y, _user_answers, _questions, _bundle_answers, _part_answers, _lectures):
X['prior_question_had_explanation'].fillna(False, inplace=True)
 X["prior_question_had_explanation_enc"] = X['prior_question_had_explanation'].apply(convertBoolean)
 X = X[['user_id', 'content_id', 'prior_question_elapsed_time','prior_question_had_explanation_enc']]
X = X.merge(_user_answers, how = 'left', on = 'user_id')
 X = X.merge(_questions, how = 'left', left_on = 'content_id', right_on = 'question_id')
 X = X.merge(_bundle_answers, how = 'left', on = 'bundle_id')
 X = X.merge(_part_answers, how = 'left', on = 'part')
 X = X.merge(_lectures, how = 'left', on=['user_id'])
 X["toeic_section"] = X['part'].apply(TOEICSection)
 cat_ce = count_en.fit_transform(X["toeic_section"])
 X = X.join(cat_ce.add_suffix("_ce"))
X["tags_count"] = X['tags'].apply(tagsCount)
 X = X.drop(columns=['user_id', 'content_id', 'bundle_id', 'correct_answer', 'tags', 'toeic_section'])
 X['sum_answered_correctly_user'].fillna(0, inplace=True)
 X['part_1'].fillna(0, inplace = True)
 X['part_2'].fillna(0, inplace = True)
 X['part_3'].fillna(0, inplace = True)
 X['part_4'].fillna(0, inplace = True)
 X['part_5'].fillna(0, inplace = True)
 X['part_6'].fillna(0, inplace = True)
 X['part_7'].fillna(0, inplace = True)
 X['type_of_concept'].fillna(0, inplace = True)
 X['type_of_intention'].fillna(0, inplace = True)
 X['type_of_solving_question'].fillna(0, inplace = True)
 X['type_of_starter'].fillna(0, inplace = True)
 X['part_1_boolean'].fillna(0, inplace = True)
 X['part_2_boolean'].fillna(0, inplace = True)
 X['part_3_boolean'].fillna(0, inplace = True)
 X['part_4_boolean'].fillna(0, inplace = True)
 X['part_5_boolean'].fillna(0, inplace = True)
 X['part_6_boolean'].fillna(0, inplace = True)
 X['part_7_boolean'].fillna(0, inplace = True)
 X['type_of_concept_boolean'].fillna(0, inplace = True)
 X['type_of_intention_boolean'].fillna(0, inplace = True)
 X['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X['type_of_starter_boolean'].fillna(0, inplace = True)
 X.fillna(0.5, inplace=True)
 target_columns = ['question_id', 'part']
target_enc = ce.TargetEncoder(cols=target_columns)
 target_enc.fit(X[target_columns], Y)
 X = X.join(target_enc.transform(X[target_columns] ).add_suffix('_target'))
catboost_columns = ['prior_question_elapsed_time', 'prior_question_had_explanation_enc']
catboost_enc = ce.CatBoostEncoder(cols=catboost_columns)
 catboost_enc.fit(X[catboost_columns], Y)
 X = X.join(catboost_enc.transform(X[catboost_columns] ).add_suffix('_catboost'))
return X, target_enc, catboost_enc<split>
class Metrics(Callback):
def on_train_begin(self, logs={}):
self.val_kappas = []
def on_epoch_end(self, epoch, logs={}):
X_val, y_val = self.validation_data[:2]
y_val = y_val.sum(axis=1)- 1
y_pred = self.model.predict(X_val)> 0.5
y_pred = y_pred.astype(int ).sum(axis=1)- 1
_val_kappa = cohen_kappa_score(
 y_val,
y_pred,
weights='quadratic' 
)
 self.val_kappas.append(_val_kappa)
 print(f"val_kappa: {_val_kappa:.4f}")
 if _val_kappa == max(self.val_kappas):
print("Validation Kappa has improved.Saving model.")
 self.model.save('model.h5')
 return
X_full, Y_full = split_X_Y(train )<feature_engineering>
kappa_metrics = Metrics()
history = model.fit_generator(
 data_generator,
steps_per_epoch=x_train.shape[0] / BATCH_SIZE,
epochs=15,
validation_data=(x_val, y_val)
 )
user_answers_df, questions_df, bundle_answers_df, part_answers_df = addFeatures(train, questions )<drop_column>
y_test = model.predict(x_test)
 y_test
train = None
del(train)
 gc.collect()<split>
y_test = y_test > 0.37757874193797547
y_test
X_full, target_enc, catboost_enc = handleMissing(X_full, Y_full, user_answers_df, questions_df, bundle_answers_df, part_answers_df, user_lecture_stats_part )<import_modules>
y_test.astype(int ).sum(axis=1 )
from sklearn.metrics import roc_auc_score
from xgboost import XGBClassifier, plot_importance
import matplotlib.pyplot as plt<choose_model_class>
y_test.astype(int ).sum(axis=1)- 1
model = XGBClassifier(
 tree_method="hist",
learning_rate=0.1,
gamma=0.2,
n_estimators=200,
max_depth=8,
min_child_weight=40,
subsample=0.87,
colsample_bytree=0.95,
reg_alpha=0.04,
reg_lambda=0.073,
objective='binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27
)<train_model>
y_test = y_test.astype(int ).sum(axis=1)- 1
y_test
model.fit(X_full, Y_full.values.ravel() )<compute_test_metric>
test_df['diagnosis'] = y_test
test_df.to_csv('submission.csv',index=False )