Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<define_variables>
treino = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)
all_results = [xgb_score, boost_score, log_score, rf_score,
svm_score, bag_score, xt_score, knn_score, cat_score, gbm_score]
result_names = ["XGBoost",
"Gradient Boosting",
"Logistic Regression",
"Random Forest",
"SVM",
"Bagging Classifier",
"Extra Trees Classifier",
"KNN",
"CatBoost",
"LightGBM"]
fig = plt.figure(figsize =(14,7))
fig.suptitle("Algorithm Comparison")
 ax = fig.add_subplot(111)
 plt.boxplot(all_results)
 ax.set_xticklabels(result_names)
 plt.show()<install_modules>
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'}
)
%%capture
!pip install keras-tuner<import_modules>
print(f'Number of Rows: {train_df.shape[0]}')
 print(f'Number of Cols: {train_df.shape[1]}' )
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from kerastuner.tuners import RandomSearch<split>
train_df = train_df.astype(data_types_dict )
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
random_state = 0,
test_size = 0.25 )<choose_model_class>
train_df.isna().sum()
%%capture
def tune_model(hp):
nn = Sequential()
nn.add(Dense(hp.Choice("units_1", [512, 1024, 2048]), activation = "relu", input_shape =(18,)))
 nn.add(Dropout(hp.Choice("dropout_1", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_2", [128, 256, 512]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_2", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_3", [32, 64, 128]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_3", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(hp.Choice("units_4", [8, 16, 32]), activation = "relu"))
nn.add(Dropout(hp.Choice("dropout_4", [0.0, 0.15, 0.2, 0.25])))
 nn.add(Dense(1, activation = "relu"))
lr = hp.Choice("learning_rate", [0.0001, 0.001, 0.01, 0.1])
 mm = hp.Choice("momentum", [0.0, 0.2, 0.4, 0.6, 0.8])
 nn.compile(optimizer = keras.optimizers.RMSprop(learning_rate = lr, momentum = mm),
loss = "binary_crossentropy",
metrics = ["accuracy"])
 return nn
tuner = RandomSearch(tune_model,
objective = "val_accuracy",
max_trials = 3)
 tuner.search(x = X_train,
y = Y_train,
verbose = 3,
epochs = 400,
batch_size = 25,
validation_data =(X_test, Y_test))<train_model>
treino = treino.loc[treino.content_type_id == False]
treino['tempo_exercicio'] = pd.DataFrame(treino.prior_question_elapsed_time.shift(-1))
treino.loc[treino.timestamp == 0, ['prior_question_had_explanation']] = treino.loc[treino.timestamp == 0, ['prior_question_had_explanation']].fillna(False)
 treino = treino.sort_values(['timestamp'], ascending=True ).reset_index(drop = True)
print("Best Params: " + str(tuner.oracle.get_best_trials(num_trials = 1)[0].hyperparameters.values))
nn = tuner.get_best_models() [0]
print("")
 print("Test score: " + str(nn.evaluate(X_test, Y_test)))
 print("")
 print("Train score: " + str(nn.evaluate(X_train, Y_train)) )<predict_on_test>
treino = pd.merge(treino, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left')
 treino.part = treino.part - 1
Y_predicted_nn = nn.predict(test.drop(["PassengerId"], axis = 1))
Y_predicted_nn =(Y_predicted_nn.ravel() > 0.5 ).astype(int )<import_modules>
treino.drop(['timestamp', 'content_type_id','question_id'], axis=1, inplace=True )
from sklearn.metrics import roc_curve, roc_auc_score<find_best_params>
tempo_medio_estudantef = treino[['user_id','tempo_exercicio']].groupby(['user_id'] ).agg(['mean'])
 tempo_medio_estudantef.columns = ['tempo_medio_estudante']
tempo_medio_exerciciof = treino[['content_id','tempo_exercicio']].groupby(['content_id'] ).agg(['mean'])
 tempo_medio_exerciciof.columns = ['tempo_medio_exercicio']
tempo_medio_tipo_exerciciof = treino[['tempo_exercicio','part']].groupby(['part'] ).agg(['mean'])
 tempo_medio_tipo_exerciciof.columns = ['tempo_medio_tipo_exercicio']
acerto_medio_estudantef = treino[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'count'])
 acerto_medio_estudantef.columns = ['acerto_medio_estudante', 'count']
media_numero_exerciciof = acerto_medio_estudantef[['acerto_medio_estudante','count']].groupby(['count'] ).agg(['mean'])
 media_numero_exerciciof.columns = ["media_numero_exercicio"]
acerto_medio_exerciciof = treino[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 acerto_medio_exerciciof.columns = ["acerto_medio_exercicio"]
acerto_medio_tipo_exerciciof = treino[['part','answered_correctly']].groupby(['part'] ).agg(['mean'])
 acerto_medio_tipo_exerciciof.columns = ["acerto_medio_tipo_exercicio"]
treino = treino[~treino['prior_question_had_explanation'].isna() ]
viu_medio_estudantef = treino[['prior_question_had_explanation']].astype('int8' ).groupby(treino['user_id'] ).agg(['mean','count'])
 viu_medio_estudantef.columns = ['viu_medio_estudante','count_had']
media_numero_viuf = viu_medio_estudantef[['viu_medio_estudante','count_had']].groupby(['count_had'] ).agg(['mean'])
 media_numero_viuf.columns = ["media_numero_viu"]
viu_medio_exerciciof = treino[['prior_question_had_explanation']].astype('int8' ).groupby(treino['content_id'] ).agg(['mean'])
 viu_medio_exerciciof.columns = ["viu_medio_exercicio"]
log_fpr, log_tpr, log_treshholds = roc_curve(Y, Y_predicted_log)
 boost_fpr, boost_tpr, boost_treshholds = roc_curve(Y, Y_predicted_boost)
 svm_fpr, svm_tpr, svm_treshholds = roc_curve(Y, Y_predicted_svm)
 rf_fpr, rf_tpr, rf_treshholds = roc_curve(Y, Y_predicted_rf)
 xgb_fpr, xgb_tpr, xgb_treshholds = roc_curve(Y, Y_predicted_xgb)
 bag_fpr, bag_tpr, bag_treshholds = roc_curve(Y, Y_predicted_bag)
 xt_fpr, xt_tpr, xt_treshholds = roc_curve(Y, Y_predicted_xt)
 knn_fpr, knn_tpr, knn_treshholds = roc_curve(Y, Y_predicted_knn)
 auc_score_log = roc_auc_score(Y, Y_predicted_log)
 auc_score_boost = roc_auc_score(Y, Y_predicted_boost)
 auc_score_svm = roc_auc_score(Y, Y_predicted_svm)
 auc_score_rf = roc_auc_score(Y, Y_predicted_rf)
 auc_score_xgb = roc_auc_score(Y, Y_predicted_xgb)
 auc_score_bag = roc_auc_score(Y, Y_predicted_bag)
 auc_score_xt = roc_auc_score(Y, Y_predicted_xt)
 auc_score_knn = roc_auc_score(Y, Y_predicted_knn)
 plt.figure(figsize =(14,7))
plt.plot([0,1], [0,1])
 plt.plot(log_fpr, log_tpr, label = "Logistic Regression(AUC-Score: " + str(round(auc_score_log, 2)) + ")")
 plt.plot(boost_fpr, boost_tpr, label = "Gradient Boosting(AUC-Score: " + str(round(auc_score_boost, 2)) + ")")
 plt.plot(svm_fpr, svm_tpr, label = "SVM(AUC-Score: " + str(round(auc_score_svm, 2)) + ")")
 plt.plot(rf_fpr, rf_tpr, label = "Random Forest(AUC-Score: " + str(round(auc_score_rf, 2)) + ")")
 plt.plot(xgb_fpr, xgb_tpr, label = "XGBoost(AUC-Score: " + str(round(auc_score_xgb, 2)) + ")")
 plt.plot(bag_fpr, bag_tpr, label = "Bagging Classifier(AUC-Score: " + str(round(auc_score_bag, 2)) + ")")
 plt.plot(xt_fpr, xt_tpr, label = "Extra Trees Clasifier(AUC-Score: " + str(round(auc_score_xt, 2)) + ")")
 plt.plot(knn_fpr, knn_tpr, label = "K-Nearest-Neighbor(AUC-Score: " + str(round(auc_score_knn, 2)) + ")")
 plt.title("ROC-Curve")
 plt.xlabel("FPR")
 plt.ylabel("TPR")
 plt.legend()
plt.show()<choose_model_class>
validation = pd.DataFrame()
stack = StackingClassifier(estimators = [("XGBoost", grid_xgb), 
("GradientBoosting", grid_boost), 
("RandomForest", grid_rf), 
("Logistic Regression", grid_log), 
("SVM", grid_svm), 
("Extra Trees Classifier", grid_xt), 
("KNeighborsClassifier", grid_knn), 
("CatBoost", grid_cat), 
("LightGBM", grid_gbm)], n_jobs = -1)
 cv = StratifiedKFold(n_splits = 5)
 stack = stack.fit(X, Y)
 stack_score = cross_val_score(stack, X, Y, cv = cv, n_jobs = -1)
 print("
print("Cross-Validated-Score: " + str(round(stack_score.mean() , 6)))
 print("
Y_predicted_stack = stack.predict(test.drop(["PassengerId"], axis = 1))<save_to_csv>
for i in range(4):
last_records = treino.drop_duplicates('user_id', keep = 'last')
 treino = treino[~treino.index.isin(last_records.index)]
validation = validation.append(last_records)
 del(last_records )
submission = pd.DataFrame({"PassengerId" : test.PassengerId,
"Survived" : Y_predicted_stack})
 submission.to_csv("submission_clf.csv", index = False )<save_to_csv>
validation.drop(['tempo_exercicio'], axis=1, inplace=True )
submission_nn = pd.DataFrame({"PassengerId" : test.PassengerId,
"Survived" : Y_predicted_nn})
 submission_nn.to_csv("submission_nn.csv", index = False )<set_options>
X = pd.DataFrame()
for i in range(15):
last_records = treino.drop_duplicates('user_id', keep = 'last')
 treino = treino[~treino.index.isin(last_records.index)]
X = X.append(last_records)
 del(last_records )
warnings.filterwarnings('ignore')
<load_from_csv>
tempo_medio_estudante = treino[['user_id','tempo_exercicio']].groupby(['user_id'] ).agg(['mean'])
 tempo_medio_estudante.columns = ['tempo_medio_estudante']
tempo_medio_exercicio = treino[['content_id','tempo_exercicio']].groupby(['content_id'] ).agg(['mean'])
 tempo_medio_exercicio.columns = ['tempo_medio_exercicio']
tempo_medio_tipo_exercicio = treino[['tempo_exercicio','part']].groupby(['part'] ).agg(['mean'])
 tempo_medio_tipo_exercicio.columns = ['tempo_medio_tipo_exercicio']
acerto_medio_estudante = treino[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean', 'count'])
 acerto_medio_estudante.columns = ['acerto_medio_estudante', 'count']
media_numero_exercicio = acerto_medio_estudante[['acerto_medio_estudante','count']].groupby(['count'] ).agg(['mean'])
 media_numero_exercicio.columns = ["media_numero_exercicio"]
acerto_medio_exercicio = treino[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'])
 acerto_medio_exercicio.columns = ["acerto_medio_exercicio"]
acerto_medio_tipo_exercicio = treino[['part','answered_correctly']].groupby(['part'] ).agg(['mean'])
 acerto_medio_tipo_exercicio.columns = ["acerto_medio_tipo_exercicio"]
treino = treino[~treino['prior_question_had_explanation'].isna() ]
viu_medio_estudante = treino[['prior_question_had_explanation']].astype('int8' ).groupby(treino['user_id'] ).agg(['mean','count'])
 viu_medio_estudante.columns = ['viu_medio_estudante','count_had']
media_numero_viu = viu_medio_estudante[['viu_medio_estudante','count_had']].groupby(['count_had'] ).agg(['mean'])
 media_numero_viu.columns = ["media_numero_viu"]
viu_medio_exercicio = treino[['prior_question_had_explanation']].astype('int8' ).groupby(treino['content_id'] ).agg(['mean'])
 viu_medio_exercicio.columns = ["viu_medio_exercicio"]
del(treino )
train = pd.read_csv(".. /input/titanic/train.csv")
 test = pd.read_csv(".. /input/titanic/test.csv" )<define_variables>
X = pd.merge(X, acerto_medio_estudante, on=['user_id'], how="left")
 X = pd.merge(X, tempo_medio_estudante, on=['user_id'], how="left")
 X = pd.merge(X, acerto_medio_exercicio, on=['content_id'], how="left")
 X = pd.merge(X, tempo_medio_exercicio, on=['content_id'], how="left")
 X = pd.merge(X, acerto_medio_tipo_exercicio, on=['part'], how="left")
 X = pd.merge(X, tempo_medio_tipo_exercicio, on=['part'], how="left")
 X = pd.merge(X, viu_medio_estudante, on=['user_id'], how="left")
 X = pd.merge(X, viu_medio_exercicio, on=['content_id'], how="left")
 X = pd.merge(X, media_numero_exercicio, on=['count'], how="left")
 X = pd.merge(X, media_numero_viu, on=['count_had'], how="left")
 X
def detect_outliers(dataframe, n, features):
outliers_indices = []
for feature in features:
Quart1 = dataframe[feature].quantile(0.25)
 Quart3 = dataframe[feature].quantile(0.75)
 upper_outlier_threshold = Quart3 +(Quart3 - Quart1)* 1.5
lower_outlier_threshold = Quart1 -(Quart3 - Quart1)* 1.5
feature_outliers_list = dataframe[(dataframe[feature] > upper_outlier_threshold)|(dataframe[feature] < lower_outlier_threshold)].index
outliers_indices.extend(feature_outliers_list)
 return list(a for a, b in Counter(outliers_indices ).items() if b > n )<filter>
validation = pd.merge(validation, acerto_medio_estudante, on=['user_id'], how="left")
 validation = pd.merge(validation, tempo_medio_estudante, on=['user_id'], how="left")
 validation = pd.merge(validation, acerto_medio_exercicio, on=['content_id'], how="left")
 validation = pd.merge(validation, tempo_medio_exercicio, on=['content_id'], how="left")
 validation = pd.merge(validation, acerto_medio_tipo_exercicio, on=['part'], how="left")
 validation = pd.merge(validation, tempo_medio_tipo_exercicio, on=['part'], how="left")
 validation = pd.merge(validation, viu_medio_estudante, on=['user_id'], how="left")
 validation = pd.merge(validation, viu_medio_exercicio, on=['content_id'], how="left")
 validation = pd.merge(validation, media_numero_exercicio, on=['count'], how="left")
 validation = pd.merge(validation, media_numero_viu, on=['count_had'], how="left")
 validation
outliers = detect_outliers(train, 2, ["Age", "SibSp", "Fare", "Parch"])
 train.loc[outliers]<drop_column>
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )
train = train.drop(outliers, axis = 0 ).reset_index(drop = True )<correct_missing_values>
X.isna().sum()
def fill_age_missing_values(df):
Age_Nan_Indices = list(df[df["Age"].isnull() ].index)
 for index in Age_Nan_Indices:
temp_Pclass = df.iloc[index]["Pclass"]
temp_SibSp = df.iloc[index]["SibSp"]
temp_Parch = df.iloc[index]["Parch"]
age_median = df["Age"][(( df["Pclass"] == temp_Pclass)&(df["SibSp"] == temp_SibSp)&(df["Parch"] == temp_Parch)) ].median()
if df.iloc[index]["Age"]:
df["Age"].iloc[index] = age_median
if np.isnan(age_median):
df["Age"].iloc[index] = df["Age"].median()
return df<count_missing_values>
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )
df["Age"].isnull().sum()<feature_engineering>
X['acerto_medio_estudante'].fillna(acerto_medio_estudante.acerto_medio_estudante.mean() ,inplace=True)
 X['acerto_medio_exercicio'].fillna(acerto_medio_exercicio.acerto_medio_exercicio.mean() ,inplace=True)
 X['tempo_medio_estudante'].fillna(tempo_medio_estudante.tempo_medio_estudante.mean() ,inplace = True)
 X['tempo_medio_exercicio'].fillna(tempo_medio_exercicio.tempo_medio_exercicio.mean() ,inplace = True)
 X['viu_medio_exercicio'].fillna(viu_medio_exercicio.viu_medio_exercicio.mean() ,inplace=True)
 X_val['acerto_medio_estudante'].fillna(acerto_medio_estudante.acerto_medio_estudante.mean() ,inplace=True)
 X_val['acerto_medio_exercicio'].fillna(acerto_medio_exercicio.acerto_medio_exercicio.mean() ,inplace=True)
 X_val['tempo_medio_estudante'].fillna(tempo_medio_estudante.tempo_medio_estudante.mean() ,inplace = True)
 X_val['tempo_medio_exercicio'].fillna(tempo_medio_exercicio.tempo_medio_exercicio.mean() ,inplace = True)
 X_val['viu_medio_exercicio'].fillna(viu_medio_exercicio.viu_medio_exercicio.mean() ,inplace=True )
df["Title"] = pd.Series([name.split(",")[1].split(".")[0].strip() for name in df["Name"]])
 df.head()<categorify>
params = {
'num_leaves': 350,
'max_bin':700,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.05,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}
lgb_train = lgb.Dataset(P, yp)
 lgb_eval = lgb.Dataset(P_val, yp_val, reference=lgb_train )
df["Title"] = df["Title"].replace(['Lady', 'the Countess', 'Countess', 'Don', 'Jonkheer', 'Dona', 'Sir'], 'Royals')
 df["Title"] = df["Title"].replace(['Col', 'Dr', 'Major', 'Capt'], 'Professionals')
 df["Title"] = df["Title"].replace(["Ms", "Mme", "Mlle", "Mrs"], 'Miss')
 df["Title"] = df["Title"].replace(['Master', 'Rev'], 'Mas/Rev')
 df["Title"] = df["Title"].map({"Mas/Rev": 0, "Miss": 1, "Mr": 2, "Royals": 3, "Professionals": 4} )<feature_engineering>
model = lgb.train(
 params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=50,
num_boost_round=10000,
early_stopping_rounds=12
)
df["Ftotal"] = 1 + df["SibSp"] + df["Parch"]<categorify>
y_pred = model.predict(P_val)
 y_true = np.array(yp_val)
 y_predc = model.predict(P)
 y_truec = np.array(yp )
df["Age"] = df["Age"].astype(int)
 df.loc[(df['Age'] <= 2), 'Age Group'] = 'Baby'
df.loc[(( df["Age"] > 2)&(df['Age'] <= 10)) , 'Age Group'] = 'Child'
df.loc[(( df["Age"] > 10)&(df['Age'] <= 19)) , 'Age Group'] = 'Young Adult'
df.loc[(( df["Age"] > 19)&(df['Age'] <= 60)) , 'Age Group'] = 'Adult'
df.loc[(df["Age"] > 60), 'Age Group'] = 'Senior'
df["Age Group"] = df["Age Group"].map({"Baby": 0, "Child": 1, "Young Adult": 2, "Adult": 3, "Senior": 4} )<categorify>
confusion_matrix(y_true, y_pred.round() )
df["Sex"] = df["Sex"].map({"male": 0, "female": 1})
 df.sample(5 )<categorify>
print(roc_auc_score(y_true, y_pred))
print(classification_report(y_true, y_pred.round()))
print(roc_auc_score(y_truec, y_predc))
print(classification_report(y_truec, y_predc.round()))
<drop_column><EOS> 
import matplotlib.pyplot as plt
import seaborn as sns