Kaggle competition: Riiid Answer Correctness Prediction
train.Parch.value_counts()<count_values>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl
train.groupby('Parch' ).Survived.value_counts()<groupby>
train = dt.fread(".. /input/riiid-test-answer-prediction/train.csv" ).to_pandas()
train[['Parch', 'Survived']].groupby(['Parch'], as_index=False ).mean()<count_values>
train.SibSp.value_counts()<count_values>
data_types_dict = {
'row_id': 'int64',
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'
}
train = train.astype(data_types_dict )
train.groupby('SibSp' ).Survived.value_counts()<groupby>
train_part_len = int(9 /10 * len(train))
train[['SibSp', 'Survived']].groupby(['SibSp'], as_index=False ).mean()<filter>
def convertBoolean(x):
if str(x)== "False":
return 0
elif str(x)== "True":
return 1
else:
return 0
def TOEICSection(part):
if part >= 1 and part <= 4:
return "Listening"
elif part >= 5 and part <= 7:
return "Reading"
else:
return "Missing"
def tagsCount(tags):
arr = str(tags ).split(" ")
 return len(arr )
total_survived = train[train['Survived']==1]
total_not_survived = train[train['Survived']==0]
male_survived = train[(train['Survived']==1)&(train['Sex']=="male")]
female_survived = train[(train['Survived']==1)&(train['Sex']=="female")]
male_not_survived = train[(train['Survived']==0)&(train['Sex']=="male")]
female_not_survived = train[(train['Survived']==0)&(train['Sex']=="female")]<feature_engineering>
questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv' )
train_test_data = [train, test]
for dataset in train_test_data:
dataset['Title'] = dataset['Name'].str.extract('([A-Za-z]+)\.' )<feature_engineering>
tag = questions["tags"].str.split(" ", expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col', \
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Other')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<categorify>
questions = pd.concat([questions,tag],axis=1)
 questions['tags1'] = pd.to_numeric(questions['tags1'], errors='coerce')
 questions['tags2'] = pd.to_numeric(questions['tags2'], errors='coerce')
 questions['tags3'] = pd.to_numeric(questions['tags3'], errors='coerce')
 questions['tags4'] = pd.to_numeric(questions['tags4'], errors='coerce')
 questions['tags5'] = pd.to_numeric(questions['tags5'], errors='coerce')
 questions['tags6'] = pd.to_numeric(questions['tags6'], errors='coerce' )
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Other": 5}
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0 )<data_type_conversions>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int )<count_values>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
train.Embarked.value_counts()<data_type_conversions>
train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].fillna('S' )<data_type_conversions>
user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int )<feature_engineering>
for column in user_lecture_stats_part.columns:
bool_column = column + '_boolean'
user_lecture_stats_part[bool_column] =(user_lecture_stats_part[column] > 0 ).astype(int )
for dataset in train_test_data:
age_avg = dataset['Age'].mean()
age_std = dataset['Age'].std()
age_null_count = dataset['Age'].isnull().sum()
age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)
 dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list
dataset['Age'] = dataset['Age'].astype(int)
 train['AgeBand'] = pd.cut(train['Age'], 5)
 print(train[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean() )<feature_engineering>
train_lectures = None
del(train_lectures )
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age'] = 4<feature_engineering>
gc.collect()
for dataset in train_test_data:
dataset['Fare'] = dataset['Fare'].fillna(train['Fare'].median() )<data_type_conversions>
def split_X_Y(_train):
_train = _train[_train.content_type_id == False]
_train = _train.sort_values(['timestamp'], ascending=True)
 _train.drop(['timestamp','content_type_id'], axis=1, inplace=True)
 X = _train.iloc[train_part_len:,:]
X = X[X.answered_correctly!= -1 ]
X = X.sort_values(['user_id'])
 Y = X[["answered_correctly"]]
X = X.drop(["answered_correctly"], axis=1)
 return X, Y
for dataset in train_test_data:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int )<groupby>
count_en = ce.CountEncoder()
for dataset in train_test_data:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
print(train[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean() )<feature_engineering>
def addFeatures(_train, _questions):
train_questions_only_df = _train[_train['answered_correctly']!=-1]
grouped_by_user_df = train_questions_only_df.groupby('user_id')
 user_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count', 'sum']} ).copy()
user_answers_df.columns = ['mean_answered_correctly_user', 'questions_answered', 'sum_answered_correctly_user']
grouped_by_content_df = train_questions_only_df.groupby('content_id')
 content_answers_df = grouped_by_content_df.agg({'answered_correctly': ['mean', 'count'] } ).copy()
content_answers_df.columns = ['mean_answered_correctly_content', 'question_asked']
questions_df = _questions.merge(content_answers_df, left_on = 'question_id', right_on = 'content_id', how = 'left')
 bundle_dict = questions_df['bundle_id'].value_counts().to_dict()
questions_df['right_answers'] = questions_df['mean_answered_correctly_content'] * questions_df['question_asked']
questions_df['bundle_size'] = questions_df['bundle_id'].apply(lambda x: bundle_dict[x])
 grouped_by_bundle_df = questions_df.groupby('bundle_id')
 bundle_answers_df = grouped_by_bundle_df.agg({'right_answers': 'sum', 'question_asked': 'sum'} ).copy()
bundle_answers_df.columns = ['bundle_rignt_answers', 'bundle_questions_asked']
bundle_answers_df['bundle_accuracy'] = bundle_answers_df['bundle_rignt_answers'] / bundle_answers_df['bundle_questions_asked']
grouped_by_part_df = questions_df.groupby('part')
 part_answers_df = grouped_by_part_df.agg({'right_answers': 'sum', 'question_asked': 'sum'} ).copy()
part_answers_df.columns = ['part_rignt_answers', 'part_questions_asked']
part_answers_df['part_accuracy'] = part_answers_df['part_rignt_answers'] / part_answers_df['part_questions_asked']
return user_answers_df, questions_df, bundle_answers_df, part_answers_df
for dataset in train_test_data:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
print(train[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean() )<drop_column>
def handleMissing(X, Y, _user_answers, _questions, _bundle_answers, _part_answers, _lectures):
X['prior_question_had_explanation'].fillna(False, inplace=True)
 X["prior_question_had_explanation_enc"] = X['prior_question_had_explanation'].apply(convertBoolean)
 X = X[['user_id', 'content_id', 'prior_question_elapsed_time','prior_question_had_explanation_enc']]
X = X.merge(_user_answers, how = 'left', on = 'user_id')
 X = X.merge(_questions, how = 'left', left_on = 'content_id', right_on = 'question_id')
 X = X.merge(_bundle_answers, how = 'left', on = 'bundle_id')
 X = X.merge(_part_answers, how = 'left', on = 'part')
 X = X.merge(_lectures, how = 'left', on=['user_id'])
 X["toeic_section"] = X['part'].apply(TOEICSection)
 cat_ce = count_en.fit_transform(X["toeic_section"])
 X = X.join(cat_ce.add_suffix("_ce"))
X["tags_count"] = X['tags'].apply(tagsCount)
 X = X.drop(columns=['user_id', 'content_id', 'bundle_id', 'correct_answer', 'tags', 'toeic_section'])
 X['sum_answered_correctly_user'].fillna(0, inplace=True)
 X['part_1'].fillna(0, inplace = True)
 X['part_2'].fillna(0, inplace = True)
 X['part_3'].fillna(0, inplace = True)
 X['part_4'].fillna(0, inplace = True)
 X['part_5'].fillna(0, inplace = True)
 X['part_6'].fillna(0, inplace = True)
 X['part_7'].fillna(0, inplace = True)
 X['type_of_concept'].fillna(0, inplace = True)
 X['type_of_intention'].fillna(0, inplace = True)
 X['type_of_solving_question'].fillna(0, inplace = True)
 X['type_of_starter'].fillna(0, inplace = True)
 X['part_1_boolean'].fillna(0, inplace = True)
 X['part_2_boolean'].fillna(0, inplace = True)
 X['part_3_boolean'].fillna(0, inplace = True)
 X['part_4_boolean'].fillna(0, inplace = True)
 X['part_5_boolean'].fillna(0, inplace = True)
 X['part_6_boolean'].fillna(0, inplace = True)
 X['part_7_boolean'].fillna(0, inplace = True)
 X['type_of_concept_boolean'].fillna(0, inplace = True)
 X['type_of_intention_boolean'].fillna(0, inplace = True)
 X['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X['type_of_starter_boolean'].fillna(0, inplace = True)
 X.fillna(0.5, inplace=True)
 target_columns = ['question_id', 'part']
target_enc = ce.TargetEncoder(cols=target_columns)
 target_enc.fit(X[target_columns], Y)
 X = X.join(target_enc.transform(X[target_columns] ).add_suffix('_target'))
catboost_columns = ['prior_question_elapsed_time', 'prior_question_had_explanation_enc']
catboost_enc = ce.CatBoostEncoder(cols=catboost_columns)
 catboost_enc.fit(X[catboost_columns], Y)
 X = X.join(catboost_enc.transform(X[catboost_columns] ).add_suffix('_catboost'))
return X, target_enc, catboost_enc
features_drop = ['Name', 'SibSp', 'Parch', 'Ticket', 'Cabin', 'FamilySize']
train = train.drop(features_drop, axis=1)
 test = test.drop(features_drop, axis=1)
 train = train.drop(['PassengerId', 'AgeBand', 'FareBand'], axis=1 )<prepare_x_and_y>
X_full, Y_full = split_X_Y(train )
X_train = train.drop('Survived', axis=1)
 y_train = train['Survived']
X_test = test.drop("PassengerId", axis=1 ).copy()
X_train.shape, y_train.shape, X_test.shape<import_modules>
user_answers_df, questions_df, bundle_answers_df, part_answers_df = addFeatures(train, questions )
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC, LinearSVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import Perceptron
from sklearn.linear_model import SGDClassifier<train_model>
train = None
del(train)
 gc.collect()
clf = LogisticRegression()
clf.fit(X_train, y_train)
 y_pred_log_reg = clf.predict(X_test)
 acc_log_reg = round(clf.score(X_train, y_train)* 100, 2)
 print("Train Accuracy: " + str(acc_log_reg)+ '%' )<compute_train_metric>
X_full, target_enc, catboost_enc = handleMissing(X_full, Y_full, user_answers_df, questions_df, bundle_answers_df, part_answers_df, user_lecture_stats_part )
clf = SVC()
clf.fit(X_train, y_train)
 y_pred_svc = clf.predict(X_test)
 acc_svc = round(clf.score(X_train, y_train)* 100, 2)
 print("Train Accuracy: " + str(acc_svc)+ '%' )<compute_train_metric>
from sklearn.metrics import roc_auc_score
from xgboost import XGBClassifier, plot_importance
import matplotlib.pyplot as plt
clf = LinearSVC()
clf.fit(X_train, y_train)
 y_pred_linear_svc = clf.predict(X_test)
 acc_linear_svc = round(clf.score(X_train, y_train)* 100, 2)
 print("Train Accuracy: " + str(acc_linear_svc)+ '%' )<predict_on_test>
model = XGBClassifier(
 tree_method="hist",
learning_rate=0.1,
gamma=0.2,
n_estimators=200,
max_depth=8,
min_child_weight=40,
subsample=0.87,
colsample_bytree=0.95,
reg_alpha=0.04,
reg_lambda=0.073,
objective='binary:logistic',
nthread=4,
scale_pos_weight=1,
seed=27
)
clf = KNeighborsClassifier(n_neighbors = 3)
 clf.fit(X_train, y_train)
 y_pred_knn = clf.predict(X_test)
 acc_knn = round(clf.score(X_train, y_train)* 100, 2)
 print("Train Accuracy: " + str(acc_knn)+ '%' )<choose_model_class>
model.fit(X_full, Y_full.values.ravel() )
clf = DecisionTreeClassifier()
clf.fit(X_train, y_train)
 y_pred_decision_tree = clf.predict(X_test)
 acc_decision_tree = round(clf.score(X_train, y_train)* 100, 2)
 print("Train Accuracy: " + str(acc_decision_tree)+ '%' )<train_model>
roc_auc_score(Y_full.values, model.predict_proba(X_full)[:,1] )
clf = RandomForestClassifier(n_estimators=100)
 clf.fit(X_train, y_train)
 y_pred_random_forest = clf.predict(X_test)
 acc_random_forest = round(clf.score(X_train, y_train)* 100, 2)
 print("Train Accuracy: " + str(acc_random_forest)+ '%' )<compute_train_metric>
env = riiideducation.make_env()
iter_test = env.iter_test()
<compute_train_metric><EOS> 
for(test_df, sample_prediction_df)in iter_test:
test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = test_df['prior_question_had_explanation'].apply(convertBoolean)
 test_df = test_df.merge(user_answers_df, how = 'left', on = 'user_id')
 test_df = test_df.merge(questions_df, how = 'left', left_on = 'content_id', right_on = 'question_id')
 test_df = test_df.merge(bundle_answers_df, how = 'left', on = 'bundle_id')
 test_df = test_df.merge(part_answers_df, how = 'left', on = 'part')
 test_df = test_df.merge(user_lecture_stats_part, how = 'left', on=['user_id'])
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['part_1_boolean'].fillna(0, inplace = True)
 test_df['part_2_boolean'].fillna(0, inplace = True)
 test_df['part_3_boolean'].fillna(0, inplace = True)
 test_df['part_4_boolean'].fillna(0, inplace = True)
 test_df['part_5_boolean'].fillna(0, inplace = True)
 test_df['part_6_boolean'].fillna(0, inplace = True)
 test_df['part_7_boolean'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df["toeic_section"] = test_df['part'].apply(TOEICSection)
 cat_ce = count_en.fit_transform(test_df["toeic_section"])
 test_df = test_df.join(cat_ce.add_suffix("_ce"))
test_df["tags_count"] = test_df['tags'].apply(tagsCount)
 test_df['sum_answered_correctly_user'].fillna(0, inplace=True)
 test_df.fillna(0.5, inplace=True)
 target_columns = ['question_id', 'part']
test_df = test_df.join(target_enc.transform(test_df[target_columns] ).add_suffix('_target'))
catboost_columns = ['prior_question_elapsed_time', 'prior_question_had_explanation_enc']
test_df = test_df.join(catboost_enc.transform(test_df[catboost_columns] ).add_suffix('_catboost'))
test_df['answered_correctly'] = model.predict_proba(test_df[X_full.columns])[:,1]
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )