Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<feature_engineering>
import gc
import json
import pandas as pd
from pathlib import Path
import sqlite3
import riiideducation
import time
import xgboost as xgb
import numpy as np
for dataset in combine:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
train_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()<drop_column>
env = riiideducation.make_env()
iter_test = env.iter_test()
train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 combine = [train_df, test_df]
train_df.head()<feature_engineering>
PATH = Path('.. /input/riiid-submission' )
for dataset in combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
train_df.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10 )<correct_missing_values>
model_xgb = xgb.Booster(model_file=PATH/'model.xgb')
 print('model loaded' )
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port<sort_values>
dtypes = {
'answered_correctly': 'int8',
'answered_correctly_content_id_cumsum': 'int16',
'answered_correctly_content_id_cumsum_pct': 'int16',
'answered_correctly_cumsum': 'int16',
'answered_correctly_cumsum_pct': 'int8',
'answered_correctly_cumsum_upto': 'int8',
'answered_correctly_rollsum': 'int8',
'answered_correctly_rollsum_pct': 'int8',
'answered_incorrectly': 'int8',
'answered_incorrectly_content_id_cumsum': 'int16',
'answered_incorrectly_cumsum': 'int16',
'answered_incorrectly_rollsum': 'int8',
'bundle_id': 'uint16',
'content_id': 'int16',
'content_type_id': 'int8',
'correct_answer': 'uint8',
'lecture_id': 'uint16',
'lectures_cumcount': 'int16',
'part': 'uint8',
'part_correct_pct': 'uint8',
'prior_question_elapsed_time': 'float32',
'prior_question_elapsed_time_rollavg': 'float32',
'prior_question_had_explanation': 'bool',
'question_id': 'uint16',
'question_id_correct_pct': 'uint8',
'row_id': 'int64',
'tag': 'uint8',
'tag__0': 'uint8',
'tag__0_correct_pct': 'uint8',
'tags': 'str',
'task_container_id': 'int16',
'task_container_id_orig': 'int16',
'timestamp': 'int64',
'type_of': 'str',
'user_answer': 'int8',
'user_id': 'int32'
}
batch_cols_all = [
'user_id',
'content_id',
'row_id',
'task_container_id',
'timestamp',
'prior_question_elapsed_time',
'prior_question_had_explanation'
]
batch_cols_prior = [
'user_id',
'content_id',
'content_type_id'
]
with open(PATH/'columns.json')as cj:
test_cols = json.load(cj)
 batch_cols = ['user_id', 'content_id', 'row_id'] + [c for c in batch_cols_all if c in test_cols]
print('test_cols:')
 _ = list(map(print, test_cols))
dtypes_test = {k: v for k,v in dtypes.items() if k in test_cols}
dtypes_test = {**dtypes_test, **{'user_id': 'int32', 'content_id': 'int16'}}
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<data_type_conversions>
df_users_content = pd.read_pickle(PATH/'df_users_content.pkl')
 df_users_content.head()
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()<data_type_conversions>
df_users = df_users_content[['user_id', 'answered_correctly', 'answered_incorrectly']].groupby('user_id' ).sum().reset_index()
df_users = df_users.astype({'user_id': 'int32', 'answered_correctly': 'int16', 'answered_incorrectly': 'int16'})
 df_users.head()
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()<sort_values>
df_questions = pd.read_pickle(PATH/'df_questions.pkl')
 df_questions.head()
train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
 train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False ).mean().sort_values(by='FareBand', ascending=True )<data_type_conversions>
%%time
chunk_size = 20000
total = len(df_users_content)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users_content.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users_content', conn, method='multi', if_exists='append', index=False)
 i += 1
conn.execute('CREATE UNIQUE INDEX users_content_index ON users_content(user_id, content_id)')
 del df_users_content
gc.collect()
for dataset in combine:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand'], axis=1)
 combine = [train_df, test_df]
train_df.head(10 )<prepare_x_and_y>
%%time
pd.read_sql('SELECT * from users_content LIMIT 5', conn )
X_train = train_df.drop("Survived", axis=1)
 Y_train = train_df["Survived"]
X_test = test_df.drop("PassengerId", axis=1 ).copy()
X_train.shape, Y_train.shape, X_test.shape<compute_train_metric>
%%time
chunk_size = 20000
total = len(df_users)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users', conn, method='multi', if_exists='append', index=False)
 i += 1
_ = conn.execute('CREATE UNIQUE INDEX users_index ON users(user_id)')
 del df_users
gc.collect()
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 Y_pred1 = logreg.predict(X_test)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2)
 acc_log<compute_train_metric>
%%time
pd.read_sql('SELECT * from users LIMIT 5', conn )
svc = SVC()
svc.fit(X_train, Y_train)
 Y_pred2 = svc.predict(X_test)
 acc_svc = round(svc.score(X_train, Y_train)* 100, 2)
 acc_svc<compute_train_metric>
%%time
q_cols = [
'question_id',
'part',
'tag__0',
'part_correct_pct',
'tag__0_correct_pct',
'question_id_correct_pct'
]
df_questions[q_cols].to_sql('questions', conn, method='multi', index=False)
 _ = conn.execute('CREATE UNIQUE INDEX question_id_index ON questions(question_id)')
 del df_questions
gc.collect()
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 Y_pred3 = knn.predict(X_test)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2)
 acc_knn<predict_on_test>
%%time
pd.read_sql('SELECT * from questions LIMIT 5', conn )
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 Y_pred4 = gaussian.predict(X_test)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2)
 acc_gaussian<compute_train_metric>
db_size = pd.read_sql('SELECT page_count * page_size as size FROM pragma_page_count() , pragma_page_size() ', conn)['size'][0]
print(f'Total size of database is: {db_size/1e9:0.3f} GB' )
perceptron = Perceptron()
perceptron.fit(X_train, Y_train)
 Y_pred5 = perceptron.predict(X_test)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2)
 acc_perceptron<compute_train_metric>
def select_state(batch_cols, records):
return f
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 Y_pred6 = linear_svc.predict(X_test)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2)
 acc_linear_svc<compute_train_metric>
def update_state(df):
def get_select_params(r):
values_uc = f'({r.user_id}, {r.content_id}, {r.answered_correctly}, {1-r.answered_correctly})'
values_u = f'({r.user_id}, {r.answered_correctly}, {1-r.answered_correctly})'
return values_uc, values_u
values = df.apply(get_select_params, axis=1, result_type='expand')
 return f
sgd = SGDClassifier()
sgd.fit(X_train, Y_train)
 Y_pred7 = sgd.predict(X_test)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2)
 acc_sgd<compute_train_metric>
def data_transform(df,FEATURE_COLS, is_training = True, is_validation = True):
df = df.join(Question_df['question_average'],on=['content_id'],rsuffix='_question_average')
 df = df.join(user_df[['performance','user_average', 'user_count']],on=['user_id'],rsuffix='_right')
 df['is_beginning'] = df['user_count'] < 20
df['attemp'] = df.groupby(['user_id','content_id'] ).content_id.transform('cumcount' ).astype(np.uint8)
 df = df.loc[df['content_type_id'] == False]
if is_training or is_validation:
df = df[FEATURE_COLS1 + TARGET_COL]
else:
df = df[FEATURE_COLS1]
return df
def build_user_df(prior_df,user_df,question_df):
if(prior_df.shape[0] == 0):
return user_df
user_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'user_sum'})
 user_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['user_id'] ).size() ,columns=['user_count'])
 user_df = user_df.join(user_sum_prior,how='outer',rsuffix='_previous' ).join(user_count_prior,rsuffix='_previous')
 user_df['performance'].fillna(0,inplace=True)
 user_df['user_average'].fillna(0,inplace=True)
 user_df['user_count'].fillna(0,inplace=True)
 user_df['user_sum'].fillna(0,inplace=True)
 user_df['user_count_previous'].fillna(0,inplace=True)
 user_df['user_sum_previous'].fillna(0,inplace=True)
 user_df = user_df.join(question_average_sum_by_user(prior_df,question_df))
user_df['question_average_sum'].fillna(0,inplace=True)
 user_df['user_mean_performance'] =(user_df['user_sum'] - user_df['performance'] * user_df['user_count'] + user_df['question_average_sum'])/(user_df['user_count'] + user_df['user_count_previous'])
 user_df['user_sum'] = user_df['user_sum'] + user_df['user_sum_previous']
user_df['user_count'] = user_df['user_count'] + user_df['user_count_previous']
user_df['user_average'] = user_df['user_sum'] / user_df['user_count']
user_df['performance'] = user_df['user_average'] - user_df['user_mean_performance']
user_df.drop(['user_sum_previous','user_count_previous','question_average_sum','user_mean_performance'],axis=1,inplace=True)
 return user_df
def build_question_df(prior_df,question_df):
if(prior_df.shape[0] == 0):
return question_df
question_sum_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'])['answered_correctly_response'].sum())\ 
.rename(columns={'answered_correctly_response':'question_sum'})
 question_count_prior = pd.DataFrame(prior_df.loc[prior_df['content_type_id'] == 0]\ 
.groupby(['content_id'] ).size() ,columns=['question_count'])
 question_df = question_df.join(question_sum_prior,rsuffix='_previous' ).join(question_count_prior,rsuffix='_previous')
 question_df['question_average'].fillna(0,inplace=True)
 question_df['question_count'].fillna(0,inplace=True)
 question_df['question_sum'].fillna(0,inplace=True)
 question_df['question_sum_previous'].fillna(0,inplace=True)
 question_df['question_count_previous'].fillna(0,inplace=True)
 question_df['question_sum'] = question_df['question_sum'] + question_df['question_sum_previous']
question_df['question_count'] = question_df['question_count'] + question_df['question_count_previous']
question_df['question_average'] = question_df['question_sum'] / question_df['question_count']
question_df.drop(['question_count_previous','question_sum_previous'],inplace=True,axis=1)
 return question_df
def add_answers_to_prior_df(current_df,prior_df):
prior_df_ = prior_df.copy()
if(prior_df.shape[0] > 0):
val = eval(current_df.iloc[0]['prior_group_answers_correct'])
 if(len(val)== prior_df.shape[0]):
prior_df_['answered_correctly_response'] = val
return prior_df_
def question_average_sum_by_user(df,question_df):
my_dict = {}
group = df.groupby(['user_id'])
 for user, val in group:
average_sum = 0.0
for row_index, row in val.iterrows() :
if(row['content_type_id'] == False):
question_id = row['content_id']
question_average = question_df.at[question_id,'question_average']
average_sum += question_average
my_dict[user] = [average_sum]
return pd.DataFrame.from_dict(my_dict,orient='index',columns=['question_average_sum'])
 prior_df = pd.DataFrame()
current_df = pd.DataFrame()
TARGET_COL = ['answered_correctly']
FEATURE_COLS1 = ['row_id', 'user_count','performance', 'question_average']
FEATURE_COLS2 = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 Y_pred8 = decision_tree.predict(X_test)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2)
 acc_decision_tree<compute_train_metric>
Question_df = pd.read_csv('.. /input/featuresriid/question.csv')
 user_df = pd.read_csv('.. /input/featuresriid/user.csv' )
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 Y_pred9 = random_forest.predict(X_test)
 random_forest.score(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2)
 acc_random_forest<create_dataframe>
model_lgb1 = lgb.Booster(model_file='.. /input/featuresriid/model_twofeatures.txt')
 model_lgb2 = lgb.Booster(model_file='.. /input/7features760/model.txt' )
models = pd.DataFrame({
'Model': ['Support Vector Machines', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron',
'Stochastic Gradient Decent', 'Linear SVC',
'Decision Tree'],
'Score': [acc_svc, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_sgd, acc_linear_svc, acc_decision_tree]})
 models.sort_values(by='Score', ascending=False )<save_to_csv>
def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['user_id']].values):
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": Y_pred9
})
 submission.to_csv('submission_titanic.csv', index=False )<load_from_csv>
%%time
df_batch_prior = None
counter = 0
iter_nb = 0
TARGET = 'answered_correctly'
for test_batch in iter_test:
counter += 1 
(current_df, sample_prediction_df)= test_batch
if(iter_nb != 0):
prior_df = add_answers_to_prior_df(current_df,prior_df)
 Question_df = build_question_df(prior_df,Question_df)
 user_df = build_user_df(prior_df,user_df,Question_df)
 answers = eval(test_batch[0]['prior_group_answers_correct'].iloc[0])
 df_batch_prior['answered_correctly'] = answers
cursor.executescript(update_state(df_batch_prior[df_batch_prior.content_type_id == 0]))
if not counter % 100:
conn.commit()
prior_df = current_df.copy()
current_df = data_transform(current_df,FEATURE_COLS1,False,False)
 predictions0 = model_lgb1.predict(current_df.iloc[:,1:])
 iter_nb = 1
print('--')
(test_df, sample_prediction_df)= test_batch
answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 content_df = pd.read_csv('.. /input/7fetscontent/content.csv')
 qUestion_df = pd.read_csv('.. /input/7fetscontent/question.csv')
 if df_batch_prior is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df = pd.merge(test_df, qUestion_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(25439.41)
 predictions1 = model_lgb2.predict(test_df[FEATURE_COLS2])
 df_batch_prior = test_batch[0][batch_cols_prior].astype({k: dtypes[k] for k in batch_cols_prior})
 df_batch = test_batch[0][test_batch[0].content_type_id == 0]
records = df_batch[batch_cols].fillna(0 ).to_records(index=False)
 df_batch = pd.read_sql(select_state(batch_cols, records), conn)
 predictions2 = model_xgb.predict(xgb.DMatrix(df_batch[test_cols]))
df_batch['answered_correctly'] = predictions0*0.4+predictions2*0.6
env.predict(df_batch[['row_id', 'answered_correctly']])
<data_type_conversions><EOS> 
predictions2,predictions1,predictions0