Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<create_dataframe>
import gc
import json
import pandas as pd
from pathlib import Path
import sqlite3
import riiideducation
import time
import xgboost as xgb
stat_age = all_df.pivot_table(values='Age', index=group_features, aggfunc=['mean','std'] ).round(2 )<sort_values>
env = riiideducation.make_env()
iter_test = env.iter_test()
def sort_remap(crosstab_df, key):
alives = list(crosstab_df[crosstab_df['Survived']==1][key])
 deads = list(crosstab_df[crosstab_df['Survived']==0][key])
 alives = list(set(deads)- set(alives)) + alives
sorted_map = {key:value for value, key in enumerate(alives)}
return sorted_map<train_model>
PATH = Path('.. /input/riiid-submission' )
kdis = KBinsDiscretizer(n_bins=8, encode='ordinal', strategy='uniform' ).fit(all_df[['Age', 'Fare']] )<categorify>
model = xgb.Booster(model_file=PATH/'model.xgb')
 print('model loaded' )
cat_age_fare = kdis.transform(all_df[['Age', 'Fare']] )<data_type_conversions>
dtypes = {
'answered_correctly': 'int8',
'answered_correctly_content_id_cumsum': 'int16',
'answered_correctly_content_id_cumsum_pct': 'int16',
'answered_correctly_cumsum': 'int16',
'answered_correctly_cumsum_pct': 'int8',
'answered_correctly_cumsum_upto': 'int8',
'answered_correctly_rollsum': 'int8',
'answered_correctly_rollsum_pct': 'int8',
'answered_incorrectly': 'int8',
'answered_incorrectly_content_id_cumsum': 'int16',
'answered_incorrectly_cumsum': 'int16',
'answered_incorrectly_rollsum': 'int8',
'bundle_id': 'uint16',
'content_id': 'int16',
'content_type_id': 'int8',
'correct_answer': 'uint8',
'lecture_id': 'uint16',
'lectures_cumcount': 'int16',
'part': 'uint8',
'part_correct_pct': 'uint8',
'prior_question_elapsed_time': 'float32',
'prior_question_elapsed_time_rollavg': 'float32',
'prior_question_had_explanation': 'bool',
'question_id': 'uint16',
'question_id_correct_pct': 'uint8',
'row_id': 'int64',
'tag': 'uint8',
'tag__0': 'uint8',
'tag__0_correct_pct': 'uint8',
'tags': 'str',
'task_container_id': 'int16',
'task_container_id_orig': 'int16',
'timestamp': 'int64',
'type_of': 'str',
'user_answer': 'int8',
'user_id': 'int32'
}
batch_cols_all = [
'user_id',
'content_id',
'row_id',
'task_container_id',
'timestamp',
'prior_question_elapsed_time',
'prior_question_had_explanation'
]
batch_cols_prior = [
'user_id',
'content_id',
'content_type_id'
]
with open(PATH/'columns.json')as cj:
test_cols = json.load(cj)
 batch_cols = ['user_id', 'content_id', 'row_id'] + [c for c in batch_cols_all if c in test_cols]
print('test_cols:')
 _ = list(map(print, test_cols))
dtypes_test = {k: v for k,v in dtypes.items() if k in test_cols}
dtypes_test = {**dtypes_test, **{'user_id': 'int32', 'content_id': 'int16'}}
all_df['Cat_Age'] = cat_age_fare[:,0].astype('int')
 all_df['Cat_Fare'] = cat_age_fare[:,1].astype('int' )<define_variables>
df_users_content = pd.read_pickle(PATH/'df_users_content.pkl')
 df_users_content.head()
features_on_off = {'PassengerId':False,
'Survived':False,
'Pclass':True,
'Name':False,
'Sex':True,
'Age':False,
'SibSp':True,
'Parch':True,
'Ticket':False,
'Fare':False,
'Cabin':False,
'Embarked':True,
'Title':True,
'Deck':True,
'Alone':True,
'Relatives':True,
'Cat_Age':True,
'Cat_Fare':True}<create_dataframe>
df_users = df_users_content[['user_id', 'answered_correctly', 'answered_incorrectly']].groupby('user_id' ).sum().reset_index()
df_users = df_users.astype({'user_id': 'int32', 'answered_correctly': 'int16', 'answered_incorrectly': 'int16'})
 df_users.head()
poly = PolynomialFeatures(degree=2, include_bias=False ).fit(all_df[features_on])
 poly_features = poly.transform(all_df[features_on])
 poly_df = pd.DataFrame(data=poly_features, columns=poly.get_feature_names(features_on))<normalization>
df_questions = pd.read_pickle(PATH/'df_questions.pkl')
 df_questions.head()
std_scaler = StandardScaler().fit(poly_df)
 scaled_features = std_scaler.transform(poly_df)
 scaled_df = pd.DataFrame(data=scaled_features, columns=poly_df.columns )<create_dataframe>
%%time
chunk_size = 20000
total = len(df_users_content)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users_content.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users_content', conn, method='multi', if_exists='append', index=False)
 i += 1
conn.execute('CREATE UNIQUE INDEX users_content_index ON users_content(user_id, content_id)')
 del df_users_content
gc.collect()
new_df = scaled_df.copy()
new_df['Survived'] = all_df['Survived']<prepare_x_and_y>
%%time
pd.read_sql('SELECT * from users_content LIMIT 5', conn )
X_train = scaled_df.loc[range(train_df.shape[0])]
y_train = all_df.loc[range(train_df.shape[0]), 'Survived']
X_test = scaled_df.loc[range(train_df.shape[0], train_df.shape[0]+test_df.shape[0])]
y_test = pd.read_csv('.. /input/true-labels/submission_true.csv')['Survived']<train_on_grid>
%%time
chunk_size = 20000
total = len(df_users)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users', conn, method='multi', if_exists='append', index=False)
 i += 1
_ = conn.execute('CREATE UNIQUE INDEX users_index ON users(user_id)')
 del df_users
gc.collect()
log_reg = LogisticRegressionCV(Cs=1000, cv=5, refit=True, random_state=101, max_iter=200 ).fit(X_train, y_train )<compute_test_metric>
%%time
pd.read_sql('SELECT * from users LIMIT 5', conn )
print(f'Accuracy on the training set: {100*log_reg.score(X_train, y_train):.1f}%' )<compute_test_metric>
%%time
q_cols = [
'question_id',
'part',
'tag__0',
'part_correct_pct',
'tag__0_correct_pct',
'question_id_correct_pct'
]
df_questions[q_cols].to_sql('questions', conn, method='multi', index=False)
 _ = conn.execute('CREATE UNIQUE INDEX question_id_index ON questions(question_id)')
 del df_questions
gc.collect()
print(f'Accuracy on the test set: {100*log_reg.score(X_test, y_test):.1f}%' )<compute_train_metric>
%%time
pd.read_sql('SELECT * from questions LIMIT 5', conn )
p = np.linspace(0.01, 1, 50)
 error_rate = {'train':[], 'test':[]}
for p_value in p:
prob = log_reg.predict_proba(X_train)
 y_pred = np.apply_along_axis(lambda pair: 1 if pair[1] > p_value else 0, 1, prob)
 error_rate['train'].append(mean_squared_error(y_train, y_pred))
prob = log_reg.predict_proba(X_test)
 y_pred = np.apply_along_axis(lambda pair: 1 if pair[1] > p_value else 0, 1, prob)
 error_rate['test'].append(mean_squared_error(y_test, y_pred))<sort_values>
db_size = pd.read_sql('SELECT page_count * page_size as size FROM pragma_page_count() , pragma_page_size() ', conn)['size'][0]
print(f'Total size of database is: {db_size/1e9:0.3f} GB' )
best_p = p[np.array(error_rate['test'] ).argmin() ]<find_best_params>
def select_state(batch_cols, records):
return f
print(f'In this particular case, the best probability limit is around {best_p:.2}.This means that if the model provides a survival value greater than {best_p:.2}, we accept that the passenger will survive or die otherwise.' )<predict_on_test>
def update_state(df):
def get_select_params(r):
values_uc = f'({r.user_id}, {r.content_id}, {r.answered_correctly}, {1-r.answered_correctly})'
values_u = f'({r.user_id}, {r.answered_correctly}, {1-r.answered_correctly})'
return values_uc, values_u
values = df.apply(get_select_params, axis=1, result_type='expand')
 return f
<predict_on_test><EOS> 
%%time
df_batch_prior = None
counter = 0
for test_batch in iter_test:
counter += 1
if df_batch_prior is not None:
answers = eval(test_batch[0]['prior_group_answers_correct'].iloc[0])
 df_batch_prior['answered_correctly'] = answers
cursor.executescript(update_state(df_batch_prior[df_batch_prior.content_type_id == 0]))
if not counter % 100:
conn.commit()
df_batch_prior = test_batch[0][batch_cols_prior].astype({k: dtypes[k] for k in batch_cols_prior})
 df_batch = test_batch[0][test_batch[0].content_type_id == 0]
records = df_batch[batch_cols].fillna(0 ).to_records(index=False)
 df_batch = pd.read_sql(select_state(batch_cols, records), conn)
 predictions = model.predict(xgb.DMatrix(df_batch[test_cols]))
df_batch['answered_correctly'] = predictions
env.predict(df_batch[['row_id', 'answered_correctly']] )