Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<data_type_conversions>
import gc
import json
import pandas as pd
from pathlib import Path
import sqlite3
import riiideducation
import time
import xgboost as xgb
merged['Fare'].fillna(value= merged['Fare'].median() , inplace=True )<categorify>
env = riiideducation.make_env()
iter_test = env.iter_test()
df = merged.loc[:, ['Sex', 'Pclass', 'Embarked', 'Title', 'Family_size', 'Parch', 'SibSp', 'Cabin', 'Ticket']]
LE = LabelEncoder()
df = df.apply(LE.fit_transform)
 df.head(5)
<feature_engineering>
PATH = Path('.. /input/riiid-submission' )
df['Age'] = merged['Age']
df.head(2 )<drop_column>
model = xgb.Booster(model_file=PATH/'model.xgb')
 print('model loaded' )
df = df.set_index('Age' ).reset_index()
df.head(2 )<data_type_conversions>
dtypes = {
'answered_correctly': 'int8',
'answered_correctly_content_id_cumsum': 'int16',
'answered_correctly_content_id_cumsum_pct': 'int16',
'answered_correctly_cumsum': 'int16',
'answered_correctly_cumsum_pct': 'int8',
'answered_correctly_cumsum_upto': 'int8',
'answered_correctly_rollsum': 'int8',
'answered_correctly_rollsum_pct': 'int8',
'answered_incorrectly': 'int8',
'answered_incorrectly_content_id_cumsum': 'int16',
'answered_incorrectly_cumsum': 'int16',
'answered_incorrectly_rollsum': 'int8',
'bundle_id': 'uint16',
'content_id': 'int16',
'content_type_id': 'int8',
'correct_answer': 'uint8',
'lecture_id': 'uint16',
'lectures_cumcount': 'int16',
'part': 'uint8',
'part_correct_pct': 'uint8',
'prior_question_elapsed_time': 'float32',
'prior_question_elapsed_time_rollavg': 'float32',
'prior_question_had_explanation': 'bool',
'question_id': 'uint16',
'question_id_correct_pct': 'uint8',
'row_id': 'int64',
'tag': 'uint8',
'tag__0': 'uint8',
'tag__0_correct_pct': 'uint8',
'tags': 'str',
'task_container_id': 'int16',
'task_container_id_orig': 'int16',
'timestamp': 'int64',
'type_of': 'str',
'user_answer': 'int8',
'user_id': 'int32'
}
batch_cols_all = [
'user_id',
'content_id',
'row_id',
'task_container_id',
'timestamp',
'prior_question_elapsed_time',
'prior_question_had_explanation'
]
batch_cols_prior = [
'user_id',
'content_id',
'content_type_id'
]
with open(PATH/'columns.json')as cj:
test_cols = json.load(cj)
 batch_cols = ['user_id', 'content_id', 'row_id'] + [c for c in batch_cols_all if c in test_cols]
print('test_cols:')
 _ = list(map(print, test_cols))
dtypes_test = {k: v for k,v in dtypes.items() if k in test_cols}
dtypes_test = {**dtypes_test, **{'user_id': 'int32', 'content_id': 'int16'}}
merged['Age'] = merged.groupby(['Title', 'Pclass'])['Age'].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
df_users_content = pd.read_pickle(PATH/'df_users_content.pkl')
 df_users_content.head()
merged.loc[:, ['Pclass', 'Sex', 'Embarked', 'Cabin', 'Title', 'Family_size', 'Ticket']] = merged.loc[:, ['Pclass', 'Sex', 'Embarked', 'Cabin', 'Title', 'Family_size', 'Ticket']].astype('category')
 merged['Survived'] = merged['Survived'].dropna().astype('int' )<drop_column>
df_users = df_users_content[['user_id', 'answered_correctly', 'answered_incorrectly']].groupby('user_id' ).sum().reset_index()
df_users = df_users.astype({'user_id': 'int32', 'answered_correctly': 'int16', 'answered_incorrectly': 'int16'})
 df_users.head()
merged.drop(columns = ['Name', 'Age','SibSp', 'Parch','Fare'], inplace = True, axis = 1)
 merged.columns<categorify>
df_questions = pd.read_pickle(PATH/'df_questions.pkl')
 df_questions.head()
merged = pd.get_dummies(merged, drop_first=True)
 merged.head(2 )<split>
%%time
chunk_size = 20000
total = len(df_users_content)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users_content.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users_content', conn, method='multi', if_exists='append', index=False)
 i += 1
conn.execute('CREATE UNIQUE INDEX users_content_index ON users_content(user_id, content_id)')
 del df_users_content
gc.collect()
train = merged.iloc[:891, :]
test = merged.iloc[891:, :]<drop_column>
%%time
pd.read_sql('SELECT * from users_content LIMIT 5', conn )
train = train.drop(columns = ['PassengerId'], axis = 1)
 test = test.drop(columns = ['Survived'], axis = 1 )<prepare_x_and_y>
%%time
chunk_size = 20000
total = len(df_users)
 n_chunks =(total // chunk_size + 1)
 i = 0
while i < n_chunks:
df_users.iloc[i * chunk_size:(i + 1)* chunk_size].to_sql('users', conn, method='multi', if_exists='append', index=False)
 i += 1
_ = conn.execute('CREATE UNIQUE INDEX users_index ON users(user_id)')
 del df_users
gc.collect()
X_train = train.drop(columns = ['Survived'], axis = 1)
 y_train = train['Survived']
X_test = test.drop("PassengerId", axis = 1 ).copy()<choose_model_class>
%%time
pd.read_sql('SELECT * from users LIMIT 5', conn )
LR = LogisticRegression()
KNN = KNeighborsClassifier()
DT = DecisionTreeClassifier()
RF = RandomForestClassifier()
SVM = SVC(gamma='auto')
 XGB = XGBClassifier(n_jobs=-1, random_state=42 )<train_model>
%%time
q_cols = [
'question_id',
'part',
'tag__0',
'part_correct_pct',
'tag__0_correct_pct',
'question_id_correct_pct'
]
df_questions[q_cols].to_sql('questions', conn, method='multi', index=False)
 _ = conn.execute('CREATE UNIQUE INDEX question_id_index ON questions(question_id)')
 del df_questions
gc.collect()
def train_accuracy(model):
model.fit(X_train, y_train)
 train_accuracy = model.score(X_train, y_train)
 train_accuracy = np.round(train_accuracy*100,2)
 return train_accuracy<create_dataframe>
%%time
pd.read_sql('SELECT * from questions LIMIT 5', conn )
train_accuracy = pd.DataFrame({'Training accuracy %':[train_accuracy(LR),train_accuracy(KNN),train_accuracy(DT),train_accuracy(RF),train_accuracy(SVM), train_accuracy(XGB)]})
 train_accuracy.index = ['Logistic Regression','KNN','Decision Tree','Random Forest','SVM','XGB']
sorted_train_accuracy = train_accuracy.sort_values(by='Training accuracy %', ascending = False)
 sorted_train_accuracy<compute_train_metric>
db_size = pd.read_sql('SELECT page_count * page_size as size FROM pragma_page_count() , pragma_page_size() ', conn)['size'][0]
print(f'Total size of database is: {db_size/1e9:0.3f} GB' )
def val_score(model):
val_score = cross_val_score(model, X_train, y_train, cv = 10, scoring = 'accuracy' ).mean()
val_score = np.round(val_score*100, 2)
 return val_score
val_score = pd.DataFrame({'val_score(%)':[val_score(LR), val_score(KNN), val_score(DT), val_score(RF), val_score(SVM), val_score(XGB)]})
 val_score.index = ['Logistic Regression', 'KNN','Decision Tree', 'Random Forest', 'SVC','XGB']
sorted_val_score = val_score.sort_values(by = 'val_score(%)', ascending = False)
 sorted_val_score 
<init_hyperparams>
def select_state(batch_cols, records):
return f
lr_params = {'penalty':['l1', 'l2'],
'C': np.logspace(0, 2, 4, 8 ,10)}
knn_params = {'n_neighbors':[4,5,6,7,8,9,10],
'weights':['uniform', 'distance'],
'algorithm':['auto', 'ball_tree','kd_tree','brute'],
'p':[1,2]}
dt_params = {'max_features': ['auto', 'sqrt', 'log2'],
'min_samples_split': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
'min_samples_leaf':[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
'random_state':[46]}
rf_params = {'criterion':['gini','entropy'],
'n_estimators':[ 10, 30, 200, 400],
'min_samples_leaf':[1, 2, 3],
'min_samples_split':[3, 4, 6, 7],
'max_features':['sqrt', 'auto', 'log2'],
'random_state':[46]}
svc_params = {'C': [0.1, 1, 10,100],
'kernel': ['linear', 'rbf', 'poly', 'sigmoid'],
'gamma': [ 1, 0.1, 0.001, 0.0001]}
xgb_params = xgb_params_grid = {"learning_rate" : [0.05, 0.10, 0.15, 0.20, 0.25, 0.30 ] ,
"max_depth" : [ 3, 4, 5, 6, 8, 10, 12, 15],
"min_child_weight" : [ 1, 3, 5, 7 ],
"gamma" : [ 0.0, 0.1, 0.2 , 0.3, 0.4 ],
"colsample_bytree" : [ 0.3, 0.4, 0.5 , 0.7 ] }<train_on_grid>
def update_state(df):
def get_select_params(r):
values_uc = f'({r.user_id}, {r.content_id}, {r.answered_correctly}, {1-r.answered_correctly})'
values_u = f'({r.user_id}, {r.answered_correctly}, {1-r.answered_correctly})'
return values_uc, values_u
values = df.apply(get_select_params, axis=1, result_type='expand')
 return f
<train_model><EOS> 
%%time
df_batch_prior = None
counter = 0
for test_batch in iter_test:
counter += 1
if df_batch_prior is not None:
answers = eval(test_batch[0]['prior_group_answers_correct'].iloc[0])
 df_batch_prior['answered_correctly'] = answers
cursor.executescript(update_state(df_batch_prior[df_batch_prior.content_type_id == 0]))
if not counter % 100:
conn.commit()
df_batch_prior = test_batch[0][batch_cols_prior].astype({k: dtypes[k] for k in batch_cols_prior})
 df_batch = test_batch[0][test_batch[0].content_type_id == 0]
records = df_batch[batch_cols].fillna(0 ).to_records(index=False)
 df_batch = pd.read_sql(select_state(batch_cols, records), conn)
 predictions = model.predict(xgb.DMatrix(df_batch[test_cols]))
df_batch['answered_correctly'] = predictions
env.predict(df_batch[['row_id', 'answered_correctly']] )