Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<compute_train_metric>
import gc
import joblib
import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score
from collections import defaultdict
from tqdm.notebook import tqdm
import lightgbm as lgb
from numba import njit
from scipy.optimize import minimize, fsolve
clf = DecisionTreeClassifier(random_state=0)
 scoring = {'acc': 'accuracy',
'prec_macro': 'precision_macro',
'rec_macro': 'recall_macro',
'f1_macro': 'f1_macro'}
scores = cross_validate(clf, train_feature, train_class, cv=10, scoring=scoring)
 print('Accuracy score : %.3f' % scores['test_acc'].mean())
 print('Precisoin score : %.3f' % scores['test_prec_macro'].mean())
 print('Recall score : %.3f' % scores['test_rec_macro'].mean())
 print('F1 score : %.3f' % scores['test_f1_macro'].mean() )<train_on_grid>
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'
debug = False
validaten_flg = False
para_grid = {
'min_samples_split' : range(10,500,20),
'max_depth': range(1,20,2),
'criterion':("gini", "entropy")
 }
clf_tree = DecisionTreeClassifier()
clf_cv = GridSearchCV(clf_tree,
para_grid,
scoring='accuracy',
cv=5,
n_jobs=-1)
 clf_cv.fit(train_feature,train_class)
 best_parameters = clf_cv.best_params_
print(best_parameters )<compute_train_metric>
def add_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly']].values)) :
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['user_id']].values):
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
clf = clf_cv.best_estimator_
scoring = {'acc': 'accuracy',
'prec_macro': 'precision_macro',
'rec_macro': 'recall_macro',
'f1_macro': 'f1_macro'}
scores = cross_validate(clf, train_feature, train_class, cv=10, scoring=scoring)
 print('Accuracy score : %.3f' % scores['test_acc'].mean())
 print('Precisoin score : %.3f' % scores['test_prec_macro'].mean())
 print('Recall score : %.3f' % scores['test_rec_macro'].mean())
 print('F1 score score : %.3f' % scores['test_f1_macro'].mean() )<compute_train_metric>
feld_needed = ['row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']
train = pd.read_pickle(train_pickle)[feld_needed]
valid = pd.read_pickle(valid_pickle)[feld_needed]
if debug:
train = train[:1000000]
valid = valid[:10000]
train = train.loc[train.content_type_id == False].reset_index(drop=True)
 valid = valid.loc[valid.content_type_id == False].reset_index(drop=True)
 content_df = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'] ).reset_index()
content_df.columns = ['content_id', 'answered_correctly_avg_c']
train = pd.merge(train, content_df, on=['content_id'], how="left")
 valid = pd.merge(valid, content_df, on=['content_id'], how="left")
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 train = add_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)
 valid = add_user_feats(valid, answered_correctly_sum_u_dict, count_u_dict)
 prior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()
train['prior_question_elapsed_time_mean'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 valid['prior_question_elapsed_time_mean'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 questions_df = pd.read_csv(question_file)
 train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8' )
clf2 = clf_cv.best_estimator_
clf2.fit(X_train,y_train)
 predictions = clf2.predict(X_test)
 print(metrics.classification_report(y_test,predictions, target_names=targ_names, digits=3))<compute_train_metric>
joblib.dump(answered_correctly_sum_u_dict, "answered_correctly_sum_u_dict.pkl.zip")
 joblib.dump(count_u_dict, "count_u_dict.pkl.zip")
 joblib.dump(prior_question_elapsed_time_mean, "prior_question_elapsed_time_mean.pkl.zip" )
fig, ax = plt.subplots(figsize=(3,3))
cm = ConfusionMatrix(clf2, classes=[0, 1], cmap='RdPu')
 cm.score(X_test, y_test)
 for label in cm.ax.texts:
label.set_size(14)
 cm.poof()<train_model>
content_df.to_feather("content_df.feather")
 questions_df.to_feather("questions_df.feather" )
modelviz = clf_cv.best_estimator_
visualizer = ROCAUC(modelviz, classes=["Dead", "Survived"])
 visualizer.fit(X_train, y_train)
 visualizer.score(X_test, y_test)
 visualizer.show()<save_to_csv>
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']
dro_cols = list(set(train.columns)- set(FEATS))
y_tr = train[TARGET]
y_va = valid[TARGET]
train.drop(dro_cols, axis=1, inplace=True)
 valid.drop(dro_cols, axis=1, inplace=True)
 _ = gc.collect()
data = export_graphviz(clf,out_file=None,feature_names=feat_names,class_names=targ_names,
filled=True, rounded=True,
special_characters=True)
 graph = graphviz.Source(data)
 graph<load_from_csv>
lgb_train = lgb.Dataset(train[FEATS], y_tr)
 lgb_valid = lgb.Dataset(valid[FEATS], y_va)
 del train, y_tr
_ = gc.collect()
test = pd.read_csv('.. /input/titanic/test.csv')
 clf.fit(train_feature, train_class)
 meanFare = dataset['Fare'].mean()
test = test.fillna({"Fare": meanFare})
 test = pd.get_dummies(test, columns=['Sex'])
 test = pd.get_dummies(test, columns=['Embarked'])
 ids = test['PassengerId']
test_feature = test[feat_names]
predictions = clf.predict(test_feature)
 output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 output.head()<save_to_csv>
model = lgb.train(
 {'objective': 'binary'},
lgb_train,
valid_sets=[lgb_train, lgb_valid],
verbose_eval=100,
num_boost_round=10000,
early_stopping_rounds=10
)
output.to_csv('submission.csv', index=False )<load_from_csv>
model.save_model(f"fold0_lgb_model.txt" )
train= pd.read_csv('.. /input/train.csv')
 test= pd.read_csv('.. /input/test.csv')
 print("shape of train dataset:", train.shape)
 print("shape of test dataset:", test.shape )<sort_values>
joblib.dump(model.best_iteration, "fold0_lgb_model_best_iteration.pkl.zip" )
correlation= train.corr()
correlation=correlation['SalePrice'].sort_values(ascending=False)
 pos_correlation=correlation.head(25)
 pos_correlation<drop_column>
preds1 = model.predict(valid[FEATS], num_iteration=400)
 print('auc at 400:', roc_auc_score(y_va, preds1))
print("Before:",train.shape)
 train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index, inplace=True)
 train.drop(train[(train['GarageArea']>1200)&(train['SalePrice']<100000)].index, inplace=True)
 train.drop(train[(train['TotalBsmtSF']>6000)&(train['SalePrice']<200000)].index, inplace=True)
 train.reset_index(drop=True, inplace=True)
 print("After:",train.shape )<concatenate>
preds2 = model.predict(valid[FEATS], num_iteration=700)
 print('auc at 700:', roc_auc_score(y_va, preds2))
data= pd.concat([train.drop(['SalePrice'], axis=1), test] )<data_type_conversions>
preds3 = model.predict(valid[FEATS], num_iteration=model.best_iteration)
 print(f'auc at {model.best_iteration}', roc_auc_score(y_va, preds3))
train['GarageQual'].fillna('None', inplace=True)
 test['GarageQual'].fillna('None', inplace=True)
 train['GarageFinish'].fillna('None', inplace=True)
 test['GarageFinish'].fillna('None', inplace=True)
 train['GarageYrBlt'].fillna('None', inplace=True)
 test['GarageYrBlt'].fillna('None', inplace=True)
 train['GarageType'].fillna('None', inplace=True)
 test['GarageType'].fillna('None', inplace=True)
 train['GarageCond'].fillna('None', inplace=True)
 test['GarageCond'].fillna('None', inplace=True)
 test.loc[test['Id']==2577, 'GarageType']='None'
test['GarageCars'].fillna(0, inplace=True)
 test['GarageArea'].fillna(0, inplace=True )<filter>
oof = np.zeros(( 3, y_va.shape[0]))
oof[0] = preds1
oof[1] = preds2
oof[2] = preds3
data[(data['TotalBsmtSF']!=0)&(data['BsmtCond'].isnull() ==True)][['Id','TotalBsmtSF','BsmtCond','BsmtQual','BsmtExposure',
'BsmtFinType1','BsmtFinSF1','BsmtUnfSF']]<categorify>
@njit
def grad_func_jit(weights):
oof_clip = np.minimum(1 - 1e-15, np.maximum(oof, 1e-15))
gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
test.loc[test['Id']==2041, 'BsmtCond']='Gd'
test.loc[test['Id']==2186, 'BsmtCond']='TA'
test.loc[test['Id']==2525, 'BsmtCond']='TA'
train['BsmtCond'].fillna('None', inplace=True)
 test['BsmtCond'].fillna('None', inplace=True )<filter>
def func_numpy_metric(weights):
oof_blend = np.tensordot(weights, oof, axes =(( 0),(0)))
 return -roc_auc_score(y_va, oof_blend )
data[(data['TotalBsmtSF']!=0)&(data['BsmtExposure'].isnull() ==True)][['Id','TotalBsmtSF','BsmtCond','BsmtQual','BsmtExposure',
'BsmtFinType1','BsmtFinSF1','BsmtUnfSF']]<categorify>
import datetime
from time import time
train.loc[train['Id']==949, 'BsmtExposure']='Gd'
test.loc[test['Id']==1488, 'BsmtExposure']='Gd'
test.loc[test['Id']==2349, 'BsmtExposure']='Gd'
train['BsmtExposure'].fillna('None', inplace=True)
 test['BsmtExposure'].fillna('None', inplace=True )<filter>
tol = 1e-10
init_guess = [1 / oof.shape[0]] * oof.shape[0]
bnds = [(0, 1)for _ in range(oof.shape[0])]
cons = {'type': 'eq',
'fun': lambda x: np.sum(x)- 1,
'jac': lambda x: [1] * len(x)}
print('Inital Ensemble OOF:', func_numpy_metric(init_guess))
start_time = time()
res_scipy = minimize(fun = func_numpy_metric,
x0 = init_guess,
method = 'SLSQP',
jac = grad_func_jit,
bounds = bnds,
constraints = cons,
tol = tol)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}] Optimised Ensemble OOF:', res_scipy.fun)
 print('Optimised Weights:', res_scipy.x )
data[(data['TotalBsmtSF']!=0)&(data['BsmtQual'].isnull() ==True)][['Id','TotalBsmtSF','BsmtQual','BsmtCond','BsmtExposure',
'BsmtFinType1','BsmtFinSF1','BsmtUnfSF']]<categorify>
preds = oof[0] * res_scipy.x[0] + oof[1] * res_scipy.x[1] + oof[2] * res_scipy.x[2]
print(f'auc at {model.best_iteration}', roc_auc_score(y_va, preds))
test.loc[test['Id']==2218, 'BsmtQual']='Fa'
test.loc[test['Id']==2219, 'BsmtQual']='TA'
train['BsmtQual'].fillna('None', inplace=True)
 test['BsmtQual'].fillna('None', inplace=True )<filter>
optimized_weights = res_scipy.x
joblib.dump(optimized_weights, "optimized_weights.pkl.zip" )
data[(data['TotalBsmtSF']!=0)&(data['BsmtFinType1'].isnull() ==True)][['Id','TotalBsmtSF','BsmtQual','BsmtCond','BsmtExposure',
'BsmtFinType1','BsmtFinSF1','BsmtUnfSF']]<filter>
class Iter_Valid(object):
def __init__(self, df, max_user=1000):
df = df.reset_index(drop=True)
 self.df = df
self.user_answer = df['user_answer'].astype(str ).values
self.answered_correctly = df['answered_correctly'].astype(str ).values
df['prior_group_responses'] = "[]"
df['prior_group_answers_correct'] = "[]"
self.sample_df = df[df['content_type_id'] == 0][['row_id']]
self.sample_df['answered_correctly'] = 0
self.len = len(df)
 self.user_id = df.user_id.values
self.task_container_id = df.task_container_id.values
self.content_type_id = df.content_type_id.values
self.max_user = max_user
self.current = 0
self.pre_user_answer_list = []
self.pre_answered_correctly_list = []
def __iter__(self):
return self
def fix_df(self, user_answer_list, answered_correctly_list, pre_start):
df= self.df[pre_start:self.current].copy()
sample_df = self.sample_df[pre_start:self.current].copy()
df.loc[pre_start,'prior_group_responses'] = '[' + ",".join(self.pre_user_answer_list)+ ']'
df.loc[pre_start,'prior_group_answers_correct'] = '[' + ",".join(self.pre_answered_correctly_list)+ ']'
self.pre_user_answer_list = user_answer_list
self.pre_answered_correctly_list = answered_correctly_list
return df, sample_df
def __next__(self):
added_user = set()
pre_start = self.current
pre_added_user = -1
pre_task_container_id = -1
pre_content_type_id = -1
user_answer_list = []
answered_correctly_list = []
while self.current < self.len:
crr_user_id = self.user_id[self.current]
crr_task_container_id = self.task_container_id[self.current]
crr_content_type_id = self.content_type_id[self.current]
if crr_user_id in added_user and(crr_user_id != pre_added_user or(crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)) :
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 if len(added_user)== self.max_user:
if crr_user_id == pre_added_user and(crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
else:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 added_user.add(crr_user_id)
 pre_added_user = crr_user_id
pre_task_container_id = crr_task_container_id
pre_content_type_id = crr_content_type_id
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
if pre_start < self.current:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 else:
raise StopIteration()
data[(data['TotalBsmtSF']!=0)&(data['BsmtFullBath'].isnull() ==True)][['Id','TotalBsmtSF','BsmtQual','BsmtCond',
'BsmtExposure','BsmtFullBath']]<data_type_conversions>
if validaten_flg:
target_df = pd.read_pickle(valid_pickle)
 if debug:
target_df = target_df[:10000]
iter_test = Iter_Valid(target_df,max_user=1000)
 predicted = []
def set_predict(df):
predicted.append(df)
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 train = pd.read_pickle(train_pickle)[['user_id','answered_correctly','content_type_id']]
if debug:
train = train[:1000000]
train = train[train.content_type_id == False].reset_index(drop=True)
 update_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)
 del train
else:
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
train['BsmtFinType1'].fillna('None', inplace=True)
 test['BsmtFinType1'].fillna('None', inplace=True)
 test['BsmtFinSF1'].fillna(0, inplace=True)
 train['BsmtFinType2'].fillna('None', inplace=True)
 test['BsmtFinType2'].fillna('None', inplace=True)
 test['BsmtUnfSF'].fillna(0, inplace=True)
 test['TotalBsmtSF'].fillna(0, inplace=True)
 test['BsmtFullBath'].fillna(0, inplace=True)
 test['BsmtHalfBath'].fillna(0, inplace=True)
 test['BsmtFinSF1'].fillna(0, inplace=True)
 test['BsmtFinSF2'].fillna(0, inplace=True )<data_type_conversions>
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 preds = optimized_weights[0] * model.predict(test_df[FEATS], num_iteration=400)
 preds += optimized_weights[1] * model.predict(test_df[FEATS], num_iteration=700)
 preds += optimized_weights[2] * model.predict(test_df[FEATS], num_iteration=model.best_iteration)
 test_df[TARGET] = preds
set_predict(test_df[['row_id', TARGET]] )
train['FireplaceQu'].fillna('None', inplace=True)
 test['FireplaceQu'].fillna('None', inplace=True)
 train.loc[train['LotFrontage'].isnull() ==True, 'LotFrontage']= train['LotFrontage'].mean()
test.loc[test['LotFrontage'].isnull() ==True, 'LotFrontage']= test['LotFrontage'].mean()
train['MasVnrType'].fillna('None', inplace=True)
 test['MasVnrType'].fillna('None', inplace=True)
 train['MasVnrArea'].fillna(0, inplace=True)
 test['MasVnrArea'].fillna(0, inplace=True)
 train['PoolQC'].fillna('None', inplace=True)
 test['PoolQC'].fillna('None', inplace=True)
 train['MiscFeature'].fillna('None', inplace=True)
 test['MiscFeature'].fillna('None', inplace=True)
 train['Alley'].fillna('None', inplace=True)
 test['Alley'].fillna('None', inplace=True)
 train['Fence'].fillna('None', inplace=True)
 test['Fence'].fillna('None', inplace=True)
 train['MSZoning'].fillna('RL', inplace=True)
 test['MSZoning'].fillna('RL', inplace=True)
 train['Electrical'].fillna('SBrkr', inplace=True)
 test['Functional'].fillna('Typ', inplace=True)
 test['SaleType'].fillna('WD', inplace=True)
 test['Exterior1st'].fillna('VinylSd', inplace=True)
 test['KitchenQual'].fillna('TA', inplace=True)
 test['Exterior2nd'].fillna('VinylSd', inplace=True )<drop_column>
if validaten_flg:
y_true = target_df[target_df.content_type_id == 0].answered_correctly
y_pred = pd.concat(predicted ).answered_correctly
print(roc_auc_score(y_true, y_pred))