Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<categorify>
import gc
import joblib
import pandas as pd
import numpy as np
import lightgbm as lgb
train = pd.get_dummies(train, columns = ["Sex", "Embarked", "NameTitle", "AgeGroup", "FareGroup"])
 test = pd.get_dummies(test, columns = ["Sex", "Embarked", "NameTitle", "AgeGroup", "FareGroup"] )<data_type_conversions>
def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['user_id']].values):
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
upper = corr.where(np.triu(np.ones(corr.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] > 0.90)]
print(to_drop )<prepare_x_and_y>
answered_correctly_sum_u_dict = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/answered_correctly_sum_u_dict.pkl.zip")
 count_u_dict = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/count_u_dict.pkl.zip")
 questions_df = pd.read_feather('.. /input/lgbm-with-loop-feature-engineering-dataset/questions_df.feather')
 content_df = pd.read_feather('.. /input/lgbm-with-loop-feature-engineering-dataset/content_df.feather')
 prior_question_elapsed_time_mean = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/prior_question_elapsed_time_mean.pkl.zip" )
X = train.drop(["PassengerId", "Survived"], axis = 1)
 Y = train["Survived"]<import_modules>
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u',
'answered_correctly_avg_c', 'part', 'prior_question_had_explanation',
'prior_question_elapsed_time'
]
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, BaggingClassifier, ExtraTreesClassifier, StackingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from catboost import CatBoostClassifier
from xgboost import XGBClassifier
import lightgbm as lgb<set_options>
model = lgb.Booster(model_file=".. /input/lgbm-with-loop-feature-engineering-dataset/fold0_lgb_model.txt")
 model.best_iteration = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/fold0_lgb_model_best_iteration.pkl.zip" )
style.use("seaborn-whitegrid")
 %matplotlib inline
%config InlineBackend.figure_format = "retina"
def model_builder(X, Y, pipeline, params, kfolds, classifier_name, data_split = False):

 if data_split == True:
X_train, X_test, Y_train, Y_test = train_test_split(X,
Y,
random_state = 0,
test_size = 0.25)
 cv = StratifiedKFold(n_splits = kfolds)
 start = time.time()
grid = GridSearchCV(pipeline, params, cv = cv, verbose = 0, n_jobs = -1)
 grid = grid.fit(X_train, Y_train)
 print("Best Params:")
 print("")
 print(grid.best_params_)
 grid = grid.best_estimator_
classifier_score = cross_val_score(grid, X_test, Y_test, cv = cv, n_jobs = -1)
 Y_predicted = grid.predict(X_test)
 stop = time.time()
print("")
 print("
print("Cross-Validated-Score: " + str(round(classifier_score.mean() , 6)))
 print("
print("")
 print("Training time: " + str(round(stop - start, 2)) + "s")
 train_size, train_score, test_score = learning_curve(grid, X_train, Y_train, cv = cv, n_jobs = -1)
 print("")
 print("Learning Curve:")
 plt.figure(figsize =(12,6))
plt.plot(train_size, np.mean(train_score, axis = 1), label = "Train scores")
 plt.plot(train_size, np.mean(test_score, axis = 1), label = "Test scores")
 plt.title(classifier_name)
 plt.legend()
plt.show()
return X_train, X_test, Y_train, Y_test, classifier_score, grid
if data_split == False:
cv = StratifiedKFold(n_splits = kfolds)
 start = time.time()
grid = GridSearchCV(pipeline, params, cv = cv, verbose = 0, n_jobs = -1)
 grid = grid.fit(X, Y)
 print("Best Params:")
 print("")
 print(grid.best_params_)
 grid = grid.best_estimator_
classifier_score = cross_val_score(grid, X, Y, cv = cv, n_jobs = -1)
 Y_predicted = grid.predict(X)
 stop = time.time()
print("")
 print("
print("Cross-Validated-Score: " + str(round(classifier_score.mean() , 6)))
 print("
print("")
 print("Training time: " + str(round(stop - start, 2)) + "s")
 train_size, train_score, test_score = learning_curve(grid, X, Y, cv = cv, n_jobs = -1)
 print("")
 print("Learning Curve:")
 plt.figure(figsize =(12,6))
plt.plot(train_size, np.mean(train_score, axis = 1), label = "Train scores")
 plt.plot(train_size, np.mean(test_score, axis = 1), label = "Test scores")
 plt.title(classifier_name)
 plt.legend()
plt.show()
return classifier_score, grid<find_best_model_class>
optimized_weights = joblib.load(".. /input/lgbm-with-loop-feature-engineering-dataset/optimized_weights.pkl.zip" )
pipe_xgb = Pipeline([ 
("scaler", StandardScaler()), 
("pca", PCA()), 
("xgb", XGBClassifier())])
 params_xgb = {
"pca__n_components" : [2, 6, 10, 18],
"xgb__n_estimators" : [300, 500, 700],
"xgb__learning_rate" : [0.005, 0.1],
"xgb__max_depth" : [5, 7],
"xgb__max_features" : [3, 5],
"xgb__gamma" : [0.5, 0.6, 0.7]}
xgb_score, grid_xgb = model_builder(X, Y, pipe_xgb, params_xgb, 10, "XGBoost")
 Y_predicted_xgb = grid_xgb.predict_proba(X)[:, 1]<find_best_params>
class Iter_Valid(object):
def __init__(self, df, max_user=1000):
df = df.reset_index(drop=True)
 self.df = df
self.user_answer = df['user_answer'].astype(str ).values
self.answered_correctly = df['answered_correctly'].astype(str ).values
df['prior_group_responses'] = "[]"
df['prior_group_answers_correct'] = "[]"
self.sample_df = df[df['content_type_id'] == 0][['row_id']]
self.sample_df['answered_correctly'] = 0
self.len = len(df)
 self.user_id = df.user_id.values
self.task_container_id = df.task_container_id.values
self.content_type_id = df.content_type_id.values
self.max_user = max_user
self.current = 0
self.pre_user_answer_list = []
self.pre_answered_correctly_list = []
def __iter__(self):
return self
def fix_df(self, user_answer_list, answered_correctly_list, pre_start):
df= self.df[pre_start:self.current].copy()
sample_df = self.sample_df[pre_start:self.current].copy()
df.loc[pre_start,'prior_group_responses'] = '[' + ",".join(self.pre_user_answer_list)+ ']'
df.loc[pre_start,'prior_group_answers_correct'] = '[' + ",".join(self.pre_answered_correctly_list)+ ']'
self.pre_user_answer_list = user_answer_list
self.pre_answered_correctly_list = answered_correctly_list
return df, sample_df
def __next__(self):
added_user = set()
pre_start = self.current
pre_added_user = -1
pre_task_container_id = -1
pre_content_type_id = -1
user_answer_list = []
answered_correctly_list = []
while self.current < self.len:
crr_user_id = self.user_id[self.current]
crr_task_container_id = self.task_container_id[self.current]
crr_content_type_id = self.content_type_id[self.current]
if crr_user_id in added_user and(crr_user_id != pre_added_user or(crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)) :
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 if len(added_user)== self.max_user:
if crr_user_id == pre_added_user and(crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
else:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 added_user.add(crr_user_id)
 pre_added_user = crr_user_id
pre_task_container_id = crr_task_container_id
pre_content_type_id = crr_content_type_id
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
if pre_start < self.current:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 else:
raise StopIteration()
pipe_boost = Pipeline([ 
("scaler", StandardScaler()), 
("pca", PCA()), 
("boost", GradientBoostingClassifier())])
 params_boost = {
"pca__n_components" : [2, 6, 10, 18],
"boost__n_estimators" : [300, 500, 700],
"boost__learning_rate" : [0.005, 0.1],
"boost__max_depth" : [3, 5],
"boost__max_features" : [3, 5]}
boost_score, grid_boost = model_builder(X, Y, pipe_boost, params_boost, 10, "Gradient Boosting")
 Y_predicted_boost = grid_boost.predict_proba(X)[:, 1]<find_best_model_class>
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
<find_best_params><EOS> 
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 preds = optimized_weights[0] * model.predict(test_df[FEATS], num_iteration=400)
 preds += optimized_weights[1] * model.predict(test_df[FEATS], num_iteration=700)
 preds += optimized_weights[2] * model.predict(test_df[FEATS], num_iteration=model.best_iteration)
 test_df[TARGET] = preds
set_predict(test_df[['row_id', TARGET]] )