Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<count_values>
warnings.simplefilter('ignore')
pd.qcut(train['Cabin_num'],3 ).value_counts()<groupby>
SEED = 123
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED )
train['Survived'].groupby(pd.qcut(train['Cabin_num'], 3)).mean()<feature_engineering>
model = pd.read_pickle('.. /input/the-best-of-model-on-lightgbm/The_best_of_model_on_LightGBM.pickle')
 print('Load model:', model )
def names(train, test):
for i in [train, test]:
i['Name_Len'] = i['Name'].apply(lambda x: len(x))
i['Name_Title'] = i['Name'].apply(lambda x: x.split(',')[1] ).apply(lambda x: x.split() [0])
 del i['Name']
return train, test<feature_engineering>
params = pd.read_pickle('.. /input/the-best-of-all-parameters-on-lightgbm/The_best_of_all_parameters_on_LightGBM.pickle')
 print('Parameters:', params )
def age_impute(train, test):
for i in [train, test]:
i['Age_Null_Flag'] = i['Age'].apply(lambda x: 1 if pd.isnull(x)else 0)
 data = train.groupby(['Name_Title', 'Pclass'])['Age']
i['Age'] = data.transform(lambda x: x.fillna(x.mean()))
return train, test<drop_column>
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_file = '.. /input/riiid-test-answer-prediction/questions.csv'
def fam_size(train, test):
for i in [train, test]:
i['Fam_Size'] = np.where(( i['SibSp']+i['Parch'])== 0 , 'Alone',
np.where(( i['SibSp']+i['Parch'])<= 3,'Small', 'Big'))
del i['SibSp']
del i['Parch']
return train, test<drop_column>
def add_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(tqdm(df[['user_id','answered_correctly']].values)) :
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def add_user_feats_without_update(df, answered_correctly_sum_u_dict, count_u_dict):
acsu = np.zeros(len(df), dtype=np.int32)
 cu = np.zeros(len(df), dtype=np.int32)
 for cnt,row in enumerate(df[['user_id']].values):
acsu[cnt] = answered_correctly_sum_u_dict[row[0]]
cu[cnt] = count_u_dict[row[0]]
user_feats_df = pd.DataFrame({'answered_correctly_sum_u':acsu, 'count_u':cu})
 user_feats_df['answered_correctly_avg_u'] = user_feats_df['answered_correctly_sum_u'] / user_feats_df['count_u']
df = pd.concat([df, user_feats_df], axis=1)
 return df
def update_user_feats(df, answered_correctly_sum_u_dict, count_u_dict):
for row in df[['user_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
answered_correctly_sum_u_dict[row[0]] += row[1]
count_u_dict[row[0]] += 1
def cabin(train, test):
for i in [train, test]:
i['Cabin_Lett'] = i['Cabin'].apply(lambda x: str(x)[0])
 del i['Cabin']
return train, test<categorify>
feld_needed = ['row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly', 'prior_question_elapsed_time', 'prior_question_had_explanation']
train = pd.read_pickle(train_pickle)[feld_needed]
valid = pd.read_pickle(valid_pickle)[feld_needed]
train = train.loc[train.content_type_id == False].reset_index(drop=True)
 valid = valid.loc[valid.content_type_id == False].reset_index(drop=True)
 content_df = train[['content_id','answered_correctly']].groupby(['content_id'] ).agg(['mean'] ).reset_index()
content_df.columns = ['content_id', 'answered_correctly_avg_c']
train = pd.merge(train, content_df, on=['content_id'], how="left")
 valid = pd.merge(valid, content_df, on=['content_id'], how="left")
 answered_correctly_sum_u_dict = defaultdict(int)
 count_u_dict = defaultdict(int)
 train = add_user_feats(train, answered_correctly_sum_u_dict, count_u_dict)
 valid = add_user_feats(valid, answered_correctly_sum_u_dict, count_u_dict)
 prior_question_elapsed_time_mean = train.prior_question_elapsed_time.dropna().values.mean()
train['prior_question_elapsed_time_mean'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 valid['prior_question_elapsed_time_mean'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 questions_df = pd.read_csv(question_file)
 train = pd.merge(train, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 valid = pd.merge(valid, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8' )
def cabin_num(train, test):
for i in [train, test]:
i['Cabin_num1'] = i['Cabin'].apply(lambda x: str(x ).split(' ')[-1][1:])
 i['Cabin_num1'].replace('an', np.NaN, inplace = True)
 i['Cabin_num1'] = i['Cabin_num1'].apply(lambda x: int(x)if not pd.isnull(x)and x != '' else np.NaN)
 i['Cabin_num'] = pd.qcut(train['Cabin_num1'],3)
 train = pd.concat(( train, pd.get_dummies(train['Cabin_num'], prefix = 'Cabin_num')) , axis = 1)
 test = pd.concat(( test, pd.get_dummies(test['Cabin_num'], prefix = 'Cabin_num')) , axis = 1)
 del train['Cabin_num']
del test['Cabin_num']
del train['Cabin_num1']
del test['Cabin_num1']
return train, test<count_values>
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_sum_u', 'count_u', 'answered_correctly_avg_c', 'part', 'prior_question_had_explanation', 'prior_question_elapsed_time']
dro_cols = list(set(train.columns)- set(FEATS))
y_tr = train[TARGET]
y_va = valid[TARGET]
train.drop(dro_cols, axis=1, inplace=True)
 valid.drop(dro_cols, axis=1, inplace=True)
 _=gc.collect()
train.Embarked.value_counts()<data_type_conversions>
class Iter_Valid(object):
def __init__(self, df, max_user=1000):
df = df.reset_index(drop=True)
 self.df = df
self.user_answer = df['user_answer'].astype(str ).values
self.answered_correctly = df['answered_correctly'].astype(str ).values
df['prior_group_responses'] = "[]"
df['prior_group_answers_correct'] = "[]"
self.sample_df = df[df['content_type_id'] == 0][['row_id']]
self.sample_df['answered_correctly'] = 0
self.len = len(df)
 self.user_id = df.user_id.values
self.task_container_id = df.task_container_id.values
self.content_type_id = df.content_type_id.values
self.max_user = max_user
self.current = 0
self.pre_user_answer_list = []
self.pre_answered_correctly_list = []
def __iter__(self):
return self
def fix_df(self, user_answer_list, answered_correctly_list, pre_start):
df= self.df[pre_start:self.current].copy()
sample_df = self.sample_df[pre_start:self.current].copy()
df.loc[pre_start,'prior_group_responses'] = '[' + ",".join(self.pre_user_answer_list)+ ']'
df.loc[pre_start,'prior_group_answers_correct'] = '[' + ",".join(self.pre_answered_correctly_list)+ ']'
self.pre_user_answer_list = user_answer_list
self.pre_answered_correctly_list = answered_correctly_list
return df, sample_df
def __next__(self):
added_user = set()
pre_start = self.current
pre_added_user = -1
pre_task_container_id = -1
pre_content_type_id = -1
user_answer_list = []
answered_correctly_list = []
while self.current < self.len:
crr_user_id = self.user_id[self.current]
crr_task_container_id = self.task_container_id[self.current]
crr_content_type_id = self.content_type_id[self.current]
if crr_user_id in added_user and(crr_user_id != pre_added_user or(crr_task_container_id != pre_task_container_id and crr_content_type_id == 0 and pre_content_type_id == 0)) :
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 if len(added_user)== self.max_user:
if crr_user_id == pre_added_user and(crr_task_container_id == pre_task_container_id or crr_content_type_id == 1):
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
continue
else:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 added_user.add(crr_user_id)
 pre_added_user = crr_user_id
pre_task_container_id = crr_task_container_id
pre_content_type_id = crr_content_type_id
user_answer_list.append(self.user_answer[self.current])
 answered_correctly_list.append(self.answered_correctly[self.current])
 self.current += 1
if pre_start < self.current:
return self.fix_df(user_answer_list, answered_correctly_list, pre_start)
 else:
raise StopIteration()
train['Embarked'].fillna('S', inplace=True )<filter>
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
test[test.Fare.isnull() ]<filter>
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_user_feats(previous_test_df, answered_correctly_sum_u_dict, count_u_dict)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df = add_user_feats_without_update(test_df, answered_correctly_sum_u_dict, count_u_dict)
 test_df = pd.merge(test_df, content_df, on='content_id', how="left")
 test_df = pd.merge(test_df, questions_df, left_on='content_id', right_on='question_id', how='left')
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time_mean'] = test_df.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 test_df[TARGET] = model.predict(test_df[FEATS], num_iteration=model.best_iteration)
 set_predict(test_df[['row_id', TARGET]] )
train[(train.Pclass==3)&(train.Age>50)].Fare.median()<correct_missing_values>
simulation = pd.read_csv('./submission.csv')
 simulation = simulation['answered_correctly']
simulation.head()
test['Fare'].fillna(7.75, inplace=True )<categorify>
def bootstrap(x, repeats):
mean_x = np.average(x)
 sem_x = np.std(x)/np.sqrt(len(x))
median_x = np.median(x)
 vec = np.zeros(( 2, repeats))
for i in np.arange(repeats):
re = np.random.choice(len(x),len(x),replace=True)
 re_x = x[re]
vec[0,i] = np.mean(re_x)
 vec[1,i] = np.median(re_x)
 sns.distplot(vec[1,:], kde=True, rug=True, hist=True, norm_hist=True, axlabel='Frequency distribution')
 plt.show()
b_mean_sem = np.std(vec[0,:])
 b_median_sem = np.std(vec[1,:])
 return b_mean_sem, b_median_sem
def dummies(train, test, columns = ['Pclass', 'Sex', 'Embarked', 'Ticket_Lett', 'Cabin_Lett', 'Name_Title', 'Fam_Size']):
for column in columns:
train[column] = train[column].apply(lambda x: str(x))
test[column] = test[column].apply(lambda x: str(x))
good_cols = [column+'_'+i for i in train[column].unique() if i in test[column].unique() ]
train = pd.concat(( train, pd.get_dummies(train[column], prefix = column)[good_cols]), axis = 1)
 test = pd.concat(( test, pd.get_dummies(test[column], prefix = column)[good_cols]), axis = 1)
 del train[column]
del test[column]
return train, test<drop_column>
def Statistical_significance_test(A, B):
A = np.array(A)
 B = np.array(B)
 print('Paired t-test:', stats.ttest_rel(A, B))
A_var = np.var(A, ddof=1)
 B_var = np.var(B, ddof=1)
 A_df = len(A)- 1
B_df = len(B)- 1
f = A_var / B_var
one_sided_pval1 = stats.f.cdf(f, A_df, B_df)
 one_sided_pval2 = stats.f.sf(f, A_df, B_df)
 two_sided_pval = min(one_sided_pval1, one_sided_pval2)* 2
print("Student's t-test:")
 print('F: ', round(f, 3))
print('p-value: ', round(two_sided_pval, 3))
two_sided_pval = min(one_sided_pval1, one_sided_pval2)* 2
print("Welch's t-test:")
 print('F: ', round(f, 3))
print('p-value: ', round(two_sided_pval, 4))
print("Mann-Whitney's U-test:", stats.mannwhitneyu(A, B, alternative='two-sided'))
print("Wilcoxon signed rank sum test:", stats.wilcoxon(A, B))
return None
def drop(train, test, bye = ['PassengerId']):
for i in [train, test]:
for z in bye:
del i[z]
return train, test<load_from_csv>
b_mean_sem, b_median_sem = bootstrap(np.array(simulation), 1000000)
 Statistical_significance_test(A = simulation, B = np.repeat(b_mean_sem, len(simulation)) )
<choose_model_class><EOS> 
print('Operation completed.' )