Kaggle competition: Riiid Answer Correctness Prediction
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 train_df.head()<sort_values>
df_train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv', usecols = ['row_id', 'user_id', 'content_id', 'content_type_id', 'answered_correctly'],
dtype={'row_id': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8', 'answered_correctly': 'int8'
}, index_col = [0])
train_df['AgeBand'] = pd.cut(train_df['Age'], 5)
 train_df[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )<feature_engineering>
lt = 150
df_train = df_train[df_train['content_type_id']==0]
df_train = df_train[['content_id', 'user_id', 'answered_correctly']]
df_train['user_count'] = df_train.groupby('user_id' ).cumcount()
df_question_meanstd = df_train.groupby('content_id')['answered_correctly'].agg(question_mean = np.mean)
 df_question_meanstd['question_std'] =(df_question_meanstd['question_mean']*(1-df_question_meanstd['question_mean'])) **0.5
df_train = df_train.groupby('user_id' ).tail(lt)
 df_train = pd.merge(df_train, df_question_meanstd, left_on = 'content_id', right_index = True, how = 'left')
 df_user_mean = df_train.groupby('user_id' ).agg({'answered_correctly':(pd.Series.count, np.mean), 'question_mean': np.mean})
 df_user_mean.columns = ['user_count', 'ur_mean', 'uqr_mean']
df_user_mean['uqr_std'] =(df_user_mean['uqr_mean']*(1-df_user_mean['uqr_mean'])) **0.5
df_user_mean['uqr_std'] = df_user_mean['uqr_std'].fillna(0)
 df_user_mean['ur_std_diff'] =(df_user_mean['ur_mean'] - df_user_mean['uqr_mean'])/df_user_mean['uqr_std']
df_train = pd.DataFrame()
gc.collect()
for dataset in combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
train_df.head()<drop_column>
env = riiideducation.make_env()
<sort_values><EOS> 
iter_test = env.iter_test()
df_prev_test = None
for(df_test, df_sample_prediction)in iter_test:
df_test = pd.merge(df_test, df_question_meanstd, left_on = 'content_id', right_index = True, how = 'left')
 df_test = pd.merge(df_test, df_user_mean, left_on = 'user_id', right_index = True, how = 'left')
 df_test[['ur_mean', 'uqr_mean']] = df_test[['ur_mean', 'uqr_mean']].fillna (.6)
 df_test[['user_count', 'uqr_std', 'ur_std_diff']] = df_test[['user_count', 'uqr_std', 'ur_std_diff']].fillna(0)
 if df_prev_test is not None:
df_prev_test['answered_correctly'] = eval(df_test['prior_group_answers_correct'].iloc[0])
 df_prev_test = df_prev_test[df_prev_test['answered_correctly'] != -1]
df_test = df_test.set_index('user_id')
 df_prev_test = df_prev_test.groupby('user_id' ).agg({'content_id': 'count', 'answered_correctly':np.mean, 'user_count': 'last', 'ur_mean': 'last', 'uqr_mean': 'last', 'ur_std_diff':'last', 'question_mean':(np.mean), 'question_std':(np.mean)})
 df_prev_test.columns = ['un_count', 'answered_correctly', 'user_count', 'ur_mean', 'uqr_mean', 'ur_std_diff', 'question_mean', 'question_std']
df_prev_test['ur_mean'] =(((df_prev_test['ur_mean']*(lt - df_prev_test['un_count'])) +(df_prev_test['answered_correctly']*df_prev_test['un_count'])) /lt ).where(df_prev_test['user_count']>lt,(( df_prev_test['ur_mean']*df_prev_test['user_count'])+(df_prev_test['answered_correctly']*df_prev_test['un_count'])) /(df_prev_test['user_count'] + df_prev_test['un_count']))
df_prev_test['uqr_mean'] =(((df_prev_test['uqr_mean']*(lt-df_prev_test['un_count'])) +(df_prev_test['question_mean']*df_prev_test['un_count'])) /lt ).where(df_prev_test['user_count']>lt,(( df_prev_test['uqr_mean']*df_prev_test['user_count'])+(df_prev_test['question_mean']*df_prev_test['un_count'])) /(df_prev_test['user_count'] + df_prev_test['un_count']))
df_prev_test['user_count'] = df_prev_test['user_count'] + df_prev_test['un_count']
df_prev_test['uqr_std'] =(df_prev_test['uqr_mean']*(1-df_prev_test['uqr_mean'])) **0.5
df_prev_test['ur_std_diff'] =(df_prev_test['ur_mean'] - df_prev_test['uqr_mean'])/df_prev_test['uqr_std']
df_prev_test = df_prev_test[['user_count', 'ur_mean', 'uqr_mean', 'ur_std_diff']]
df_test.update(df_prev_test, overwrite=True, errors='ignore')
 for i in df_prev_test.itertuples() :
user = i.Index
df_user_mean.loc[user, ['user_count', 'ur_mean', 'uqr_mean', 'ur_std_diff']] = [i.user_count, i.ur_mean, i.uqr_mean, i.ur_std_diff]
df_test = df_test.reset_index()
df_test = df_test.sort_values('row_id')
 df_prev_test = df_test.copy()
df_test = df_test[df_test['content_type_id'] == 0]
df_test['base_prob'] = df_test['question_mean'] +(df_test['question_std'] * df_test['ur_std_diff'])
 df_test['jump'] = df_test['question_mean'] - df_test['uqr_mean']
df_test['adj'] = df_test['base_prob'].where(df_test['jump'].between (.2,.4)== False, df_test['base_prob'] -(df_test['jump']/3))
df_test['answered_correctly'] = df_test['base_prob'].where(df_test['ur_std_diff']>-.35, df_test['adj'])
 gc.collect()
env.predict(df_test[['row_id', 'answered_correctly']] )