Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<data_type_conversions>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
if(combined_train_test_df['GarageType'].isnull().sum() != 0):
combined_train_test_df['GarageType'].fillna(combined_train_test_df['GarageType'].mode().iloc[0], inplace = True )<data_type_conversions>
path = Path('/kaggle/input')
 assert path.exists()
if(combined_train_test_df['GarageCond'].isnull().sum() != 0):
combined_train_test_df['GarageCond'].fillna(combined_train_test_df['GarageCond'].mode().iloc[0], inplace = True )<set_options>
def add_features(df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum,
timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum,
elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update=True):
answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)
 elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)
 explanation_u_avg = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)
 answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)
 elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)
 explanation_q_avg = np.zeros(len(df), dtype = np.float32)
 answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)
 for num, row in enumerate(df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'timestamp']].values):
if answered_correctly_u_count[row[0]] != 0:
answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]
elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]
explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]
else:
answered_correctly_u_avg[num] = np.nan
elapsed_time_u_avg[num] = np.nan
explanation_u_avg[num] = np.nan
print(timestamp_u[row[0]])
 if len(timestamp_u[row[0]])== 0:
timestamp_u_recency_1[num] = np.nan
timestamp_u_recency_2[num] = np.nan
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 1:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]
timestamp_u_recency_2[num] = np.nan
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 2:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]
timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 3:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]
timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]
timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]
if len(timestamp_u_incorrect[row[0]])== 0:
timestamp_u_incorrect_recency[num] = np.nan
else:
timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]
if answered_correctly_q_count[row[2]] != 0:
answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]
elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]
explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]
else:
answered_correctly_q_avg[num] = np.nan
elapsed_time_q_avg[num] = np.nan
explanation_q_avg[num] = np.nan
answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]
answered_correctly_u_count[row[0]] += 1
elapsed_time_u_sum[row[0]] += row[3]
explanation_u_sum[row[0]] += int(row[4])
 if len(timestamp_u[row[0]])== 3:
timestamp_u[row[0]].pop(0)
 timestamp_u[row[0]].append(row[5])
 else:
timestamp_u[row[0]].append(row[5])
 answered_correctly_q_count[row[2]] += 1
elapsed_time_q_sum[row[2]] += row[3]
explanation_q_sum[row[2]] += int(row[4])
 answered_correctly_uq[row[0]][row[2]] += 1
if update:
answered_correctly_u_sum[row[0]] += row[1]
if row[1] == 0:
if len(timestamp_u_incorrect[row[0]])== 1:
timestamp_u_incorrect[row[0]].pop(0)
 timestamp_u_incorrect[row[0]].append(row[5])
 else:
timestamp_u_incorrect[row[0]].append(row[5])
 answered_correctly_q_sum[row[2]] += row[1]
user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg,
'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg,
'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,
'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency})
 df.reset_index(drop=True, inplace=True)
 user_df.reset_index(drop=True, inplace=True)
 df = pd.concat([df, user_df], axis=1)
 return df
if(combined_train_test_df['GarageQual'].isnull().sum() != 0):
combined_train_test_df['GarageQual'].fillna(combined_train_test_df['GarageQual'].mode().iloc[0], inplace = True )<categorify>
def update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect):
for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp']].values:
if row[3] == 0:
answered_correctly_u_sum[row[0]] += row[1]
if row[1] == 0:
if len(timestamp_u_incorrect[row[0]])== 1:
timestamp_u_incorrect[row[0]].pop(0)
 timestamp_u_incorrect[row[0]].append(row[4])
 else:
timestamp_u_incorrect[row[0]].append(row[4])
 answered_correctly_q_sum[row[2]] += row[1]
return
if(combined_train_test_df['GarageYrBlt'].isnull().sum() != 0):
combined_train_test_df['GarageYrBlt'].fillna(combined_train_test_df['YearBuilt'], inplace = True )<prepare_x_and_y>
questions_df = pd.read_csv(path/'riiid-test-answer-prediction/questions.csv')
 questions_df['part'] = questions_df['part'].astype(np.int32)
 questions_df['bundle_id'] = questions_df['bundle_id'].astype(np.int32 )
if(combined_train_test_df['LotFrontage'].isnull().sum() != 0):
df_LF_LA = combined_train_test_df[['LotFrontage', 'LotArea']]
X_train_LF_LA = df_LF_LA[df_LF_LA['LotFrontage'].notnull() ].drop(labels =('LotFrontage'), axis = 1)
 y_train_LF_LA = df_LF_LA[df_LF_LA['LotFrontage'].notnull() ].drop(labels = ['LotArea'], axis = 1)
 X_test_LF_LA = df_LF_LA[df_LF_LA['LotFrontage'].isnull() ].drop(['LotFrontage'], axis = 1)
 y_test_LF_LA = fill_missing_data_basic_reg(X_train_LF_LA, y_train_LF_LA, X_test_LF_LA)
 combined_train_test_df.loc[(combined_train_test_df.LotFrontage.isnull()), 'LotFrontage'] = y_test_LF_LA<feature_engineering>
gbm = lgb.Booster(model_file='.. /input/lightgbmmodel/model.txt' )
if(combined_train_test_df['FireplaceQu'].isnull().sum().sum() != 0):
combined_train_test_df['FireplaceQu'] = combined_train_test_df['FireplaceQu'].fillna('NoFirePlace' )<feature_engineering>
json_file = ".. /input/lightgbm-features-dicts/lightgbm_features_dicts.json"
f = open(json_file)
 features_dicts = json.load(f )
if(combined_train_test_df['Fence'].isnull().sum().sum() != 0):
combined_train_test_df['Fence'] = combined_train_test_df['Fence'].fillna('NoFence' )<feature_engineering>
del f
if(combined_train_test_df['Alley'].isnull().sum().sum() != 0):
combined_train_test_df['Alley'] = combined_train_test_df['Alley'].fillna('NoAlley' )<feature_engineering>
def change_type(temp):
result = defaultdict(int)
 for k,v in temp.items() :
key = float(k)
 value = int(float(v))
result[key] =value
return result
answered_correctly_u_count = change_type(answered_correctly_u_count_)
 answered_correctly_u_sum = change_type(answered_correctly_u_sum_)
 elapsed_time_u_sum = change_type(elapsed_time_u_sum_)
 explanation_u_sum = change_type(explanation_u_sum_)
 answered_correctly_q_count = change_type(answered_correctly_q_count_)
 answered_correctly_q_sum = change_type(answered_correctly_q_sum_)
 elapsed_time_q_sum = change_type(elapsed_time_q_sum_)
 explanation_q_sum = change_type(explanation_q_sum_)
 del answered_correctly_u_count_
del answered_correctly_u_sum_
del elapsed_time_u_sum_
del explanation_u_sum_
del answered_correctly_q_count_
del answered_correctly_q_sum_
del elapsed_time_q_sum_
del explanation_q_sum_
if(combined_train_test_df['MiscFeature'].isnull().sum().sum() != 0):
combined_train_test_df['MiscFeature'] = combined_train_test_df['MiscFeature'].fillna('NoMiscFeature' )<feature_engineering>
def change_answered_correctly_uq_type(t):
result = defaultdict(lambda: defaultdict(int))
for k,v in t.items() :
key = float(k)
 temp = defaultdict(int)
 for k1,v1 in v.items() :
k1 = float(k1)
 temp[k1] = v1
result[key] = temp
return result
answered_correctly_uq = change_answered_correctly_uq_type(answered_correctly_uq_)
 del answered_correctly_uq_
if(combined_train_test_df['PoolQC'].isnull().sum().sum() != 0):
combined_train_test_df['PoolQC'] = combined_train_test_df['PoolQC'].fillna('NoPoolQC' )<count_missing_values>
def changetolist(t):
result = defaultdict(list)
 for k,v in t.items() :
key = float(k)
 value = list(v)
 result[key] = value
return result
timestamp_u = changetolist(timestamp_u_)
 timestamp_u_incorrect = changetolist(timestamp_u_incorrect_)
 del timestamp_u_
del timestamp_u_incorrect_
<define_variables><EOS> 
TARGET = 'answered_correctly'
FEATURES = ['answered_correctly_u_avg', 'explanation_u_avg', 'elapsed_time_u_avg',
'answered_correctly_q_avg', 'explanation_q_avg', 'elapsed_time_q_avg',
'answered_correctly_uq_count', 'timestamp_u_recency_1',
'timestamp_u_recency_2', 'timestamp_u_recency_3',
'timestamp_u_incorrect_recency']
prior_question_elapsed_time_mean = 25423.810042960366
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_features(previous_test_df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect)
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)
 test_df = pd.merge(test_df, questions_df[['question_id', 'part']], left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df[TARGET] = 0
test_df = add_features(test_df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = False)
 test_df[TARGET] = gbm.predict(test_df[FEATURES],num_iteration=gbm.best_iteration)
 set_predict(test_df[['row_id', TARGET]])
 print('Job Done' )