Kaggle competition: Titanic - Machine Learning from Disaster
actual = data[int(9.5e7):, 'answered_correctly'].to_numpy()
print("Baseline Score to beat: {:.4f}".format(roc_auc_score(
 actual, data[int(9.5e7):, 'lgb_pred'].to_pandas() 
)))
 print("
Model Score Comparison: Online: {:.4f} | Offline: {:.4f}".format(
 roc_auc_score(actual, np.concatenate(ol_preds)) ,
roc_auc_score(actual, np.concatenate(preds)) 
))<load_pretrained>
print('reading input files.. ')
 data = pd.read_csv('.. /input/train.csv')
 sampl = pd.read_csv('.. /input/gender_submission.csv' )
%%time
TREELITE = False
if TREELITE:
models = []
for model in sorted(glob.glob(".. /input/treelite-converted-ensemble-8-models-25m/tl_*.so")) :
models.append(treelite_runtime.Predictor(model, verbose=False, nthread=1))
else:
models = []
for file in sorted(glob.glob(".. /input/riiid-final-model-inputs/trained_model_*.txt")) :
models.append(lgb.Booster(model_file=file))
model = ensemble(models, treelite=TREELITE)
 model, model.n_models, type(model.models[0] )<define_variables>
test = pd.read_csv('.. /input/test.csv' )
load_from_file = True
cat_cols = ['tagF', 'tagS', 'tagT', 'tagL']<load_from_disk>
df = data.append(test, sort = False )
start_time = time.time()
if not load_from_file:
pq_shifted = pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'task_container_id', 'content_type_id',
'prior_question_elapsed_time', 'prior_question_had_explanation'])
 q_mask = pq_shifted['content_type_id'] == 0
pq_shifted = pq_shifted.loc[q_mask]
pq_shifted['prior_question_had_explanation'] =(
 pq_shifted['prior_question_had_explanation'] 
.fillna(False ).astype('uint8'))
temp =(pq_shifted.groupby(['user_id', 'task_container_id'])
 [['prior_question_elapsed_time', 'prior_question_had_explanation']] 
.mean())
 temp = temp.groupby("user_id" ).shift(-1)
 temp.columns = ['pqet_shifted', 'pqhe_shifted']
pq_shifted.drop(
 ['content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation'],
axis=1, inplace=True)
 gc.collect()
pq_shifted = pq_shifted.merge(temp, left_on=['user_id', 'task_container_id'], right_index=True, how='left')
 pq_shifted.drop(['user_id'], axis=1, inplace=True)
 pq_shifted['pqet_shifted'] = pq_shifted['pqet_shifted'].astype('float32')
 pq_shifted['pqhe_shifted'] = pq_shifted['pqhe_shifted'].fillna(False ).astype(bool)
 pq_shifted.reset_index().to_feather("./pq_shifted.feather")
 content_df =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['content_id', 'answered_correctly'] ).loc[q_mask])
 content_df = pd.concat([content_df, pq_shifted], axis=1)
 del temp, pq_shifted
gc.collect()
et_med = content_df.groupby(["content_id", 'answered_correctly'])['pqet_shifted'].median()
et_med = et_med.unstack()
et_med.columns = ['wrong_et_med', 'right_et_med']
che_sum = content_df.groupby(['content_id'])['pqhe_shifted'].sum().rename("che_sum")
 tmed = content_df.groupby('content_id')['task_container_id'].median().rename("tmed")
 content_df =(content_df.groupby('content_id')['answered_correctly'].agg(
 content_c='count', c_mean='mean', c_std='std'))
content_df = pd.concat([content_df, et_med, tmed, che_sum], axis=1)
 del et_med, tmed, che_sum
gc.collect()
content_df = content_df.merge(
 ql.loc[~ql.content_type_id, ['content_id', *cat_cols, 'part', 'bundle_q_count', 'bundle_id']] 
.set_index("content_id"), on='content_id', how='left')
 else:
content_df = pd.read_csv(".. /input/riiid-final-model-inputs/content-df.csv", index_col=0)
 content_df[cat_cols + ['bundle_q_count', 'part']] = content_df[cat_cols + ['bundle_q_count', 'part']].astype('uint8')
 content_df[['wrong_et_med', 'right_et_med']] = content_df[['wrong_et_med', 'right_et_med']].astype('float32')
 content_df[['c_mean', 'c_std']] = content_df[['c_mean', 'c_std']].astype('float32')
 content_df[['content_c', 'tmed', 'che_sum', 'bundle_id']] =(
 content_df[['content_c', 'tmed', 'che_sum', 'bundle_id']] ).astype('uint16')
 print(f"Time Elapsed: {(time.time() - start_time):6.2f} Sec")
 print(f"Memory Usage: {content_df.memory_usage(deep=True ).sum() /(2**20):6.2f}", 'MB')
 print(f"Length : {len(content_df):7}")
 content_df.sample(5 )<merge>
totalt = df.isnull().sum().sort_values(ascending=False)
 percent =(df.isnull().sum() /df.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([totalt, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(6 )
start_time = time.time()
LEC_RECENT_ROLL = 10
SESSION_DURATION = 15 * 60 * 1000
if not load_from_file:
user_df =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'answered_correctly', 'timestamp', 'content_id',
'prior_question_elapsed_time', 'prior_question_had_explanation']))
user_df = user_df.merge(
 content_df[['part', 'c_mean', 'bundle_id']], left_on='content_id',
right_index=True, how='left')
 uf_bundle = user_df.groupby("user_id")['bundle_id'].first().rename("uf_bundle")
 user_df.drop(['bundle_id', 'content_id'], axis=1, inplace=True)
 user_df['sessions'] = user_df.groupby("user_id")['timestamp'].transform(lambda x: rt_func(x.values))
user_df['sessions'] =(user_df['sessions'] > SESSION_DURATION ).astype('uint8')
 sess = pd.concat([
user_df.groupby('user_id')['sessions'].sum().rename("sessions"),
user_df.groupby('user_id')['sessions'].agg(
 lambda x: np.argmax(x.values[::-1])if x.any() else len(x)- 1 ).rename("sess_event_count")
 ], axis=1)
 user_ts = user_df.groupby(['user_id', 'part'])['timestamp'].max().unstack().fillna(0)
 user_ts.columns = [f'ts_max_{i}' for i in range(1, 8)]
user_ts = pd.concat([
user_df.groupby("user_id")['timestamp'].agg(ts_max='max', uic='count'),
user_ts
], axis=1)
 user_df.drop(['sessions', 'timestamp'], axis=1, inplace=True)
 lec_recent = user_df[['user_id', 'answered_correctly']].groupby("user_id" ).tail(LEC_RECENT_ROLL)
 lec_recent['content_type_id'] = lec_recent['answered_correctly'].map({-1: 1, 0: 0, 1: 0})
 lec_recent.drop("answered_correctly", axis=1, inplace=True)
 lec_recent =(
 lec_recent.groupby('user_id')['content_type_id'] 
.apply(lambda x: np.where(x)[0][-1] + 1 if x.any() else 0)
.rename("lec_recent")
)
 user_df = user_df[user_df.answered_correctly != -1]
user_df['prior_question_had_explanation'] = user_df['prior_question_had_explanation'].astype(bool ).fillna(False)
 pq_sum = user_df.groupby("user_id")[['prior_question_had_explanation', 'prior_question_elapsed_time']].sum()
user_df.drop(['prior_question_had_explanation', 'prior_question_elapsed_time'], axis=1, inplace=True)
 gc.collect()
seen_exp = pd.read_feather("./pq_shifted.feather", columns=['index', 'pqhe_shifted'] ).set_index('index')
 user_df = pd.concat([user_df, seen_exp], axis=1)
 seen_exp = user_df.groupby(['user_id', 'answered_correctly'])['pqhe_shifted'].sum().unstack()
seen_exp.columns = ['seen_exp_when_wrong', 'seen_exp_when_right']
seen_exp.fillna(0, inplace=True)
 user_df.drop(['pqhe_shifted'], axis=1, inplace=True)
 u_crct_sum = user_df.groupby("user_id")['answered_correctly'].sum().rename("ucs")
 up_org_sum = user_df.groupby(['user_id', 'part'])['answered_correctly'].agg('sum' ).unstack()
up_org_sum = up_org_sum.astype("float32" ).fillna(0)
 user_df['answered_correctly'] = modify_ac(user_df['answered_correctly'].values, user_df['c_mean'].values)
 user_df.drop(["c_mean"], axis=1, inplace=True)
 gc.collect()
user_df = user_df.groupby(['user_id', 'part'])['answered_correctly'].agg(['sum', 'count'] ).unstack()
user_df = user_df.astype('float32' ).fillna(0)
 user_df['u_sum'] = user_df['sum'].sum(1)
 user_df['u_count'] = user_df['count'].sum(1)
 user_df = pd.concat([
user_ts, sess, lec_recent, up_org_sum,
user_df, uf_bundle, u_crct_sum,
pq_sum, seen_exp
], axis=1)
 del user_ts, pq_sum, u_crct_sum, lec_recent
del uf_bundle, sess, seen_exp
gc.collect()
user_df.columns =(
 ['ts_max', 'uic'] + [f'ts_max_{i}' for i in range(1, 8)] +
['sess_max', 'sess_event_sum'] + ['lec_recent'] +
[f'ups{i}' for i in range(1, 8)] +
[f's{i}' for i in range(1, 8)] +
[f'c{i}' for i in range(1, 8)] +
['us', 'uc'] + ['uf_bundle'] +
['ucs'] + ['pqhe_sum', 'pqet_sum'] +
['seen_exp_when_wrong', 'seen_exp_when_right'])
 else:
user_df = pd.read_csv(".. /input/riiid-final-model-inputs/user-df.csv", index_col=0)
 user_df['lec_recent'] = user_df['lec_recent'].astype('uint8')
 user_df[[f'ups{i}' for i in range(1, 8)]] = user_df[[f'ups{i}' for i in range(1, 8)]].astype("uint16")
 user_df[[f'c{i}' for i in range(1, 8)]] = user_df[[f'c{i}' for i in range(1, 8)]].astype('uint16')
 user_df[['uic', 'uc', 'ucs', 'pqhe_sum']] = user_df[['uic', 'uc', 'ucs', 'pqhe_sum']].astype("uint16")
 user_df[['sess_event_sum', 'sess_max']] = user_df[['sess_event_sum', 'sess_max']].astype("uint16")
 user_df[['seen_exp_when_wrong', 'seen_exp_when_right', 'uf_bundle']] =(
 user_df[['seen_exp_when_wrong', 'seen_exp_when_right', 'uf_bundle']].astype("uint16"))
print(f"Time Elapsed: {(time.time() - start_time):7.2f} Sec")
 print(f"Memory Usage: {user_df.memory_usage(deep=True ).sum() /(2**20):7.2f}", 'MB')
 print(f"Length : {len(user_df):8}")
 user_df.sample(5 ).T<categorify>
ticketNum = pd.DataFrame(df.Ticket.value_counts())
 ticketNum.rename(columns = {'Ticket' : 'TicketNum'}, inplace = True)
 ticketNum['TicketId'] = pd.Categorical(ticketNum.index ).codes
ticketNum.loc[ticketNum.TicketNum < 3, 'TicketId'] = -1
df = pd.merge(left = df, right = ticketNum, left_on = 'Ticket',
right_index = True, how = 'left', sort = False)
 df = df.drop(['TicketNum'],axis=1)
 df.head()
max_q = ques.question_id.max() + 1
def to_ba(indices, max_q=max_q):
'Function to convert indices to bitarray'
ba = np.zeros(max_q, dtype=bool)
 ba[indices] = 1
return bitarray(list(ba))
def parallelize(data, func, num_of_processes=8):
data_split = np.array_split(data, num_of_processes)
 pool = Pool(num_of_processes)
 data = pd.concat(pool.map(func, data_split))
pool.close()
pool.join()
return data
def run_on_subset(func, data_subset):
return data_subset.apply(func)
 def parallelize_on_rows(data, func, num_of_processes=8):
return parallelize(data, partial(run_on_subset, func), num_of_processes)
 start_time = time.time()
if not load_from_file:
repeat_c =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_id', 'answered_correctly']))
repeat_c = repeat_c[repeat_c['answered_correctly'] != -1]
repeat_c = repeat_c.groupby("user_id")['content_id'].unique()
repeat_c = parallelize_on_rows(repeat_c, to_ba)
 else:
repeat_c = pd.read_pickle(".. /input/riiid-final-model-inputs/repeat-c.pkl")
 print(f"Time Elapsed: {(time.time() - start_time):6.2f} Sec")
 print(f"Memory Usage: {sys.getsizeof(repeat_c)/(2 ** 20):6.2f} MB")
 print(f"{len(repeat_c)} users have been extracted!")
 repeat_c.sample(3 )<merge>
df['FamilyName'] = df.Name.apply(lambda x : str.split(x, ',')[0] )
start_time = time.time()
ROLL_WINDOW = 10
UROLL_NULL_FILL = -2
if not load_from_file:
u_roll =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'answered_correctly', 'content_id']))
u_roll = u_roll[u_roll.answered_correctly != -1]
u_roll = u_roll.groupby("user_id" ).tail(ROLL_WINDOW)
 u_roll = u_roll.merge(content_df['c_mean'], left_on='content_id', right_index=True, how='left')
 u_roll['answered_correctly'] = modify_ac(u_roll['answered_correctly'].values, u_roll['c_mean'].values)
 u_roll.drop(['c_mean', 'content_id'], axis=1, inplace=True)
 gc.collect()
u_roll = u_roll.groupby("user_id")['answered_correctly'].apply(
 lambda x: np.concatenate([np.repeat(UROLL_NULL_FILL, ROLL_WINDOW - len(x)) , x.values])
)
 u_roll = parallelize_on_rows(u_roll, pd.Series)
 gc.collect()
else:
u_roll = pd.read_csv(".. /input/riiid-final-model-inputs/u_roll.csv", index_col=0)
 u_roll = u_roll.astype("float32")
 print(f"Time Elapsed: {(time.time() - start_time):7.2f} Sec")
 print(f"Memory Usage: {u_roll.memory_usage(deep=True ).sum() /(2**20):7.2f}", 'MB')
 print(f"Length : {len(u_roll):7}")
 u_roll.sample(5 )<load_from_disk>
df['FamilySurv'] = 0.5
for _, grup in df.groupby(['FamilyName','Fare']):
if len(grup)!= 1:
for index, row in grup.iterrows() :
smax = grup.drop(index ).Survived.max()
smin = grup.drop(index ).Survived.min()
pid = row.PassengerId
if smax == 1:
df.loc[df.PassengerId == pid, 'FamilySurv'] = 1.0
elif smin == 0:
df.loc[df.PassengerId == pid, 'FamilySurv'] = 0.0
for _, grup in df.groupby(['Ticket']):
if len(grup)!= 1:
for index, row in grup.iterrows() :
if(row.FamilySurv == 0.0 or row.FamilySurv == 0.5):
smax = grup.drop(index ).Survived.max()
smin = grup.drop(index ).Survived.min()
pid = row.PassengerId
if smax == 1:
df.loc[df.PassengerId == pid, 'FamilySurv'] = 1.0
elif smin == 0:
df.loc[df.PassengerId == pid, 'FamilySurv'] = 0.0
df.FamilySurv.value_counts()
start_time = time.time()
ROLL_WINDOW_PQET = 10
PQET_ROLL_NULL_FILL = -1
if not load_from_file:
pqet_roll =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'prior_question_elapsed_time', 'answered_correctly']))
pqet_roll = pqet_roll[pqet_roll.answered_correctly != -1]
pqet_roll.drop(['answered_correctly'], axis=1, inplace=True)
 pqet_roll = pqet_roll.groupby("user_id" ).tail(ROLL_WINDOW_PQET)
 pqet_roll = pqet_roll.groupby("user_id")['prior_question_elapsed_time'].apply(
 lambda x: np.concatenate([np.repeat(PQET_ROLL_NULL_FILL, ROLL_WINDOW_PQET - len(x)) , x.values])
)
 pqet_roll = parallelize_on_rows(pqet_roll, pd.Series)
 gc.collect()
pqet_roll = pqet_roll.fillna(PQET_ROLL_NULL_FILL)
 else:
pqet_roll = pd.read_csv(".. /input/riiid-final-model-inputs/pqet_roll.csv", index_col=0)
 pqet_roll = pqet_roll.astype("float32")
 print(f"Time Elapsed: {(time.time() - start_time):7.2f} Sec")
 print(f"Memory Usage: {pqet_roll.memory_usage(deep=True ).sum() /(2**20):7.2f}", 'MB')
 print(f"Length : {len(pqet_roll):7}")
 pqet_roll.sample(5 )<categorify>
def CabinNum(data):
data.Cabin = data.Cabin.fillna('0')
 regex = re.compile('\s*(\w+)\s*')
 data['CabinNum'] = data.Cabin.apply(lambda x : len(regex.findall(x)))
 CabinNum(df )
start_time = time.time()
TS_RECENCY_PERIOD = 10
TS_ROLL_NULL_FILL = np.nan
if not load_from_file:
ts_roll =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'timestamp']))
ts_roll = ts_roll.groupby("user_id" ).tail(TS_RECENCY_PERIOD)
 ts_roll = ts_roll.groupby("user_id")['timestamp'].apply(
 lambda x: np.concatenate([np.repeat(TS_ROLL_NULL_FILL, TS_RECENCY_PERIOD - len(x)) , x.values])
)
 ts_roll = parallelize_on_rows(ts_roll, pd.Series)
 gc.collect()
ts_roll = ts_roll.fillna(TS_ROLL_NULL_FILL)
 else:
ts_roll = pd.read_csv(".. /input/riiid-final-model-inputs/ts_roll.csv", index_col=0)
 print(f"Time Elapsed: {(time.time() - start_time):7.2f} Sec")
 print(f"Memory Usage: {ts_roll.memory_usage(deep=True ).sum() /(2**20):7.2f}", 'MB')
 print(f"Length : {len(ts_roll):7}")
 ts_roll.sample(5 )<save_to_csv>
df.CabinNum.value_counts()
%%time
user_df.to_csv("user-df.csv")
 content_df.to_csv("content-df.csv")
 u_roll.to_csv("u_roll.csv")
 repeat_c.to_pickle("repeat-c.pkl")
 pqet_roll.to_csv("pqet_roll.csv")
 ts_roll.to_csv("ts_roll.csv")
 if os.path.exists(".. /input/riiid-final-model-inputs/pq_shifted.feather"):
! cp.. /input/riiid-final-model-inputs/pq_shifted.feather./pq_shifted.feather
with open("sample-batches.pkl", 'wb')as f:
pickle.dump(batches, f )<feature_engineering>
df.loc[df['Fare'].isnull() ]
def insert_lecture(batches):
'A simple function to randomly insert a lecture in between, for debugging purposes!'
temp = deepcopy(batches)
 i = np.random.choice(len(temp))
j = np.random.choice(len(temp[i][0]))
print(f"Lecture inserted at {i+1} batch at {j} index!")
 temp[i][0].iloc[j, 4] = 1
temp[i][0].iloc[j, 3] = np.random.choice(lectures['lecture_id'].values)
 temp[i][0].iloc[j, 6] = np.nan
temp[i][0].iloc[j, 7] = np.nan
if len(temp)> i + 1:
pgac = eval(temp[i + 1][0].iloc[0]['prior_group_answers_correct'])
 pgr = eval(temp[i + 1][0].iloc[0]['prior_group_responses'])
 pgac[j] = -1
pgr[j] = -1
temp[i + 1][0].iloc[0, -2] = str(pgac)
 temp[i + 1][0].iloc[0, -1] = str(pgr)
 return temp<define_variables>
df.loc[(df['Age'] >= 60)&(df['Pclass'] ==3)&(df['Sex'] == 'male')&(df['Embarked'] =='S')]
SUBMIT = False
if not SUBMIT:
print("Validation Mode.")
 iter_test = iter(insert_lecture(batches))
op = []
else:
print("Prediction Mode.")
 iter_test = env.iter_test()<train_model>
df.loc[df['Fare'].isnull() , 'Fare'] = 7
%%time
i, prev_test = 0, tuple()
for i, batch in enumerate(iter_test):
if len(prev_test):
processed_batch = post_process(prev_test[0], batch[0])
 ftrl.fit(dt.Frame(prev_test[1])[:, FTRL_COLS],
dt.Frame(processed_batch.loc[q_mask, 'answered_correctly']))
for _, ts, user, content, ans, pqhe, pqet in(
 processed_batch[[
'timestamp', 'user_id', 'content_id', 'answered_correctly',
'prior_question_had_explanation', 'prior_question_elapsed_time']].itertuples()):
org_ans = ans
if org_ans == -1:
part = lectures.loc[lectures.lecture_id == content, 'part'].values[0]
c_mean = None
else:
part = content_df.at[content, 'part']
c_mean = content_df.at[content, 'c_mean']
ans =(1 - c_mean)if ans == 1 else(- c_mean)
 pqet = pqet if not pd.isna(pqet)else np.nan
pqhe = int(pqhe)if not pd.isna(pqhe)else 0
if user not in user_df.index:
user_df.loc[user] = [0] * user_df.shape[1]
ts_roll.loc[user] =([TS_ROLL_NULL_FILL] *(TS_RECENCY_PERIOD - 1)) + [ts]
repeat_c.at[user] = to_ba([content])
 u_roll.loc[user] =([UROLL_NULL_FILL] *(ROLL_WINDOW - 1)) + [ans]
pqet_roll.loc[user] = [PQET_ROLL_NULL_FILL] * ROLL_WINDOW_PQET
user_df.at[user, 'uf_bundle'] = content_df.at[content, 'bundle_id']
else:
ts_roll.loc[user] = ts_roll.loc[user].values.tolist() [1:] + [ts]
if org_ans != -1:
u_roll.loc[user] = u_roll.loc[user].values.tolist() [1:] + [ans]
temp = repeat_c.at[user]
temp[content] = True
repeat_c.at[user] = temp
pqet_roll.loc[user] = pqet_roll.loc[user].values.tolist() [1:] + [pqet]
user_df.at[user, 'ts_max'] = ts
user_df.at[user, f'ts_max_{int(part)}'] = ts
user_df.at[user, 'uic'] = user_df.at[user, 'uic'] + 1
temp =(ts - user_df.at[user, 'ts_max'])> SESSION_DURATION
user_df.at[user, 'sess_max'] =(user_df.at[user, 'sess_max'] + 1 if temp else user_df.at[user, 'sess_max'])
 user_df.at[user, 'sess_event_sum'] = 0 if temp else user_df.at[user, 'sess_event_sum'] + 1
if org_ans == -1:
user_df.at[user, 'lec_recent'] = LEC_RECENT_ROLL
else:
temp = user_df.at[user, 'lec_recent']
user_df.at[user, 'lec_recent'] =(temp -1)if temp > 0 else 0
user_df.at[user, 'uc'] = user_df.at[user, 'uc'] + 1
user_df.at[user, 'us'] = user_df.at[user, 'us'] + ans
user_df.at[user, f'c{part}'] = user_df.at[user, f'c{part}'] + 1
user_df.at[user, f's{part}'] = user_df.at[user, f's{part}'] + ans
user_df.at[user, f'ups{part}'] = user_df.at[user, f'ups{part}'] + org_ans
user_df.at[user, 'ucs'] = user_df.at[user, 'ucs'] + org_ans
user_df.at[user, 'pqhe_sum'] = user_df.at[user, 'pqhe_sum'] + pqhe
if np.isnan(pqet):
user_df.at[user, 'pqet_sum'] = np.nan
elif np.isnan(user_df.at[user, 'pqet_sum']):
user_df.at[user, 'pqet_sum'] = pqet
else:
user_df.at[user, 'pqet_sum'] = user_df.at[user, 'pqet_sum'] + pqet
if pqhe:
if org_ans == 1:
user_df.at[user, 'seen_exp_when_right'] = user_df.at[user, 'seen_exp_when_right'] + 1
else:
user_df.at[user, 'seen_exp_when_wrong'] = user_df.at[user, 'seen_exp_when_wrong'] + 1
q_mask = batch[0]['content_type_id'] == 0
pred = batch[0].loc[q_mask, [
'row_id', 'timestamp', 'task_container_id', 'content_id',
'user_id', 'prior_question_elapsed_time']].reset_index()
pred = pd.concat([
pred, content_df.reindex(pred['content_id'] ).reset_index(drop=True)],
axis=1)
 pred[f'ummr_{ROLL_WINDOW}'] = u_roll.reindex(pred['user_id'] ).apply(
 lambda x: float(np.ma.masked_values(x, UROLL_NULL_FILL ).mean()), axis=1 ).values
pred[f'pqetmr_{ROLL_WINDOW_PQET}'] = pqet_roll.reindex(pred['user_id'] ).apply(
 lambda x: float(np.ma.masked_values(x, PQET_ROLL_NULL_FILL ).mean()), axis=1 ).values
pred['ts_recency_10'] = pred['timestamp'] - ts_roll.reindex(pred['user_id'] ).iloc[:, 0].values
pred['ts_recency_5'] = pred['timestamp'] - ts_roll.reindex(pred['user_id'] ).iloc[:, 4].values
temp = user_df.reindex(pred['user_id'] ).reset_index(drop=True)
 temp = pd.concat([temp, pred['part']], axis=1)
 temp['up_count'] = temp.apply(lambda x: x[f"c{int(x['part'])}"] , axis=1)
 temp['up_mean'] = temp.apply(lambda x: x[f"s{int(x['part'])}"] / x["up_count"], axis=1)
 temp['uph_mean'] = temp.apply(lambda x: x[f"ups{int(x['part'])}"] / x["up_count"], axis=1)
 temp['up_recency'] = temp.apply(lambda x: x[f"ts_max_{int(x['part'])}"], axis=1)
 temp['up_count'] = temp['up_count'] / temp['uc'].values
temp['uwrong_sum'] = temp['uc'] - temp['ucs'].values
temp['u_mean'] =(temp['us'] / temp['uc'].values)
 temp['h_mean'] =(temp['ucs'] / temp['uc'].values)
 temp['pqet_mean'] = temp['pqet_sum'] /(temp['uc'].values - 1)
 temp['seen_ratio'] = temp['pqhe_sum'] / temp['uc'].values
temp['lec_recent'] = temp['lec_recent'].fillna(0 ).astype(bool)
 temp = temp[['ts_max', 'up_count', 'up_mean', 'u_mean', 'uwrong_sum', 'lec_recent',
'h_mean', 'pqet_mean', 'seen_ratio', 'uic', 'uph_mean', 'uf_bundle',
'seen_exp_when_right', 'seen_exp_when_wrong', 'up_recency',
'sess_max', 'sess_event_sum']]
pred = pd.concat([pred, temp], axis=1 ).set_index("group_num")
 pred['uq_per_hr'] = pred['uic'] /(pred['timestamp'] / 60000)
 pred['h_mean'] =(2 * pred['h_mean'] * pred['c_mean'])/(pred['h_mean'] + pred['c_mean'])
 pred['uph_mean'] =(2 * pred['uph_mean'] * pred['c_mean'])/(pred['uph_mean'] + pred['c_mean'])
 pred['repeat_c'] = repeat_c.reindex(pred['user_id'] ).values
pred['repeat_c'] =(pred[['repeat_c', 'content_id']].apply(
 lambda x: x['repeat_c'][x['content_id']] if type(x['repeat_c'])!= float
else False, axis=1))
pred['ts_max'] = pred['ts_max'].fillna(0)
 pred['up_recency'] = pred['timestamp'] - pred['up_recency']
pred['response_time'] = pred['timestamp'] - pred['ts_max']
pred['rt_per_task'] = pred['response_time'] / pred['bundle_q_count']
pred['sessions'] =(pred['response_time'] > SESSION_DURATION ).astype(bool)
 pred['sess_event_count'] = np.where(pred['sessions'], 0, pred['sess_event_sum'] + 1)
 pred['sessions'] = pred['sess_max'] + pred['sessions']
pred['uf_bundle'] = pred['uf_bundle'].fillna(pred['bundle_id'])
 pred[['seen_exp_when_wrong', 'seen_exp_when_right', 'up_mean', 'up_count',
'up_recency', 'seen_ratio', 'rt_per_task', 'response_time', 'sessions',
'sess_event_count', 'uwrong_sum']] =(
 pred[['seen_exp_when_wrong', 'seen_exp_when_right', 'up_mean', 'up_count',
'up_recency', 'seen_ratio', 'rt_per_task', 'response_time', 'sessions',
'sess_event_count', 'uwrong_sum']].fillna(0))
pred['lgb_pred'] = model.predict(pred)
 pred['lgb_75'] = pred['lgb_pred'] > 0.75
pred['lgb_50'] = pred['lgb_pred'] > 0.5
pred['lgb_25'] = pred['lgb_pred'] > 0.25
pred['ummr_10_50'] = pred['ummr_10'] > 0
pred['h_mean_50'] = pred['h_mean'] > 0.5
pred['c_mean_50'] = pred['c_mean'] > 0.5
pred['c_mean_25'] = pred['c_mean'] > 0.25
pred['c_mean_75'] = pred['c_mean'] > 0.75
pred['pqet_proximity'] =(( pred['prior_question_elapsed_time'] - pred['right_et_med'] ).abs() < 
(pred['prior_question_elapsed_time'] - pred['wrong_et_med'] ).abs())
 df_to_dt_format(pred)
 pred['answered_correctly'] = ftrl.predict(dt.Frame(pred)[:, FTRL_COLS] ).to_list() [0]
if SUBMIT:
env.predict(pred[['row_id', 'answered_correctly']])
 else:
op.append(pred)
 prev_test =(batch[0].copy(deep=True), pred)
 else:
print(f"Sucessfully completed after {i + 1} iterations!" )<import_modules>
def FareFunc(data):
data['FareCat'] = 0
data.loc[data['Fare'] < 8, 'FareCat'] = 0
data.loc[(data['Fare'] >= 8)&(data['Fare'] < 16),'FareCat' ] = 1
data.loc[(data['Fare'] >= 16)&(data['Fare'] < 30),'FareCat' ] = 2
data.loc[(data['Fare'] >= 30)&(data['Fare'] < 45),'FareCat' ] = 3
data.loc[(data['Fare'] >= 45)&(data['Fare'] < 80),'FareCat' ] = 4
data.loc[(data['Fare'] >= 80)&(data['Fare'] < 160),'FareCat' ] = 5
data.loc[(data['Fare'] >= 160)&(data['Fare'] < 270),'FareCat' ] = 6
data.loc[(data['Fare'] >= 270), 'FareCat'] = 7
FareFunc(df )
import gc
import os
import time
import json
import psutil
import numpy as np
import pandas as pd
import riiideducation
import torch
import torch.nn as nn
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score, accuracy_score<set_options>
def FamlSize(data):
data['FamlSize'] = 0
data['FamlSize'] = data['SibSp'] + data['Parch'] + 1
FamlSize(df )
device='cuda' if torch.cuda.is_available() else 'cpu'
print(device )<init_hyperparams>
def LablFunc(data):
lsr = {'Title','Cabin'}
for i in lsr:
le.fit(data[i].astype(str))
data[i] = le.transform(data[i].astype(str))
LablFunc(df )
MAX_SEQ=100
MAX_LAG_TIME=2160
MAX_PREV_ELAPSE_TIME=600
n_questions=13523
n_parts=7
n_responses=3
n_lagtimes=2161
n_prev_elapsed=601
d_model=160
nhead=8
dim_feedforward=250
max_lr=0.0025<define_variables>
features = ['Pclass','SibSp','Parch','TicketId','Fare','CabinNum','Title']
def AgeFunc(df):
Etr = ETRg(n_estimators = 200, random_state = 2)
 AgeX_Train = df[features][df.Age.notnull() ]
AgeY_Train = df['Age'][df.Age.notnull() ]
AgeX_Test = df[features][df.Age.isnull() ]
Etr.fit(AgeX_Train,np.ravel(AgeY_Train))
AgePred = Etr.predict(AgeX_Test)
 df.loc[df.Age.isnull() , 'Age'] = AgePred
AgeFunc(df )
class TestDataset(torch.utils.data.Dataset):
def __init__(self, test_df, max_seq=100):
self.test_df=test_df
self.max_seq=max_seq
def __len__(self):
return len(self.test_df)
 def __getitem__(self, idx):
row=self.test_df.iloc[idx]
content_id=row.content_id
part=row.part
timestamp=row.timestamp
prior_question_elapsed_time=int(row.prior_question_elapsed_time)
 prev_seq=row.prev_seq 
(q_, p_, r_, lag_, prev_elapsed_)=prev_seq
seq_len=q_.size
q_=torch.tensor(q_, dtype=int)
 p_=torch.tensor(p_, dtype=int)
 r_=torch.tensor(r_, dtype=int)
 lag_=torch.tensor(lag_, dtype=int)
 prev_elapsed_=torch.tensor(prev_elapsed_, dtype=int)
 q=torch.zeros(self.max_seq, dtype=int)
 p=torch.zeros(self.max_seq, dtype=int)
 r=torch.zeros(self.max_seq, dtype=int)
 lag=torch.zeros(self.max_seq, dtype=int)
 prev_elapsed=torch.zeros(self.max_seq, dtype=int)
 label_mask=0
if seq_len == 0:
q[0]=content_id
p[0]=part
r[0]=2
prev_elapsed[0]=prior_question_elapsed_time
lag[0]=0
label_mask=0
elif seq_len == self.max_seq:
q[:-1]=q_[1:]
p[:-1]=p_[1:]
prev_elapsed[:-1]=prev_elapsed_[1:]
lag[:-1]=lag_[1:]
r[:]=r_[:]
q[-1]=content_id
p[-1]=part
prev_elapsed[-1]=prior_question_elapsed_time
lag[-1]=timestamp
label_mask=seq_len-1
else:
q[:seq_len]=q_[:]
p[:seq_len]=p_[:]
prev_elapsed[:seq_len]=prev_elapsed_[:]
lag[:seq_len]=lag_[:]
r[1:seq_len+1]=r_[:]
q[seq_len]=content_id
p[seq_len]=part
prev_elapsed[seq_len]=prior_question_elapsed_time
lag[seq_len]=timestamp
r[0]=2
label_mask=seq_len
lag[1:]=lag[1:]-lag[:-1]
lag=lag/60000
lag[lag>1440] =1441+(lag[lag>1440] - 1440)/60
lag[0]=0
lag[lag<0]=0
lag[lag>MAX_LAG_TIME]=MAX_LAG_TIME
lag=lag.type(torch.long)
 return(q, p, r, lag, prev_elapsed, label_mask )<define_search_model>
def AgeCat(data):
data['AgeCat'] = 0
data.loc[(data['Age'] <= 5), 'AgeCat'] = 0
data.loc[(data['Age'] <= 12)&(data['Age'] > 5), 'AgeCat'] = 1
data.loc[(data['Age'] <= 18)&(data['Age'] > 12), 'AgeCat'] = 2
data.loc[(data['Age'] <= 22)&(data['Age'] > 18), 'AgeCat'] = 3
data.loc[(data['Age'] <= 32)&(data['Age'] > 22), 'AgeCat'] = 4
data.loc[(data['Age'] <= 45)&(data['Age'] > 32), 'AgeCat'] = 5
data.loc[(data['Age'] <= 60)&(data['Age'] > 45), 'AgeCat'] = 6
data.loc[(data['Age'] <= 70)&(data['Age'] > 60), 'AgeCat'] = 7
data.loc[(data['Age'] > 70), 'AgeCat'] = 8
AgeCat(df )
%%time
class FFN(nn.Module):
def __init__(self, d_model=80, dim_feedforward=512, dropout=0.1):
super(FFN, self ).__init__()
self.fc1=nn.Linear(d_model, dim_feedforward)
 self.relu=nn.ReLU()
self.fc2=nn.Linear(dim_feedforward, d_model)
 self.dropout=nn.Dropout(dropout)
 def forward(self, x):
x=self.fc1(x)
 x=self.relu(x)
 x=self.fc2(x)
 x=self.dropout(x)
 return x
class TransformerLayer(nn.Module):
def __init__(self,
d_model=80,
nhead=5,
dim_feedforward=512,
dropout=0.1 
):
super(TransformerLayer, self ).__init__()
self.multihead_attn=nn.MultiheadAttention(d_model,
num_heads=nhead,
dropout=dropout)
 self.ffn=FFN(d_model, dim_feedforward, dropout)
 self.layernorm1=nn.LayerNorm(d_model)
 self.layernorm2=nn.LayerNorm(d_model)
 def forward(self, Q, K, V, attn_mask=None):
Q=self.layernorm1(Q)
 attn_output, _=self.multihead_attn(Q, K, V, attn_mask=attn_mask)
 attn_output=Q+attn_output
attn_output=self.layernorm2(attn_output)
 ffn_out=self.ffn(attn_output)
 layer_out=attn_output + ffn_out
return layer_out
class Transformer(nn.Module):
def __init__(self,
d_model=80,
nhead=5,
dim_feedforward=512,
num_layers=1,
dropout=0.1 
):
super(Transformer, self ).__init__()
self.transformer_layer=TransformerLayer(d_model,nhead,dim_feedforward, dropout)
 def forward(self, Q, K, V, attn_mask):
y=self.transformer_layer(Q, K, V, attn_mask)
 return y
class KTModel(nn.Module):
def __init__(self,
n_questions,
n_parts,
n_responses,
n_lagtimes=14400,
n_prev_elapsed=121,
MAX_SEQ=100,
d_model=80,
nhead=5,
dim_feedforward=512,
num_layers=1,
dropout=0.1,
device='cpu' 
):
super(KTModel, self ).__init__()
self.device=device
self.pos_ids=torch.arange(MAX_SEQ, device=device)
 self.pos_embedding=nn.Embedding(MAX_SEQ, d_model)
 self.q_embedding=nn.Embedding(n_questions, d_model)
 self.p_embedding=nn.Embedding(n_parts+1, d_model)
 self.r_embedding=nn.Embedding(n_responses, d_model)
 self.lag_embedding=nn.Embedding(n_lagtimes, d_model)
 self.prev_elapsed_embedding=nn.Embedding(n_prev_elapsed, d_model)
 self.encoder_transformer_layer=TransformerLayer(d_model, nhead,dim_feedforward, dropout)
 self.decoder_transformer_layer=TransformerLayer(d_model, nhead,dim_feedforward, dropout)
 self.transformer_layer=TransformerLayer(d_model, nhead,dim_feedforward, dropout)
 self.dropout=nn.Dropout(dropout)
 self.out=nn.Linear(d_model, 1)
 def get_attention_mask(self, sz):
attn_mask=torch.tensor(np.triu(np.ones(( sz, sz)) , k=1 ).astype('bool'))
attn_mask=attn_mask.to(self.device)
 return attn_mask
def get_encoder_inputs(self, q, p):
pos_embedd=self.dropout(self.pos_embedding(self.pos_ids))
q_embedd=self.dropout(self.q_embedding(q))
p_embedd=self.dropout(self.p_embedding(p))
encoder_in=pos_embedd+q_embedd+p_embedd
return encoder_in
def get_decoder_inputs(self, lag_times, prev_elapsed, r):
pos_embedd=self.dropout(self.pos_embedding(self.pos_ids))
r_embedd=self.dropout(self.r_embedding(r))
lag_embedd=self.dropout(self.lag_embedding(lag_times))
prev_elapsed_embedd=self.dropout(self.prev_elapsed_embedding(prev_elapsed))
decoder_in=pos_embedd+r_embedd+lag_embedd+prev_elapsed_embedd
return decoder_in
def forward(self, q, p, lag_times, prev_elapsed, r):
attn_mask=self.get_attention_mask(q.size(1))
encoder_in=self.get_encoder_inputs(q, p)
 decoder_in=self.get_decoder_inputs(lag_times, prev_elapsed, r)
 encoder_in=encoder_in.permute(1, 0, 2)
 decoder_in=decoder_in.permute(1, 0, 2)
 encoder_out=self.encoder_transformer_layer(encoder_in, encoder_in, encoder_in, attn_mask=attn_mask)
 decoder_out=self.decoder_transformer_layer(decoder_in, decoder_in, decoder_in, attn_mask=attn_mask)
 y=self.transformer_layer(decoder_out, encoder_out, encoder_out, attn_mask=attn_mask)
 y=y.permute(1, 0, 2)
 yout=self.out(y ).squeeze(-1)
 return yout<load_pretrained>
df.loc[df['Embarked'].isnull() ]
model=KTModel(n_questions,
n_parts,
n_responses,
n_lagtimes=n_lagtimes,
n_prev_elapsed=n_prev_elapsed,
MAX_SEQ=MAX_SEQ,
d_model=d_model,
nhead=nhead,
dim_feedforward=dim_feedforward,
device=device ).to(device)
 model.load_state_dict(torch.load('.. /input/saint-v2/sakt_saint(2 ).pth'))<feature_engineering>
def FillEmbk(data):
var = 'Embarked'
data.loc[(data.Embarked.isnull()),'Embarked']= 'C'
FillEmbk(df )
def update_group(test_df, prev_test_df):
if prev_test_df is None or(psutil.virtual_memory().percent>=90):
return
prev_answered_correctly=eval(test_df.prior_group_answers_correct.values[0])
 prev_test_df['answered_correctly']=prev_answered_correctly
prev_test_df=prev_test_df[prev_test_df.content_type_id==0]
prev_group=prev_test_df.groupby('user_id' ).apply(lambda row:(row.content_id.values[-MAX_SEQ:],
row.part.values[-MAX_SEQ:],
row.answered_correctly.values[-MAX_SEQ:],
row.timestamp.values[-MAX_SEQ:],
row.prior_question_elapsed_time.values[-MAX_SEQ:] 
))
for user_id in prev_group.index.values:
if user_id not in group.index:
group[user_id]=prev_group[user_id]
else: 
(prev_q, prev_p, prev_r, prev_lag, prev_q_elapsed)=prev_group[user_id]
group[user_id]=(
 np.append(group[user_id][0], prev_q),
np.append(group[user_id][1], prev_p),
np.append(group[user_id][2], prev_r),
np.append(group[user_id][3], prev_lag),
np.append(group[user_id][4], prev_q_elapsed)
)
 if len(group[user_id][0])> MAX_SEQ:
new_q=group[user_id][0][-MAX_SEQ:]
new_p=group[user_id][1][-MAX_SEQ:]
new_r=group[user_id][2][-MAX_SEQ:]
new_lag=group[user_id][3][-MAX_SEQ:]
new_q_elapsed=group[user_id][4][-MAX_SEQ:]
group[user_id]=(new_q, new_p, new_r, new_lag, new_q_elapsed )<load_from_csv>
def LablFunc(data):
lst = {'Embarked','Sex'}
for i in lst:
le.fit(data[i].astype(str))
data[i] = le.transform(data[i].astype(str))
LablFunc(df )
%%time
print('Load Group Data')
 group=pd.read_pickle('.. /input/saint-group-submission/saint_group.pkl')
 questions_df=pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv')
 questions_df.rename(columns={'question_id': 'content_id'}, inplace=True )<split>
target = data['Survived'].values
select_features = ['Pclass', 'Age','AgeCat','SibSp', 'Parch', 'Fare',
'Embarked', 'TicketId', 'CabinNum', 'Title','Cabin',
'FareCat', 'FamlSize','FamilySurv','Sex']
scaler = StandardScaler()
dfScaled = scaler.fit_transform(df[select_features])
 train = dfScaled[0:891].copy()
test = dfScaled[891:].copy()
env = riiideducation.make_env()
iter_test = env.iter_test()<merge>
selector = SelectKBest(f_classif, len(select_features))
selector.fit(train, target)
 scores = -np.log10(selector.pvalues_)
 indices = np.argsort(scores)[::-1]
print('Features importance:')
 for i in range(len(scores)) :
print('%.2f %s' %(scores[indices[i]], select_features[indices[i]]))
%%time
prev_test_df=None
for(test_df, sample_prediction_df)in iter_test:
test_df=test_df[['row_id', 'user_id', 'content_id', 'timestamp',
'content_type_id', 'prior_question_elapsed_time',
'prior_group_answers_correct']].merge(
 questions_df[['content_id', 'part']],how='left',on='content_id')
 update_group(test_df, prev_test_df)
 test_df.part=test_df.part.fillna(5)
 test_df.prior_question_elapsed_time.fillna(MAX_PREV_ELAPSE_TIME, inplace=True)
 test_df.prior_question_elapsed_time=test_df.prior_question_elapsed_time/1000
test_df.loc[(test_df.prior_question_elapsed_time > MAX_PREV_ELAPSE_TIME), 'prior_question_elapsed_time']=MAX_PREV_ELAPSE_TIME
prev_test_df=test_df.copy()
test_df=test_df[test_df.content_type_id==0]
test_df['prev_seq']=test_df.user_id.apply(lambda user_id: group[user_id] if user_id in group else(np.array([]), np.array([]), np.array([]), np.array([]), np.array([])))
 test_dataset=TestDataset(test_df, max_seq=100)
 test_dataloader=torch.utils.data.DataLoader(test_dataset,
batch_size=1024,
shuffle=False,
pin_memory=True,
num_workers=4)
 model.eval()
y_answered=[]
with torch.no_grad() :
for(q, p, r, lag, prev_elapsed, label_mask)in test_dataloader:
q=q.to(device)
 p=p.to(device)
 r=r.to(device)
 lag=lag.to(device)
 prev_elapsed=prev_elapsed.to(device)
 y=model(q, p, lag, prev_elapsed, r)
 y_answered.extend([torch.sigmoid(y[idx][label_id] ).cpu().item() for idx, label_id in enumerate(label_mask)])
 test_df['answered_correctly']=y_answered
env.predict(test_df[['row_id', 'answered_correctly']])
 del test_df
gc.collect()<import_modules>
from sklearn.model_selection import KFold, cross_val_score
from sklearn.ensemble import RandomForestClassifier
import pandas as pd
import numpy as np
import gc
from sklearn.metrics import roc_auc_score
from collections import defaultdict
from tqdm.notebook import tqdm
import lightgbm as lgb
import riiideducation
import matplotlib.pyplot as plt
import seaborn as sns
import psutil
import random
import os<define_variables>
SrchRFC = RandomForestClassifier(max_depth = 5, min_samples_split = 4, n_estimators = 500,
random_state = 20, n_jobs = -1)
 SrchRFC.fit(train, target )
SEED = 123
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED )<data_type_conversions>
prc = SrchRFC.predict(train)
 accuracy_score(target,prc )
def add_features(df, answered_correctly_u_count, answered_correctly_u_sum, elapsed_time_u_sum, explanation_u_sum, timestamp_u, timestamp_u_incorrect, answered_correctly_q_count, answered_correctly_q_sum, elapsed_time_q_sum, explanation_q_sum, answered_correctly_uq, update = True):
answered_correctly_u_avg = np.zeros(len(df), dtype = np.float32)
 elapsed_time_u_avg = np.zeros(len(df), dtype = np.float32)
 explanation_u_avg = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_1 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_2 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_recency_3 = np.zeros(len(df), dtype = np.float32)
 timestamp_u_incorrect_recency = np.zeros(len(df), dtype = np.float32)
 answered_correctly_q_avg = np.zeros(len(df), dtype = np.float32)
 elapsed_time_q_avg = np.zeros(len(df), dtype = np.float32)
 explanation_q_avg = np.zeros(len(df), dtype = np.float32)
 answered_correctly_uq_count = np.zeros(len(df), dtype = np.int32)
 for num, row in enumerate(df[['user_id', 'answered_correctly', 'content_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'timestamp']].values):
if answered_correctly_u_count[row[0]] != 0:
answered_correctly_u_avg[num] = answered_correctly_u_sum[row[0]] / answered_correctly_u_count[row[0]]
elapsed_time_u_avg[num] = elapsed_time_u_sum[row[0]] / answered_correctly_u_count[row[0]]
explanation_u_avg[num] = explanation_u_sum[row[0]] / answered_correctly_u_count[row[0]]
else:
answered_correctly_u_avg[num] = np.nan
elapsed_time_u_avg[num] = np.nan
explanation_u_avg[num] = np.nan
if len(timestamp_u[row[0]])== 0:
timestamp_u_recency_1[num] = np.nan
timestamp_u_recency_2[num] = np.nan
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 1:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][0]
timestamp_u_recency_2[num] = np.nan
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 2:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][1]
timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][0]
timestamp_u_recency_3[num] = np.nan
elif len(timestamp_u[row[0]])== 3:
timestamp_u_recency_1[num] = row[5] - timestamp_u[row[0]][2]
timestamp_u_recency_2[num] = row[5] - timestamp_u[row[0]][1]
timestamp_u_recency_3[num] = row[5] - timestamp_u[row[0]][0]
if len(timestamp_u_incorrect[row[0]])== 0:
timestamp_u_incorrect_recency[num] = np.nan
else:
timestamp_u_incorrect_recency[num] = row[5] - timestamp_u_incorrect[row[0]][0]
if answered_correctly_q_count[row[2]] != 0:
answered_correctly_q_avg[num] = answered_correctly_q_sum[row[2]] / answered_correctly_q_count[row[2]]
elapsed_time_q_avg[num] = elapsed_time_q_sum[row[2]] / answered_correctly_q_count[row[2]]
explanation_q_avg[num] = explanation_q_sum[row[2]] / answered_correctly_q_count[row[2]]
else:
answered_correctly_q_avg[num] = np.nan
elapsed_time_q_avg[num] = np.nan
explanation_q_avg[num] = np.nan
answered_correctly_uq_count[num] = answered_correctly_uq[row[0]][row[2]]
answered_correctly_u_count[row[0]] += 1
elapsed_time_u_sum[row[0]] += row[3]
explanation_u_sum[row[0]] += int(row[4])
 if len(timestamp_u[row[0]])== 3:
timestamp_u[row[0]].pop(0)
 timestamp_u[row[0]].append(row[5])
 else:
timestamp_u[row[0]].append(row[5])
 answered_correctly_q_count[row[2]] += 1
elapsed_time_q_sum[row[2]] += row[3]
explanation_q_sum[row[2]] += int(row[4])
 answered_correctly_uq[row[0]][row[2]] += 1
if update:
answered_correctly_u_sum[row[0]] += row[1]
if row[1] == 0:
if len(timestamp_u_incorrect[row[0]])== 1:
timestamp_u_incorrect[row[0]].pop(0)
 timestamp_u_incorrect[row[0]].append(row[5])
 else:
timestamp_u_incorrect[row[0]].append(row[5])
 answered_correctly_q_sum[row[2]] += row[1]
user_df = pd.DataFrame({'answered_correctly_u_avg': answered_correctly_u_avg, 'elapsed_time_u_avg': elapsed_time_u_avg, 'explanation_u_avg': explanation_u_avg,
'answered_correctly_q_avg': answered_correctly_q_avg, 'elapsed_time_q_avg': elapsed_time_q_avg, 'explanation_q_avg': explanation_q_avg,
'answered_correctly_uq_count': answered_correctly_uq_count, 'timestamp_u_recency_1': timestamp_u_recency_1, 'timestamp_u_recency_2': timestamp_u_recency_2,
'timestamp_u_recency_3': timestamp_u_recency_3, 'timestamp_u_incorrect_recency': timestamp_u_incorrect_recency})
 df = pd.concat([df, user_df], axis = 1)
 return df
def update_features(df, answered_correctly_u_sum, answered_correctly_q_sum, timestamp_u_incorrect):
for row in df[['user_id', 'answered_correctly', 'content_id', 'content_type_id', 'timestamp']].values:
if row[3] == 0:
answered_correctly_u_sum[row[0]] += row[1]
if row[1] == 0:
if len(timestamp_u_incorrect[row[0]])== 1:
timestamp_u_incorrect[row[0]].pop(0)
 timestamp_u_incorrect[row[0]].append(row[4])
 else:
timestamp_u_incorrect[row[0]].append(row[4])
 answered_correctly_q_sum[row[2]] += row[1]
return 
<load_from_csv>
prdt2 = SrchRFC.predict(test)
 print('Predicted result: ', prdt2 )
<split><EOS> 
sampl['Survived'] = pd.DataFrame(prdt2)
 sampl.to_csv('submission.csv', index=False )