Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<feature_engineering>
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.model_selection import GridSearchCV
ba = bitarray(15000, endian='little')
 ba.setall(False)
 repeat_c = 0
temp = data[~data.content_type_id].loc[data.user_id == np.random.choice(data.user_id.unique()), ['content_id']]
for _, c in temp['content_id'].iteritems() :
if ba[c]:
print(f"{c:<5} was already viewed by the user!")
 repeat_c += 1
else:
ba[c] = 1
if repeat_c == 0:
print("No repeated questions for this user")
 else:
print(f"
{repeat_c} question(s)were repeated")
 np.array(ba.tolist())[temp.content_id.values].all()<normalization>
warnings.filterwarnings(action='ignore', category=FutureWarning)
@nb.njit
def shifted_expanding_mean(arr, keep_last=False):
'keep_last is set to true when we apply this func to pqet_mean.SPECIAL USE CASE!'
temp = expanding_mean(arr)
 if not keep_last:
return np.concatenate(( np.array([np.nan]), temp[:-1]))
else:
return np.concatenate(( np.array([np.nan]), temp))
shifted_expanding_mean(temp)
 %timeit -n 20 shifted_expanding_mean(temp)
 %timeit -n 20 pd.Series(temp ).expanding(1 ).mean().shift()
np.testing.assert_allclose(pd.Series(temp ).expanding(1 ).mean().shift() , shifted_expanding_mean(temp))<feature_engineering>
train_data = pd.read_csv(".. /input/train.csv")
 test_data = pd.read_csv(".. /input/test.csv" )
@nb.njit
def rt_func(arr, pred=False):
temp = np.concatenate(( np.array([np.nan]), arr[1:] - arr[:-1]))
temp = np.where(temp == 0, np.nan, temp)
 mask = np.isnan(temp)
 idx = np.arange(len(mask))
idx = np.where(mask, 0, idx)
 rmax, i = idx[0], 0
for i, val in enumerate(idx):
if val > rmax:
rmax = val
idx[i] = rmax
temp[mask] = temp[idx[mask]]
if not pred:
temp = np.where(np.isnan(temp), 0, temp)
 else:
temp = np.where(np.isnan(temp), arr, temp)
 return temp<groupby>
svc_clf = Pipeline([('vect', TfidfVectorizer()), 
('transformer', TfidfTransformer()), 
('classify', SGDClassifier(loss='hinge', penalty='l2', alpha=1e-3,
random_state=0, max_iter=5, tol=None)) ] )
%time data.groupby("user_id")['timestamp'].transform(lambda x: rt_func(x.values))
%time data.groupby("user_id")['timestamp'].transform(lambda x: x.diff().replace(0, np.nan ).fillna(method='ffill' ).fillna(0)) 
(
 data.groupby("user_id")['timestamp'].transform(lambda x: rt_func(x.values))
== data['response_time']
).all()<feature_engineering>
svc_clf.fit(train_data['Name'][:700], train_data['Survived'][:700] )
@nb.njit
def modify_ac(arr, c_mean):
temp = np.where(arr == 0, -1, 1)* c_mean
return np.where(temp < 0, temp, 1 - temp )<feature_engineering>
predictions = svc_clf.predict(train_data['Name'][700:] )
%%time
data['ac_modified'] = modify_ac(
 data['answered_correctly'].values,
data.groupby(["content_id", 'content_type_id'])['answered_correctly'].transform('mean' ).values)
 data.loc[data.content_type_id, 'ac_modified'] = 0<categorify>
survived_or_not = train_data['Survived'][700:]
@nb.njit
def fillnshift(arr):

 mask = np.isnan(arr)
 idx = np.arange(len(mask))
idx = np.where(mask, 0, idx)
 rmax, i = idx[0], 0
for i, val in enumerate(idx):
if val > rmax:
rmax = val
idx[i] = rmax
arr[mask] = arr[idx[mask]]
return np.concatenate(( np.array([np.nan]), arr[:-1]))<normalization>
np.mean(predictions == survived_or_not )
@nb.njit
def moving_average(arr, n=10, shift=True):
mask = np.isnan(arr)
 ret = np.cumsum(np.where(mask, 0, arr))
ret[n:] = ret[n:] - ret[:-n]
counts = np.cumsum(~mask)
 counts[n:] = counts[n:] - counts[:-n]
ret[~mask] /= counts[~mask]
ret[mask] = np.nan
if shift:
ret = np.concatenate(( np.array([np.nan]), ret[:-1]))
return ret<feature_engineering>
parameters = {'vect__ngram_range' : [(1, 1),(2, 2),(3 , 3)],
'transformer__use_idf' :(True, False),
'classify__alpha' :(1e-2, 1e-3),
}
data['up_mean'] = data[~data.content_type_id].groupby(['user_id', 'part'])['ac_modified'].transform(
 lambda x: shifted_expanding_mean(x.values))
data['up_count'] =(data[~data.content_type_id].groupby(['user_id', 'part'] ).cumcount()
/ data[~data.content_type_id].groupby("user_id" ).cumcount())
 data['up_count'] = data['up_count'].fillna(0 )<feature_engineering>
gs_clf = GridSearchCV(svc_clf, parameters, n_jobs=-1 )
temp = np.random.randint(0, 2, size=int(1e5)).astype('float')
 temp[0] = np.nan
temp = pd.Series(temp)
 np.testing.assert_allclose(
 temp.expanding().mean() ,
shifted_expanding_mean(temp.values[1:], keep_last=True)
 )<feature_engineering>
gs_clf.fit(train_data['Name'], train_data['Survived'] )
data['content_c'] = data.groupby(["content_id", 'content_type_id'])['row_id'].transform("count")
 data['seen_ratio'] =(data.loc[~data.content_type_id, ['user_id', 'prior_question_had_explanation']] 
.fillna(False ).astype(float ).groupby("user_id")
.transform(lambda x: expanding_mean(x.values)))
 data['pqet_mean'] =(
 data[(~data.content_type_id)].groupby("user_id")['prior_question_elapsed_time'] 
.transform(lambda x: shifted_expanding_mean(x.values[1:], keep_last=True)) 
)
 data.loc[~data.content_type_id, 'repeat_c'] =(
 data[~data.content_type_id].groupby(["user_id", 'content_id'])
.cumcount().astype(bool))
data['repeat_c'] = data['repeat_c'].fillna(False)
 data['uq_per_hr'] = data.groupby("user_id" ).cumcount() /(data['timestamp'] / 60000)
 data['tmed'] = data[~data.content_type_id].groupby("content_id")[['task_container_id']].transform('median')
 data['uwrong_sum'] =(
 data[~data.content_type_id]['answered_correctly'] 
.map({0: 1, 1: 0} ).groupby(data['user_id'])
.transform(lambda x: shifted_expanding_sum(x.values))
)<data_type_conversions>
cv_result = pd.DataFrame(gs_clf.cv_results_ )
data['lec_recent'] =(
 data.loc[data.content_type_id, 'content_type_id'] 
.reindex(data.index ).groupby(data['user_id'])
.fillna(method='ffill', limit=10)
.fillna(False ).astype(bool)
)
 data['uf_bundle'] = data.groupby("user_id")['bundle_id'].transform('first')
 data['pqetmr_10'] =(data[~data.content_type_id].groupby("user_id")['prior_question_elapsed_time'].transform(
 lambda x: moving_average(x.values.astype('float32'), shift=False, n=10)))
 data['che_sum'] = data[~data.content_type_id].groupby("content_id")['pqhe_shifted'].transform('sum')
 data['che_sum'] = data['che_sum'].fillna(0 ).astype('int' )<data_type_conversions>
gs_clf.best_params_
data['seen_exp_when_wrong'] =(data['pqhe_shifted'].fillna(False)&(data['answered_correctly'] == 0)).astype(int)
 data['seen_exp_when_wrong'] =(data[~data.content_type_id].groupby('user_id')['seen_exp_when_wrong'] 
.transform(lambda x: shifted_expanding_sum(x.values)))
 data['seen_exp_when_right'] =(data['pqhe_shifted'].fillna(False)&(data['answered_correctly'] == 1)).astype(int)
 data['seen_exp_when_right'] =(data[~data.content_type_id].groupby('user_id')['seen_exp_when_right'] 
.transform(lambda x: shifted_expanding_sum(x.values)))
 data['sessions'] =(data['response_time'] >(15*60*1000)).groupby(data['user_id'] ).cumsum()
data['sess_event_count'] = data.groupby(['user_id', 'sessions'] ).cumcount()
data['up_recency'] = data.groupby(['user_id', 'part'])['timestamp'].transform(lambda x: rt_func(x.values))
data['ts_recency_10'] = data['timestamp'] - data.groupby("user_id")['timestamp'].shift(10)
 data['ts_recency_5'] = data['timestamp'] - data.groupby("user_id")['timestamp'].shift(5 )<set_options>
best_model_svc = Pipeline([('vect', TfidfVectorizer()), 
('transformer', TfidfTransformer(use_idf=False)) , 
('classify', SGDClassifier(loss='hinge', penalty='l2', alpha=1e-3,
random_state=0, max_iter=5, tol=None)) ] )
warnings.filterwarnings("ignore", category=UserWarning)
 train_cols = [
'repeat_c',
'tagF', 'tagS', 'tagL', 'tagT',
'response_time',
'prior_question_elapsed_time',
'up_mean', 'up_count', 'uq_per_hr',
'uwrong_sum', 'lec_recent',
'pqet_mean', 'seen_ratio', 'tmed',
'up_recency', 'ts_recency_10',
'ts_recency_5',
'timestamp', 'task_container_id', 'content_c',
'che_sum', 'seen_exp_when_wrong',
'seen_exp_when_right',
'sessions','sess_event_count',
'pqetmr_10', 'uf_bundle',
]
cat_cols = ['tagF', 'tagS', 'tagT', 'tagL']
other_cols = ['train', 'user_id', 'content_id', 'ac_modified',
'pqet_shifted', 'bundle_q_count', 'part']
folds = 7
best_params = {
'learning_rate':0.075,
'num_leaves': 200,
'objective': 'binary',
'metric': 'auc'}
total_users = data.user_id.unique()
np.random.shuffle(total_users)
 val_size = len(total_users)// folds
scores = {'TRAIN': [], 'VAL': []}
models = []
for i in range(folds):
start_time = time.time()
train_users = np.setdiff1d(total_users, total_users[(i)*val_size:(i+1)*val_size])
 temp = generate_train_val(data, train_users, dyn_p=True)
 temp = temp[~temp.content_type_id]
temp = temp.loc[~temp.content_type_id, train_cols + other_cols + ['answered_correctly']]
temp[['u_mean']] =(
 temp.groupby(["user_id"])['ac_modified'] 
.transform(lambda x: shifted_expanding_mean(x.values)) 
)
 temp['ummr_10'] =(
 temp.groupby('user_id')['ac_modified'] 
.transform(lambda x: moving_average(x.values, n=10)) 
)
 temp['rt_per_task'] = temp['response_time'] / temp['bundle_q_count']
temp[cat_cols] = temp[cat_cols].fillna(0)
 temp[cat_cols] = temp[cat_cols].astype(int)
 temp = temp.merge(
 temp[temp.train].groupby("content_id")['answered_correctly'].agg(c_mean='mean', c_std='std'),
left_on=['content_id'], right_index=True, how='left')
 temp['c_mean'] = temp['c_mean'].fillna(0.5)
 temp['h_mean'] = temp.groupby("user_id")['answered_correctly'].transform(
 lambda x: shifted_expanding_mean(x.values)
)
 temp['h_mean'] =(2 * temp['h_mean'] * temp['c_mean'])/(temp['h_mean'] + temp['c_mean'])
 temp['uph_mean'] = temp.groupby(['user_id', 'part'])['answered_correctly'].transform(
 lambda x: shifted_expanding_mean(x.values))
temp['uph_mean'] =(2 * temp['uph_mean'] * temp['c_mean'])/(temp['uph_mean'] + temp['c_mean'])
 temp = temp.merge(
 temp[temp.train].groupby(["content_id", 'answered_correctly'])['pqet_shifted'].agg('median' ).unstack() ,
left_on='content_id', right_index=True, how='left')
 temp.columns = temp.columns[:-2].tolist() + ['wrong_et_med', 'right_et_med']
train, val = temp[temp['train']], temp[~temp['train']]
train, val = train.drop(other_cols, 1), val.drop(other_cols, 1)
 if i == 0:
if not best_params:
train_lgb = lgb.Dataset(
 train.drop('answered_correctly', 1), train['answered_correctly'],
free_raw_data=False, categorical_feature=cat_cols)
 val_lgb = lgb.Dataset(
 val.drop('answered_correctly', 1), val['answered_correctly'],
free_raw_data=False, categorical_feature=cat_cols)
 print("Searching for Optimal Hyperparameters! This may take some time to complete.")
 model = lgbo.train(
 params={'objective':'binary', 'metric':'auc'},
train_set=train_lgb, valid_sets=[train_lgb, val_lgb],
verbose_eval=0, early_stopping_rounds=10,
show_progress_bar=False,
time_budget=60*60, 
)
 params = model.params
else:
params = best_params
print(f"Starting folds: {'='*80}>")
 d_time = time.time() - start_time
model = lgb.LGBMClassifier(**params)
 model.fit(train.drop('answered_correctly', 1), train['answered_correctly'],
eval_set=[(val.drop('answered_correctly', 1), val['answered_correctly'])],
early_stopping_rounds=10, verbose=0)
 model = model.booster_
models.append(model)
 scores['TRAIN'] = scores.get('TRAIN')+ [roc_auc_score(train['answered_correctly'], model.predict(
 train.drop("answered_correctly", 1), categorical_feature=cat_cols)) ]
scores['VAL'] = scores.get('VAL')+ [roc_auc_score(val['answered_correctly'], model.predict(
 val.drop("answered_correctly", 1), categorical_feature=cat_cols)) ]
print("AUC for Fold 
.format(i+1, folds, scores['TRAIN'][-1], scores['VAL'][-1], d_time, time.time() - start_time))
print("Summary stats : {}
Mean AUC for {} folds: T: {:.4f} @ {:.4f} std | V: {:.4f} @ {:.4f} std" 
.format('='*76+">", folds, np.mean(scores['TRAIN']), np.std(scores['TRAIN']),
np.mean(scores['VAL']), np.std(scores['VAL'])))
 print(f"
Additional Info: {'='*80+'>'}
Train Features Used By Model: {val.shape[1] - 1} Features")
 print(f"Hypothetical Space Required For Entire Data: {(val.shape[1] * 101230332 * 32)/ 8e+9:.2f} GB(float 32)")
 print(f"Hypothetical Space Required For Entire Data: {(val.shape[1] * 101230332 * 64)/ 8e+9:.2f} GB(float 64)" )<drop_column>
best_model_svc.fit(train_data['Name'], train_data['Survived'] )
del data, temp, train, val
data = temp = train = val = None
gc.collect()<load_pretrained>
predictions = best_model_svc.predict(test_data['Name'] )
start_time = time.time()
LEC_RECENT_ROLL = 10
ROLL_WINDOW = 10
ROLL_WINDOW_PQET = 10
SESSION_DURATION = 15 * 60 * 1000
CHUNKS = 3
SAVE_LOC = f".. /input/riiid-final-model-inputs/model_train_c1.feather"
MODEL_LOC = f".. /input/riiid-final-model-inputs/trained_model.txt"
if not os.path.exists(SAVE_LOC):
data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'timestamp', 'answered_correctly'])
 q_mask = data.answered_correctly != -1
BATCH_SIZE = q_mask.sum() // CHUNKS
del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s", end=' ')
 print("| Percentage of Data used: {:.2f}% | Single chunk Size: {:9} rows".format(
 q_mask.sum() *100/len(q_mask), BATCH_SIZE 
))
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Entering Data Prep Bottleneck 1!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'timestamp', 'content_id', 'content_type_id'])
 data = data.merge(
 ql[['content_id', 'content_type_id', 'part']],
on=['content_id', 'content_type_id'], how='left')
 data.drop(['content_id', 'content_type_id'], axis=1, inplace=True)
 data['part'] = data['part'].astype('uint8')
 gc.collect()
up_recency = pd.Series(data=np.zeros(len(data), dtype='float64'), index=data.index, name='up_recency')
 for i in range(1, 8):
temp =(data[data.part == i].groupby("user_id")['timestamp'] 
.transform(lambda x: rt_func(x.values)))
 temp = temp.reindex(data.index ).fillna(0 ).values.astype('float64')
 up_recency = up_recency + temp
del temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with up_recency_{i}!")
 data['up_recency'] = up_recency
data.drop('part', axis=1, inplace=True)
 del up_recency
gc.collect()
data['ts_recency_10'] = data['timestamp'] - data.groupby('user_id')['timestamp'].shift(10)
 data['ts_recency_5'] = data['timestamp'] - data.groupby('user_id')['timestamp'].shift(5)
 data = data.loc[q_mask]
data.drop(['user_id', 'timestamp'], axis=1, inplace=True)
 data.reset_index().to_feather("ts_up_recency.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Done creating up/ts recency features!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'answered_correctly'] ).loc[q_mask]
pq_shifted = pd.read_feather(".. /input/riiid-final-model-inputs/pq_shifted.feather",
columns=['index', 'pqhe_shifted'] ).set_index("index")
 data['seen_exp_when_wrong'] =(pq_shifted['pqhe_shifted'] &(data['answered_correctly'] == 0)).astype(int)
 data['seen_exp_when_right'] =(pq_shifted['pqhe_shifted'] &(data['answered_correctly'] == 1)).astype(int)
 del pq_shifted
gc.collect()
data['seen_exp_when_wrong'] =(
 data.groupby('user_id')['seen_exp_when_wrong'] 
.transform(lambda x: shifted_expanding_sum(x.values)))
 data['seen_exp_when_right'] =(
 data.groupby('user_id')['seen_exp_when_right'] 
.transform(lambda x: shifted_expanding_sum(x.values)))
 data = data[['seen_exp_when_wrong', 'seen_exp_when_right']]
data = data.fillna(0 ).astype("uint16")
 data.astype('uint16' ).reset_index().to_feather("seen_exp.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating seen_exp features!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_id'] ).loc[q_mask]
content_df = pd.read_csv(".. /input/riiid-final-model-inputs/content-df.csv", index_col=0,
usecols=['content_id', 'bundle_id'])
 content_df['bundle_id'] = content_df['bundle_id'].astype("uint16")
 data = data.merge(content_df['bundle_id'], left_on='content_id', right_index=True, how='left')
 data.drop(['content_id'], axis=1, inplace=True)
 data = data.groupby("user_id")['bundle_id'].transform("first" ).rename("uf_bundle")
 data.astype("uint16" ).reset_index().to_feather("uf_bundle.feather")
 del data, content_df
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating uf_bundle!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'prior_question_elapsed_time'] ).loc[q_mask]
data['pqetmr_10'] = data.groupby("user_id")['prior_question_elapsed_time'].transform(
 lambda x: moving_average(x.values, n=10, shift=False)
)
 data.drop(['prior_question_elapsed_time', 'user_id'], axis=1, inplace=True)
 data.astype('float32' ).reset_index().to_feather("pqetr.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating pqet rolling features!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_id', 'answered_correctly'] ).loc[q_mask]
content_df = pd.read_csv(".. /input/riiid-final-model-inputs/content-df.csv", index_col=0,
usecols=['part', 'c_mean', 'content_id'])
 content_df['part'] = content_df['part'].astype('uint8')
 content_df['c_mean'] = content_df['c_mean'].astype('float32')
 data = data.merge(content_df, left_on='content_id', right_index=True, how='left')
 data.drop(['content_id'], axis=1, inplace=True)
 del content_df
gc.collect()
temp =(
 data.groupby("user_id")['answered_correctly'] 
.transform(lambda x: shifted_expanding_mean(x.values)) 
)
 temp =(2 * temp * data['c_mean'].values)/(temp + data['c_mean'].values)
(temp.astype('float32' ).rename("h_mean")
.reset_index().to_feather("h_mean.feather"))
del temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating h_mean!")
 data['answered_wrongly'] = data['answered_correctly'] == 0 
(data.groupby("user_id")['answered_wrongly'] 
.transform(lambda x: shifted_expanding_sum(x.values)) 
.fillna(0 ).astype("uint16" ).rename("uwrong_sum")
.reset_index().to_feather("uwrong_sum.feather"))
data.drop(['answered_wrongly'], axis=1, inplace=True)
 gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating uwrong_sum!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Entering Data Prep Bottleneck 2!")
 uph_mean = pd.Series(data=np.zeros(len(data), dtype='float32'), index=data.index, name='uph_mean')
 for i in range(1, 8):
temp =(data[data.part == i].groupby("user_id")['answered_correctly'] 
.transform(lambda x: shifted_expanding_mean(x.values)))
 temp = temp.reindex(data.index ).fillna(0 ).values.astype('float32')
 uph_mean = uph_mean + temp
del temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with uph_mean_{i}!")
 uph_mean =(2 * uph_mean * data['c_mean'].values)/(uph_mean + data['c_mean'].values)
 uph_mean.reset_index().to_feather("uph_mean.feather")
 del uph_mean
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Done creating uph_mean!")
 data['answered_correctly'] = modify_ac(data['answered_correctly'].values, data['c_mean'].values)
 data.drop(['c_mean'], 1, inplace=True)
 gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Modified labels according to Importance!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Entering Data Prep Bottleneck 3!")
 up_mean = pd.Series(data=np.zeros(len(data), dtype='float32'), index=data.index, name='up_mean')
 for i in range(1, 8):
temp =(data[data.part == i].groupby("user_id")['answered_correctly'] 
.transform(lambda x: shifted_expanding_mean(x.values)))
 temp = temp.reindex(data.index ).fillna(0 ).values.astype('float32')
 up_mean = up_mean + temp
del temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with up_mean_{i}!")
 up_mean.reset_index().to_feather("up_mean.feather")
 del up_mean
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Done creating up_mean!")
 data['up_count'] = data.groupby(['user_id', 'part'] ).cumcount().fillna(0)
 data['up_count'] = data['up_count'] / data.groupby('user_id' ).cumcount()
data['up_count'].astype('float32' ).reset_index().to_feather("up_count.feather")
 data.drop(['part'], axis=1, inplace=True)
 gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating up_count!")
(data.groupby(["user_id"])['answered_correctly'] 
.transform(lambda x: moving_average(x.values, n=ROLL_WINDOW)) 
.rename(f'ummr_{ROLL_WINDOW}' ).astype('float32' ).reset_index() 
.to_feather(f'ummr_{ROLL_WINDOW}.feather'))
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating ummr_{ROLL_WINDOW}!")
(data.groupby("user_id")['answered_correctly'] 
.transform(lambda x: shifted_expanding_mean(x.values)) 
.rename("u_mean" ).astype('float32' ).reset_index() 
.to_feather("u_mean.feather"))
del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating u_mean!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_type_id'])
 data =(data['content_type_id'].replace(False, np.nan ).groupby(data['user_id'])
.fillna(method='ffill', limit=LEC_RECENT_ROLL ).fillna(False ).astype(bool))
data.loc[q_mask].rename("lec_recent" ).reset_index().to_feather("lec_recent.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating lec_recent!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_id'])
 data = data.loc[q_mask].groupby(["user_id", 'content_id'] ).cumcount()
data.astype(bool ).rename("repeat_c" ).reset_index().to_feather("repeat_c.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating repeat_c!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'prior_question_elapsed_time', 'prior_question_had_explanation'])
 data = data.loc[q_mask]
data['prior_question_elapsed_time'] =(
 data.groupby("user_id")['prior_question_elapsed_time'].transform(
 lambda x: shifted_expanding_mean(x.values[1:], keep_last=True)))
 data['prior_question_had_explanation'] = data['prior_question_had_explanation'].fillna(False ).astype('float32')
 data['prior_question_had_explanation'] = data.groupby("user_id")['prior_question_had_explanation'].transform(
 lambda x: expanding_mean(x.values))
data = data.rename({"prior_question_had_explanation": "seen_ratio",
'prior_question_elapsed_time': "pqet_mean"}, axis=1)
 data.drop(["user_id"], axis=1, inplace=True)
 data.astype('float32' ).reset_index().to_feather("pq_se.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating pqet_mean and seen_ratio!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'timestamp', 'task_container_id'])
 data = data.groupby("user_id" ).cumcount() /(data['timestamp'] / 60000)
 data.fillna(0, inplace=True)
 data.loc[q_mask].astype("float32" ).rename('uq_per_hr' ).reset_index().to_feather("uq_per_hr.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating uq_per_hr!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Beginning intermediate file concat!")
 data = pd.DataFrame()
for name in ['up_mean.feather', "uph_mean.feather", f"ummr_{ROLL_WINDOW}.feather"]:
temp = pd.read_feather(name ).set_index("index")
 ! rm./{name}
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 data.reset_index().to_feather("./concat_intermediate_1.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with Intermediate Concat 1!")
 data = pd.DataFrame()
for name in ["repeat_c.feather", "uf_bundle.feather", "uwrong_sum.feather",
"seen_exp.feather", "u_mean.feather"]:
temp = pd.read_feather(name ).set_index("index")
 ! rm./{name}
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 data.reset_index().to_feather("./concat_intermediate_2.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with Intermediate Concat 2!")
 data = pd.DataFrame()
for name in ["pq_se.feather", "pqetr.feather"]:
temp = pd.read_feather(name ).set_index("index")
 ! rm./{name}
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 data.reset_index().to_feather("./concat_intermediate_3.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with Intermediate Concat 3!")
 data = pd.DataFrame()
for name in ["uq_per_hr.feather", "up_count.feather", "lec_recent.feather", "h_mean.feather"]:
temp = pd.read_feather(name ).set_index("index")
 ! rm./{name}
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 data.reset_index().to_feather("./concat_intermediate_4.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with Intermediate Concat 4!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Intermediate file concat Complete!")
 data =(pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['content_id'] ).loc[q_mask])
 content_df = pd.read_csv(".. /input/riiid-final-model-inputs/content-df.csv", index_col=0)
 content_df[cat_cols + ['bundle_q_count', 'part']] = content_df[cat_cols + ['bundle_q_count', 'part']].astype('uint8')
 content_df[['wrong_et_med', 'right_et_med']] = content_df[['wrong_et_med', 'right_et_med']].astype('float32')
 content_df[['c_mean', 'c_std']] = content_df[['c_mean', 'c_std']].astype('float32')
 content_df[['content_c', 'tmed', 'che_sum']] =(
 content_df[['content_c', 'tmed', 'che_sum']] ).astype('uint16')
 content_df.drop(['part', 'bundle_id'], axis=1, inplace=True)
 data = data.merge(content_df, how='left', left_on='content_id', right_index=True)
 data.drop(['content_id'], axis=1, inplace=True)
 data.reset_index().to_feather("c_df.feather")
 del data, content_df
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating content_df for merge!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['timestamp', 'user_id'])
 data['response_time'] =(
 data.groupby("user_id")['timestamp'] 
.transform(lambda x: rt_func(x.values)))
 data.drop(['timestamp'], axis=1, inplace=True)
 gc.collect()
data['sessions'] =(data['response_time'] > SESSION_DURATION ).astype(bool)
 data['sessions'] = data.groupby('user_id')['sessions'].cumsum()
data['sess_event_count'] = data.groupby(['user_id', 'sessions'] ).cumcount()
data[['sessions', 'sess_event_count']] = data[['sessions', 'sess_event_count']].astype('uint16')
 data = data.loc[q_mask]
temp = pd.read_feather("./c_df.feather", columns=['index', 'bundle_q_count'])
 temp = temp.set_index("index")
 data['rt_per_task'] = data['response_time'] / temp['bundle_q_count'].values
data.drop(['user_id'], axis=1, inplace=True)
 data.reset_index().to_feather("response_time.feather")
 del data, temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating reponse_time!")
 print("
Done creating all required intermediate files.
 ")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <== Starting merge operation!")
 for i in range(CHUNKS):
print(f"
Time Elapsed: {time.time() - start_time:10.2f} s | <= Beginning chunk {i + 1} merge!")
 front, rear = i * BATCH_SIZE,(i + 1)* BATCH_SIZE
data =(pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['timestamp', 'user_id', 'task_container_id',
'prior_question_elapsed_time', 'answered_correctly'])
.loc[q_mask] 
.iloc[front: rear])
 for name in ['c_df.feather', 'ts_up_recency.feather', "response_time.feather",
"concat_intermediate_2.feather", "concat_intermediate_4.feather",
"concat_intermediate_1.feather", "concat_intermediate_3.feather"]:
temp = pd.read_feather(name)
 temp.iloc[BATCH_SIZE:].reset_index(drop=True ).to_feather(name)
 temp = temp.iloc[:BATCH_SIZE].set_index("index")
 if name == 'c_df.feather':
temp.drop("bundle_q_count", axis=1, inplace=True)
 gc.collect()
time.sleep(3)
 assert len(data)== len(temp)== BATCH_SIZE
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done merging {name}!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Saving chunk {i + 1} onto Disk!")
 data.reset_index(drop=True, inplace=True)
 data.to_feather(f"model_train_c{i + 1}.feather")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done saving chunk {i + 1} to Disk!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Done Merging chunk {i + 1}!")
 del data
gc.collect()
time.sleep(3)
 print(f"
Time Elapsed: {time.time() - start_time:10.2f} s | <== Merge Complete!")
 print("Deleting unneeded intermediate files.
 ")
 ! rm c_df.feather concat_intermediate_*.feather ts_up_recency.feather
! rm response_time.feather
else:
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Found PreSaved File(s ).Load it manually if need be!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Copying Train File(s)back to Disk!")
 for i in range(CHUNKS):
! cp.. /input/riiid-final-model-inputs/model_train_c{i + 1}.feather./model_train_c{i + 1}.feather
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done copying Train File(s)back to Disk!" )<define_variables>
test_data['Predictions'] = predictions
def return_random_slice(batch_size, nrows):

 front = np.random.choice(nrows - batch_size)
 rear = front + batch_size
return front, rear<define_variables>
kaggle_data = test_data[['PassengerId', 'Predictions']].copy()
kaggle_data.rename(columns={'Predictions' : 'Survived'}, inplace=True)
 kaggle_data.sort_values(by=['PassengerId'] ).to_csv('kaggle_out_svc_names.csv', index=False )