Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<save_to_csv>
!pip --quiet install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl
!pip install --quiet -r.. /input/treelite-treelite-runtime-version-093/treelite/requirements.txt --no-index --find-links.. /input/treelite-treelite-runtime-version-093/treelite
tqdm.tqdm.pandas()
%matplotlib inline
env = riiideducation.make_env()
ans.to_csv("Predictions by CNN.csv", index=None )<set_options>
dtypes = {
"row_id": "int64",
"timestamp": "int64",
"user_id": "int32",
"content_id": "int16",
"content_type_id": "boolean",
"task_container_id": "int16",
"user_answer": "int8",
"answered_correctly": "int8",
"prior_question_elapsed_time": "float32",
"prior_question_had_explanation": "boolean"
}
data = pd.read_csv(".. /input/riiid-test-answer-prediction/train.csv", dtype=dtypes, nrows=int(1e6))
ques = pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv')
 lectures = pd.read_csv(".. /input/riiid-test-answer-prediction/lectures.csv")
 ex_sub = pd.read_csv('.. /input/riiid-test-answer-prediction/example_sample_submission.csv')
 ex_test = pd.read_csv('.. /input/riiid-test-answer-prediction/example_test.csv')
 data.shape, ques.shape, lectures.shape, ex_sub.shape, ex_test.shape
%matplotlib inline 
<set_options>
ql = pd.concat([ques, lectures.rename({"lecture_id": "question_id"}, axis=1)], axis=0 ).reset_index(drop=True)
 ql.tags = ql.tags.fillna(ql.tag)
 ql.type_of = ql.type_of.fillna("question")
 ql["content_type_id"] = ql["type_of"] != 'question'
ql = ql.fillna(-1)
 ql = ql.drop("tag", 1)
 ql = ql.rename({"question_id": "content_id"}, axis=1)
 ql.tags = ql.tags.apply(lambda x: [int(x)] if type(x)!= str else list(map(int, x.split())))
 ql['listening_lvl'] = ql.part.map({1: 2, 2: 2, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0})
 ql['multiple_q'] = ql.part.isin([3, 4, 6, 7])
 ql['tag_c'] = ql.tags.apply(len )
warnings.filterwarnings('ignore' )<load_from_csv>
lec_count = ql.loc[ql.content_type_id, 'tags'].transform(lambda x: x[0] ).value_counts()
ql['lec_available'] =(
 ql.loc[~ql.content_type_id, 'tags'].transform(
 lambda x: sum([lec_count.at[i] if i in lec_count.index else 0 for i in x]))
)
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<feature_engineering>
ql['bundle_q_count'] = ql.groupby("bundle_id")['content_id'].transform('count')
 ql.loc[ql.content_type_id, 'bundle_q_count'] = -1
train['SalePriceLog'] = np.log1p(train['SalePrice'] )<filter>
te = TransactionEncoder()
temp = ql[~ql.content_type_id]
temp = temp.merge(
 data[~data.content_type_id].groupby("content_id")['answered_correctly'].agg(['count', 'mean']),
on='content_id', how='left')
 temp['mean'] = temp['mean'].fillna(0.5)
 temp['count'] = temp['count'].fillna(0)
 temp = np.hstack([
te.fit_transform(temp.tags.to_list() ).astype(int),
pd.get_dummies(temp[['tag_c', 'part', 'count', 'mean']], columns=['part', 'tag_c'])
 ])
 temp = PCA(n_components=150 ).fit_transform(StandardScaler().fit_transform(temp))
ql.loc[~ql.content_type_id, 'cluster'] = KMeans(n_clusters=30 ).fit_predict(temp)
 ql.loc[ql.content_type_id, 'cluster'] = -1
ql.cluster = ql.cluster.astype(int)
 ql.sample(5 )
train = train[train['GrLivArea'] < 4000]<sort_values>
temp = ql.tags.progress_apply(pd.Series)
 ql['tagF'] = temp[0]
ql['tagS'] = temp[1]
ql['tagT'] = temp[2]
ql['tagL'] = ql.tags.apply(lambda x: x[-1])
 ql[['tagF', 'tagS', 'tagL', 'tagT']] =(ql[['tagF', 'tagS', 'tagL', 'tagT']] + 1 ).fillna(0)
 ql.sample(5 )
quantitative = [f for f in train.columns if train.dtypes[f] != 'object']
quantitative.remove('SalePrice')
 quantitative.remove('Id')
 quantitative.sort()
qualitative = [f for f in train.columns if train.dtypes[f] == 'object']
qualitative.sort()<prepare_x_and_y>
data = data.drop(['part', 'bundle_id'], 1 ).merge(ql, on=['content_id', 'content_type_id'], how='left')
 data.shape
train.reset_index(drop=True, inplace=True)
 y_train = train['SalePriceLog']
X_train = train.drop(['SalePrice', 'SalePriceLog'], axis=1)
 X_test = test<concatenate>
data = data.sort_values(by=['user_id', 'timestamp'])
 data['response_time'] =(
 data.groupby("user_id")['timestamp'] 
.transform(lambda x: x.diff().replace(0, np.nan)
.fillna(method='ffill' ).fillna(0))
)
all_data = pd.concat([X_train, test], axis=0, sort=False)
 all_data.drop(['Id'], axis=1, inplace=True)
 all_data.shape<sort_values>
data['res_time_avg'] =(
 data.timestamp - 
(data.timestamp *(data.task_container_id - 1)/ data.task_container_id)
)
 data['res_time_avg'] = data['res_time_avg'].replace(np.inf, np.nan)
 print("Corelation to response_time:
",
data.corr() ['response_time'].loc[['res_time_avg']], sep='')
 print("
Correlation to ans_correctness:
",
data.corr() ['answered_correctly'].loc[['response_time', 'res_time_avg']], sep='')
(data[['timestamp', 'response_time', 'res_time_avg']] 
.sample(10 ).fillna(0 ).astype(int))
na_total = all_data.isnull().sum().sort_values(ascending=False)
 na_ratio =(all_data.isnull().sum() / all_data.shape[0] ).sort_values(ascending=False)
 missing_data = pd.concat([na_total, na_ratio], axis=1, keys=['Total', 'Ratio'])
 missing_data.head(50 )<data_type_conversions>
data = data.sort_values(['user_id', 'timestamp'])
 data = data.merge(
(data[~data.content_type_id].groupby(['user_id', 'task_container_id'])
 [['prior_question_elapsed_time', 'prior_question_had_explanation']] 
.mean().groupby("user_id" ).shift(-1 ).reset_index() 
.rename({"prior_question_elapsed_time": 'pqet_shifted',
'prior_question_had_explanation': "pqhe_shifted"}, axis=1)) ,
on=['user_id', 'task_container_id'], how='left' )
all_data['Alley'].fillna('None', inplace=True)
 all_data['Fence'].fillna('None', inplace=True)
 all_data['FireplaceQu'].fillna('None', inplace=True)
 all_data['GarageQual'].fillna('None', inplace=True)
 all_data['GarageFinish'].fillna('None', inplace=True)
 all_data['GarageCond'].fillna('None', inplace=True)
 all_data['GarageType'].fillna('None', inplace=True)
 all_data['BsmtExposure'].fillna('None', inplace=True)
 all_data['BsmtCond'].fillna('None', inplace=True)
 all_data['BsmtQual'].fillna('None', inplace=True)
 all_data['BsmtFinType2'].fillna('None', inplace=True)
 all_data['BsmtFinType1'].fillna('None', inplace=True )<feature_engineering>
data = data.sort_values(by=['user_id', 'timestamp'])
 cut_off =(1000 * 60 * 60)
 cut_off = cut_off * 1
data['sessions'] =(
 data.groupby("user_id")['timestamp'].diff() > cut_off
).groupby(data['user_id'] ).cumsum()
all_data['MasVnrType'].fillna('None', inplace=True)
 all_data['HasMasVnr'] = all_data['MasVnrType'].apply(lambda x: 0 if x == 'None' else 1)
 all_data['MSZoning'] = all_data.groupby(['MSSubClass'])['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
all_data['Functional'].fillna('Typ', inplace=True)
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0] )<groupby>
def post_process(fn0, fn1):

 fn_processed = fn0.drop(['prior_group_answers_correct', 'prior_group_responses'], 1)
 fn_processed['answered_correctly'] = eval(fn1['prior_group_answers_correct'].iloc[0])
 fn_processed['user_answer'] = eval(fn1['prior_group_responses'].iloc[0])
 return fn_processed
all_data['LotFrontage'] = all_data.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))<feature_engineering>
with open(".. /input/riiid-final-model-inputs/sample-batches.pkl", 'rb')as f:
batches = pickle.load(f)
 print("Batch sizes for each test sample:", list(map(lambda x: x[0].shape[0], batches)) )
all_data['GarageYrBlt'] =(all_data['YearBuilt'] + all_data['YearRemodAdd'])/2<data_type_conversions>
temp = data.loc[~data.content_type_id, "answered_correctly"]
for value in [0, 1, 0.5, temp.mean() ]:
print("At {:.2f} the score is: {:.3f}".format(value, roc_auc_score(temp, np.full_like(temp, 0))))
print(all_data[all_data['GarageCars'].isnull() ][['GarageArea', 'GarageCars', 'GarageType', 'GarageYrBlt', 'GarageQual']])
 all_data['GarageArea'].fillna(0, inplace=True)
 all_data['GarageCars'].fillna(0, inplace=True )<data_type_conversions>
roc_auc_score(temp, np.random.rand(len(temp)) )
print(all_data[all_data['TotalBsmtSF'].isnull() ][
['TotalBsmtSF', 'BsmtQual', 'BsmtCond', 'BsmtFinSF2', 'BsmtUnfSF', 'BsmtFinSF1', 'BsmtFullBath','BsmtHalfBath']])
 all_data['TotalBsmtSF'].fillna(0, inplace=True)
 all_data['BsmtUnfSF'].fillna(0, inplace=True)
 all_data['BsmtFinSF1'].fillna(0, inplace=True)
 all_data['BsmtFinSF2'].fillna(0, inplace=True)
 all_data['BsmtFullBath'].fillna(0, inplace=True)
 all_data['BsmtHalfBath'].fillna(0, inplace=True )<feature_engineering>
def generate_train_val(train, train_users=None, tp=.70, vp=None, put=None, dyn_p=False, return_mask=False):

 def threshold_user(arr, put=None):
if put is None:
put = np.random.choice(np.linspace (.20,.80, 13))
return arr < np.quantile(arr, put)
 total_users = train.user_id.unique()
if train_users is None:
train_users = np.random.choice(total_users, int(len(total_users)* tp), replace=False)
 if vp is None:
vp = np.random.choice(np.linspace (.20,.80, 13))
if put is None:
if not dyn_p:
put = vp
pp = 1 - vp
remaining_users = np.setdiff1d(total_users, train_users)
 val_users = np.random.choice(remaining_users, int(len(remaining_users)* vp), replace=False)
 partial_users = np.setdiff1d(remaining_users, val_users)
 mask =(
 train[train.user_id.isin(partial_users)] 
.groupby("user_id")['timestamp'] 
.transform(lambda x: threshold_user(x.values, put)) 
.reindex(train.index, fill_value=False)|
train.user_id.isin(train_users))
if return_mask:
return mask
else:
Train = train.copy()
Train['train'] = mask
return Train
all_data['YrBltAndRemod']=all_data['YearBuilt']+all_data['YearRemodAdd']
all_data['TotalSF']=all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']
all_data['TotalSqrFootage'] =(all_data['BsmtFinSF1'] + all_data['BsmtFinSF2'] +
all_data['1stFlrSF'] + all_data['2ndFlrSF'])
 all_data['TotalBathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+
all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['TotalPorchSF'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] +
all_data['EnclosedPorch'] + all_data['ScreenPorch'] +
all_data['WoodDeckSF'] )<feature_engineering>
ba = bitarray(15000, endian='little')
 ba.setall(False)
 repeat_c = 0
temp = data[~data.content_type_id].loc[data.user_id == np.random.choice(data.user_id.unique()), ['content_id']]
for _, c in temp['content_id'].iteritems() :
if ba[c]:
print(f"{c:<5} was already viewed by the user!")
 repeat_c += 1
else:
ba[c] = 1
if repeat_c == 0:
print("No repeated questions for this user")
 else:
print(f"
{repeat_c} question(s)were repeated")
 np.array(ba.tolist())[temp.content_id.values].all()
all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<data_type_conversions>
@nb.njit
def shifted_expanding_mean(arr, keep_last=False):
'keep_last is set to true when we apply this func to pqet_mean.SPECIAL USE CASE!'
temp = expanding_mean(arr)
 if not keep_last:
return np.concatenate(( np.array([np.nan]), temp[:-1]))
else:
return np.concatenate(( np.array([np.nan]), temp))
shifted_expanding_mean(temp)
 %timeit -n 20 shifted_expanding_mean(temp)
 %timeit -n 20 pd.Series(temp ).expanding(1 ).mean().shift()
np.testing.assert_allclose(pd.Series(temp ).expanding(1 ).mean().shift() , shifted_expanding_mean(temp))
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )<categorify>
@nb.njit
def rt_func(arr, pred=False):
temp = np.concatenate(( np.array([np.nan]), arr[1:] - arr[:-1]))
temp = np.where(temp == 0, np.nan, temp)
 mask = np.isnan(temp)
 idx = np.arange(len(mask))
idx = np.where(mask, 0, idx)
 rmax, i = idx[0], 0
for i, val in enumerate(idx):
if val > rmax:
rmax = val
idx[i] = rmax
temp[mask] = temp[idx[mask]]
if not pred:
temp = np.where(np.isnan(temp), 0, temp)
 else:
temp = np.where(np.isnan(temp), arr, temp)
 return temp
all_data = pd.get_dummies(all_data ).reset_index(drop=True )<sort_values>
%time data.groupby("user_id")['timestamp'].transform(lambda x: rt_func(x.values))
%time data.groupby("user_id")['timestamp'].transform(lambda x: x.diff().replace(0, np.nan ).fillna(method='ffill' ).fillna(0)) 
(
 data.groupby("user_id")['timestamp'].transform(lambda x: rt_func(x.values))
== data['response_time']
).all()
all_data.isnull().sum().sort_values(ascending=False )<import_modules>
@nb.njit
def modify_ac(arr, c_mean):
temp = np.where(arr == 0, -1, 1)* c_mean
return np.where(temp < 0, temp, 1 - temp )
from sklearn.model_selection import cross_val_score, KFold
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import make_pipeline
from sklearn.linear_model import ElasticNet, Lasso, Ridge, ElasticNetCV, LassoCV, RidgeCV
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import GradientBoostingRegressor, RandomForestRegressor
from sklearn.svm import SVR
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from mlxtend.regressor import StackingCVRegressor<split>
%%time
data['ac_modified'] = modify_ac(
 data['answered_correctly'].values,
data.groupby(["content_id", 'content_type_id'])['answered_correctly'].transform('mean' ).values)
 data.loc[data.content_type_id, 'ac_modified'] = 0
X_train = all_data.iloc[:len(y_train), :]
X_test = all_data.iloc[len(y_train):, :]<compute_train_metric>
@nb.njit
def fillnshift(arr):

 mask = np.isnan(arr)
 idx = np.arange(len(mask))
idx = np.where(mask, 0, idx)
 rmax, i = idx[0], 0
for i, val in enumerate(idx):
if val > rmax:
rmax = val
idx[i] = rmax
arr[mask] = arr[idx[mask]]
return np.concatenate(( np.array([np.nan]), arr[:-1]))
def rmse_cv(model):
mse = cross_val_score(model, X_train.values, y_train, scoring="neg_mean_squared_error", cv=5)
 rmse = np.sqrt(-mse)
 print(f'{model.__class__.__name__} score: {rmse.mean() :.4f}, {rmse.std() :.4f}')
<train_on_grid>
@nb.njit
def moving_average(arr, n=10, shift=True):
mask = np.isnan(arr)
 ret = np.cumsum(np.where(mask, 0, arr))
ret[n:] = ret[n:] - ret[:-n]
counts = np.cumsum(~mask)
 counts[n:] = counts[n:] - counts[:-n]
ret[~mask] /= counts[~mask]
ret[mask] = np.nan
if shift:
ret = np.concatenate(( np.array([np.nan]), ret[:-1]))
return ret
lasso = Lasso()
lasso_search = GridSearchCV(lasso, {'alpha': np.logspace(-4, -3, 5)}, cv=5, scoring="neg_mean_squared_error")
 lasso_search.fit(X_train, y_train)
 lasso_search.best_estimator_<choose_model_class>
data['up_mean'] = data[~data.content_type_id].groupby(['user_id', 'part'])['ac_modified'].transform(
 lambda x: shifted_expanding_mean(x.values))
data['up_count'] =(data[~data.content_type_id].groupby(['user_id', 'part'] ).cumcount()
/ data[~data.content_type_id].groupby("user_id" ).cumcount())
 data['up_count'] = data['up_count'].fillna(0 )
lasso_model = make_pipeline(RobustScaler() , Lasso(alpha=0.0005, random_state=1))
rmse_cv(lasso_model )<train_on_grid>
temp = np.random.randint(0, 2, size=int(1e5)).astype('float')
 temp[0] = np.nan
temp = pd.Series(temp)
 np.testing.assert_allclose(
 temp.expanding().mean() ,
shifted_expanding_mean(temp.values[1:], keep_last=True)
 )
ridge = Ridge()
ridge_search = GridSearchCV(ridge, {'alpha': np.linspace(10, 30, 10)}, cv=5, scoring="neg_mean_squared_error")
 ridge_search.fit(X_train, y_train)
 ridge_search.best_estimator_<choose_model_class>
data['content_c'] = data.groupby(["content_id", 'content_type_id'])['row_id'].transform("count")
 data['seen_ratio'] =(data.loc[~data.content_type_id, ['user_id', 'prior_question_had_explanation']] 
.fillna(False ).astype(float ).groupby("user_id")
.transform(lambda x: expanding_mean(x.values)))
 data['pqet_mean'] =(
 data[(~data.content_type_id)].groupby("user_id")['prior_question_elapsed_time'] 
.transform(lambda x: shifted_expanding_mean(x.values[1:], keep_last=True)) 
)
 data.loc[~data.content_type_id, 'repeat_c'] =(
 data[~data.content_type_id].groupby(["user_id", 'content_id'])
.cumcount().astype(bool))
data['repeat_c'] = data['repeat_c'].fillna(False)
 data['uq_per_hr'] = data.groupby("user_id" ).cumcount() /(data['timestamp'] / 60000)
 data['tmed'] = data[~data.content_type_id].groupby("content_id")[['task_container_id']].transform('median')
 data['uwrong_sum'] =(
 data[~data.content_type_id]['answered_correctly'] 
.map({0: 1, 1: 0} ).groupby(data['user_id'])
.transform(lambda x: shifted_expanding_sum(x.values))
)
ridge_model = make_pipeline(RobustScaler() , Ridge(alpha=19))
rmse_cv(ridge_model )<train_on_grid>
data['lec_recent'] =(
 data.loc[data.content_type_id, 'content_type_id'] 
.reindex(data.index ).groupby(data['user_id'])
.fillna(method='ffill', limit=10)
.fillna(False ).astype(bool)
)
 data['uf_bundle'] = data.groupby("user_id")['bundle_id'].transform('first')
 data['pqetmr_10'] =(data[~data.content_type_id].groupby("user_id")['prior_question_elapsed_time'].transform(
 lambda x: moving_average(x.values.astype('float32'), shift=False, n=10)))
 data['che_sum'] = data[~data.content_type_id].groupby("content_id")['pqhe_shifted'].transform('sum')
 data['che_sum'] = data['che_sum'].fillna(0 ).astype('int' )
enet = ElasticNet()
enet_search = GridSearchCV(enet, {'alpha': np.linspace(0.0001, 0.001, 10), 'l1_ratio':np.linspace(0.5, 1.5, 10)}, cv=5, scoring="neg_mean_squared_error")
 enet_search.fit(X_train, y_train)
 enet_search.best_estimator_<choose_model_class>
data['seen_exp_when_wrong'] =(data['pqhe_shifted'].fillna(False)&(data['answered_correctly'] == 0)).astype(int)
 data['seen_exp_when_wrong'] =(data[~data.content_type_id].groupby('user_id')['seen_exp_when_wrong'] 
.transform(lambda x: shifted_expanding_sum(x.values)))
 data['seen_exp_when_right'] =(data['pqhe_shifted'].fillna(False)&(data['answered_correctly'] == 1)).astype(int)
 data['seen_exp_when_right'] =(data[~data.content_type_id].groupby('user_id')['seen_exp_when_right'] 
.transform(lambda x: shifted_expanding_sum(x.values)))
 data['sessions'] =(data['response_time'] >(15*60*1000)).groupby(data['user_id'] ).cumsum()
data['sess_event_count'] = data.groupby(['user_id', 'sessions'] ).cumcount()
data['up_recency'] = data.groupby(['user_id', 'part'])['timestamp'].transform(lambda x: rt_func(x.values))
data['ts_recency_10'] = data['timestamp'] - data.groupby("user_id")['timestamp'].shift(10)
 data['ts_recency_5'] = data['timestamp'] - data.groupby("user_id")['timestamp'].shift(5 )
enet_model = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0004, l1_ratio=1.4, random_state=3))
rmse_cv(enet_model )<choose_model_class>
warnings.filterwarnings("ignore", category=UserWarning)
 train_cols = [
'repeat_c',
'tagF', 'tagS', 'tagL', 'tagT',
'response_time',
'prior_question_elapsed_time',
'up_mean', 'up_count', 'uq_per_hr',
'uwrong_sum', 'lec_recent',
'pqet_mean', 'seen_ratio', 'tmed',
'up_recency', 'ts_recency_10',
'ts_recency_5',
'timestamp', 'task_container_id', 'content_c',
'che_sum', 'seen_exp_when_wrong',
'seen_exp_when_right',
'sessions','sess_event_count',
'pqetmr_10', 'uf_bundle',
]
cat_cols = ['tagF', 'tagS', 'tagT', 'tagL']
other_cols = ['train', 'user_id', 'content_id', 'ac_modified',
'pqet_shifted', 'bundle_q_count', 'part']
folds = 7
best_params = {
'learning_rate':0.075,
'num_leaves': 200,
'objective': 'binary',
'metric': 'auc'}
total_users = data.user_id.unique()
np.random.shuffle(total_users)
 val_size = len(total_users)// folds
scores = {'TRAIN': [], 'VAL': []}
models = []
for i in range(folds):
start_time = time.time()
train_users = np.setdiff1d(total_users, total_users[(i)*val_size:(i+1)*val_size])
 temp = generate_train_val(data, train_users, dyn_p=True)
 temp = temp[~temp.content_type_id]
temp = temp.loc[~temp.content_type_id, train_cols + other_cols + ['answered_correctly']]
temp[['u_mean']] =(
 temp.groupby(["user_id"])['ac_modified'] 
.transform(lambda x: shifted_expanding_mean(x.values)) 
)
 temp['ummr_10'] =(
 temp.groupby('user_id')['ac_modified'] 
.transform(lambda x: moving_average(x.values, n=10)) 
)
 temp['rt_per_task'] = temp['response_time'] / temp['bundle_q_count']
temp[cat_cols] = temp[cat_cols].fillna(0)
 temp[cat_cols] = temp[cat_cols].astype(int)
 temp = temp.merge(
 temp[temp.train].groupby("content_id")['answered_correctly'].agg(c_mean='mean', c_std='std'),
left_on=['content_id'], right_index=True, how='left')
 temp['c_mean'] = temp['c_mean'].fillna(0.5)
 temp['h_mean'] = temp.groupby("user_id")['answered_correctly'].transform(
 lambda x: shifted_expanding_mean(x.values)
)
 temp['h_mean'] =(2 * temp['h_mean'] * temp['c_mean'])/(temp['h_mean'] + temp['c_mean'])
 temp['uph_mean'] = temp.groupby(['user_id', 'part'])['answered_correctly'].transform(
 lambda x: shifted_expanding_mean(x.values))
temp['uph_mean'] =(2 * temp['uph_mean'] * temp['c_mean'])/(temp['uph_mean'] + temp['c_mean'])
 temp = temp.merge(
 temp[temp.train].groupby(["content_id", 'answered_correctly'])['pqet_shifted'].agg('median' ).unstack() ,
left_on='content_id', right_index=True, how='left')
 temp.columns = temp.columns[:-2].tolist() + ['wrong_et_med', 'right_et_med']
train, val = temp[temp['train']], temp[~temp['train']]
train, val = train.drop(other_cols, 1), val.drop(other_cols, 1)
 if i == 0:
if not best_params:
train_lgb = lgb.Dataset(
 train.drop('answered_correctly', 1), train['answered_correctly'],
free_raw_data=False, categorical_feature=cat_cols)
 val_lgb = lgb.Dataset(
 val.drop('answered_correctly', 1), val['answered_correctly'],
free_raw_data=False, categorical_feature=cat_cols)
 print("Searching for Optimal Hyperparameters! This may take some time to complete.")
 model = lgbo.train(
 params={'objective':'binary', 'metric':'auc'},
train_set=train_lgb, valid_sets=[train_lgb, val_lgb],
verbose_eval=0, early_stopping_rounds=10,
show_progress_bar=False,
time_budget=60*60, 
)
 params = model.params
else:
params = best_params
print(f"Starting folds: {'='*80}>")
 d_time = time.time() - start_time
model = lgb.LGBMClassifier(**params)
 model.fit(train.drop('answered_correctly', 1), train['answered_correctly'],
eval_set=[(val.drop('answered_correctly', 1), val['answered_correctly'])],
early_stopping_rounds=10, verbose=0)
 model = model.booster_
models.append(model)
 scores['TRAIN'] = scores.get('TRAIN')+ [roc_auc_score(train['answered_correctly'], model.predict(
 train.drop("answered_correctly", 1), categorical_feature=cat_cols)) ]
scores['VAL'] = scores.get('VAL')+ [roc_auc_score(val['answered_correctly'], model.predict(
 val.drop("answered_correctly", 1), categorical_feature=cat_cols)) ]
print("AUC for Fold 
.format(i+1, folds, scores['TRAIN'][-1], scores['VAL'][-1], d_time, time.time() - start_time))
print("Summary stats : {}
Mean AUC for {} folds: T: {:.4f} @ {:.4f} std | V: {:.4f} @ {:.4f} std" 
.format('='*76+">", folds, np.mean(scores['TRAIN']), np.std(scores['TRAIN']),
np.mean(scores['VAL']), np.std(scores['VAL'])))
 print(f"
Additional Info: {'='*80+'>'}
Train Features Used By Model: {val.shape[1] - 1} Features")
 print(f"Hypothetical Space Required For Entire Data: {(val.shape[1] * 101230332 * 32)/ 8e+9:.2f} GB(float 32)")
 print(f"Hypothetical Space Required For Entire Data: {(val.shape[1] * 101230332 * 64)/ 8e+9:.2f} GB(float 64)" )
gbdt_model = GradientBoostingRegressor(learning_rate=0.05, min_samples_leaf=5, min_samples_split=10, max_depth=4, n_estimators=3000)
 rmse_cv(gbdt_model )<choose_model_class>
del data, temp, train, val
data = temp = train = val = None
gc.collect()
rf_model = RandomForestRegressor(min_samples_leaf=4, min_samples_split=8)
 rmse_cv(rf_model )<define_search_model>
start_time = time.time()
LEC_RECENT_ROLL = 10
ROLL_WINDOW = 10
ROLL_WINDOW_PQET = 10
SESSION_DURATION = 15 * 60 * 1000
CHUNKS = 3
SAVE_LOC = f".. /input/riiid-final-model-inputs/model_train_c1.feather"
MODEL_LOC = f".. /input/riiid-final-model-inputs/trained_model.txt"
if not os.path.exists(SAVE_LOC):
data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'timestamp', 'answered_correctly'])
 q_mask = data.answered_correctly != -1
BATCH_SIZE = q_mask.sum() // CHUNKS
del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s", end=' ')
 print("| Percentage of Data used: {:.2f}% | Single chunk Size: {:9} rows".format(
 q_mask.sum() *100/len(q_mask), BATCH_SIZE 
))
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Entering Data Prep Bottleneck 1!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'timestamp', 'content_id', 'content_type_id'])
 data = data.merge(
 ql[['content_id', 'content_type_id', 'part']],
on=['content_id', 'content_type_id'], how='left')
 data.drop(['content_id', 'content_type_id'], axis=1, inplace=True)
 data['part'] = data['part'].astype('uint8')
 gc.collect()
up_recency = pd.Series(data=np.zeros(len(data), dtype='float64'), index=data.index, name='up_recency')
 for i in range(1, 8):
temp =(data[data.part == i].groupby("user_id")['timestamp'] 
.transform(lambda x: rt_func(x.values)))
 temp = temp.reindex(data.index ).fillna(0 ).values.astype('float64')
 up_recency = up_recency + temp
del temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with up_recency_{i}!")
 data['up_recency'] = up_recency
data.drop('part', axis=1, inplace=True)
 del up_recency
gc.collect()
data['ts_recency_10'] = data['timestamp'] - data.groupby('user_id')['timestamp'].shift(10)
 data['ts_recency_5'] = data['timestamp'] - data.groupby('user_id')['timestamp'].shift(5)
 data = data.loc[q_mask]
data.drop(['user_id', 'timestamp'], axis=1, inplace=True)
 data.reset_index().to_feather("ts_up_recency.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Done creating up/ts recency features!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'answered_correctly'] ).loc[q_mask]
pq_shifted = pd.read_feather(".. /input/riiid-final-model-inputs/pq_shifted.feather",
columns=['index', 'pqhe_shifted'] ).set_index("index")
 data['seen_exp_when_wrong'] =(pq_shifted['pqhe_shifted'] &(data['answered_correctly'] == 0)).astype(int)
 data['seen_exp_when_right'] =(pq_shifted['pqhe_shifted'] &(data['answered_correctly'] == 1)).astype(int)
 del pq_shifted
gc.collect()
data['seen_exp_when_wrong'] =(
 data.groupby('user_id')['seen_exp_when_wrong'] 
.transform(lambda x: shifted_expanding_sum(x.values)))
 data['seen_exp_when_right'] =(
 data.groupby('user_id')['seen_exp_when_right'] 
.transform(lambda x: shifted_expanding_sum(x.values)))
 data = data[['seen_exp_when_wrong', 'seen_exp_when_right']]
data = data.fillna(0 ).astype("uint16")
 data.astype('uint16' ).reset_index().to_feather("seen_exp.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating seen_exp features!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_id'] ).loc[q_mask]
content_df = pd.read_csv(".. /input/riiid-final-model-inputs/content-df.csv", index_col=0,
usecols=['content_id', 'bundle_id'])
 content_df['bundle_id'] = content_df['bundle_id'].astype("uint16")
 data = data.merge(content_df['bundle_id'], left_on='content_id', right_index=True, how='left')
 data.drop(['content_id'], axis=1, inplace=True)
 data = data.groupby("user_id")['bundle_id'].transform("first" ).rename("uf_bundle")
 data.astype("uint16" ).reset_index().to_feather("uf_bundle.feather")
 del data, content_df
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating uf_bundle!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'prior_question_elapsed_time'] ).loc[q_mask]
data['pqetmr_10'] = data.groupby("user_id")['prior_question_elapsed_time'].transform(
 lambda x: moving_average(x.values, n=10, shift=False)
)
 data.drop(['prior_question_elapsed_time', 'user_id'], axis=1, inplace=True)
 data.astype('float32' ).reset_index().to_feather("pqetr.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating pqet rolling features!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_id', 'answered_correctly'] ).loc[q_mask]
content_df = pd.read_csv(".. /input/riiid-final-model-inputs/content-df.csv", index_col=0,
usecols=['part', 'c_mean', 'content_id'])
 content_df['part'] = content_df['part'].astype('uint8')
 content_df['c_mean'] = content_df['c_mean'].astype('float32')
 data = data.merge(content_df, left_on='content_id', right_index=True, how='left')
 data.drop(['content_id'], axis=1, inplace=True)
 del content_df
gc.collect()
temp =(
 data.groupby("user_id")['answered_correctly'] 
.transform(lambda x: shifted_expanding_mean(x.values)) 
)
 temp =(2 * temp * data['c_mean'].values)/(temp + data['c_mean'].values)
(temp.astype('float32' ).rename("h_mean")
.reset_index().to_feather("h_mean.feather"))
del temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating h_mean!")
 data['answered_wrongly'] = data['answered_correctly'] == 0 
(data.groupby("user_id")['answered_wrongly'] 
.transform(lambda x: shifted_expanding_sum(x.values)) 
.fillna(0 ).astype("uint16" ).rename("uwrong_sum")
.reset_index().to_feather("uwrong_sum.feather"))
data.drop(['answered_wrongly'], axis=1, inplace=True)
 gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating uwrong_sum!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Entering Data Prep Bottleneck 2!")
 uph_mean = pd.Series(data=np.zeros(len(data), dtype='float32'), index=data.index, name='uph_mean')
 for i in range(1, 8):
temp =(data[data.part == i].groupby("user_id")['answered_correctly'] 
.transform(lambda x: shifted_expanding_mean(x.values)))
 temp = temp.reindex(data.index ).fillna(0 ).values.astype('float32')
 uph_mean = uph_mean + temp
del temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with uph_mean_{i}!")
 uph_mean =(2 * uph_mean * data['c_mean'].values)/(uph_mean + data['c_mean'].values)
 uph_mean.reset_index().to_feather("uph_mean.feather")
 del uph_mean
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Done creating uph_mean!")
 data['answered_correctly'] = modify_ac(data['answered_correctly'].values, data['c_mean'].values)
 data.drop(['c_mean'], 1, inplace=True)
 gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Modified labels according to Importance!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Entering Data Prep Bottleneck 3!")
 up_mean = pd.Series(data=np.zeros(len(data), dtype='float32'), index=data.index, name='up_mean')
 for i in range(1, 8):
temp =(data[data.part == i].groupby("user_id")['answered_correctly'] 
.transform(lambda x: shifted_expanding_mean(x.values)))
 temp = temp.reindex(data.index ).fillna(0 ).values.astype('float32')
 up_mean = up_mean + temp
del temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with up_mean_{i}!")
 up_mean.reset_index().to_feather("up_mean.feather")
 del up_mean
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Done creating up_mean!")
 data['up_count'] = data.groupby(['user_id', 'part'] ).cumcount().fillna(0)
 data['up_count'] = data['up_count'] / data.groupby('user_id' ).cumcount()
data['up_count'].astype('float32' ).reset_index().to_feather("up_count.feather")
 data.drop(['part'], axis=1, inplace=True)
 gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating up_count!")
(data.groupby(["user_id"])['answered_correctly'] 
.transform(lambda x: moving_average(x.values, n=ROLL_WINDOW)) 
.rename(f'ummr_{ROLL_WINDOW}' ).astype('float32' ).reset_index() 
.to_feather(f'ummr_{ROLL_WINDOW}.feather'))
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating ummr_{ROLL_WINDOW}!")
(data.groupby("user_id")['answered_correctly'] 
.transform(lambda x: shifted_expanding_mean(x.values)) 
.rename("u_mean" ).astype('float32' ).reset_index() 
.to_feather("u_mean.feather"))
del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating u_mean!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_type_id'])
 data =(data['content_type_id'].replace(False, np.nan ).groupby(data['user_id'])
.fillna(method='ffill', limit=LEC_RECENT_ROLL ).fillna(False ).astype(bool))
data.loc[q_mask].rename("lec_recent" ).reset_index().to_feather("lec_recent.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating lec_recent!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_id'])
 data = data.loc[q_mask].groupby(["user_id", 'content_id'] ).cumcount()
data.astype(bool ).rename("repeat_c" ).reset_index().to_feather("repeat_c.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating repeat_c!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'prior_question_elapsed_time', 'prior_question_had_explanation'])
 data = data.loc[q_mask]
data['prior_question_elapsed_time'] =(
 data.groupby("user_id")['prior_question_elapsed_time'].transform(
 lambda x: shifted_expanding_mean(x.values[1:], keep_last=True)))
 data['prior_question_had_explanation'] = data['prior_question_had_explanation'].fillna(False ).astype('float32')
 data['prior_question_had_explanation'] = data.groupby("user_id")['prior_question_had_explanation'].transform(
 lambda x: expanding_mean(x.values))
data = data.rename({"prior_question_had_explanation": "seen_ratio",
'prior_question_elapsed_time': "pqet_mean"}, axis=1)
 data.drop(["user_id"], axis=1, inplace=True)
 data.astype('float32' ).reset_index().to_feather("pq_se.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating pqet_mean and seen_ratio!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'timestamp', 'task_container_id'])
 data = data.groupby("user_id" ).cumcount() /(data['timestamp'] / 60000)
 data.fillna(0, inplace=True)
 data.loc[q_mask].astype("float32" ).rename('uq_per_hr' ).reset_index().to_feather("uq_per_hr.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating uq_per_hr!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Beginning intermediate file concat!")
 data = pd.DataFrame()
for name in ['up_mean.feather', "uph_mean.feather", f"ummr_{ROLL_WINDOW}.feather"]:
temp = pd.read_feather(name ).set_index("index")
 ! rm./{name}
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 data.reset_index().to_feather("./concat_intermediate_1.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with Intermediate Concat 1!")
 data = pd.DataFrame()
for name in ["repeat_c.feather", "uf_bundle.feather", "uwrong_sum.feather",
"seen_exp.feather", "u_mean.feather"]:
temp = pd.read_feather(name ).set_index("index")
 ! rm./{name}
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 data.reset_index().to_feather("./concat_intermediate_2.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with Intermediate Concat 2!")
 data = pd.DataFrame()
for name in ["pq_se.feather", "pqetr.feather"]:
temp = pd.read_feather(name ).set_index("index")
 ! rm./{name}
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 data.reset_index().to_feather("./concat_intermediate_3.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with Intermediate Concat 3!")
 data = pd.DataFrame()
for name in ["uq_per_hr.feather", "up_count.feather", "lec_recent.feather", "h_mean.feather"]:
temp = pd.read_feather(name ).set_index("index")
 ! rm./{name}
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 data.reset_index().to_feather("./concat_intermediate_4.feather")
 del data
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done with Intermediate Concat 4!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Intermediate file concat Complete!")
 data =(pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['content_id'] ).loc[q_mask])
 content_df = pd.read_csv(".. /input/riiid-final-model-inputs/content-df.csv", index_col=0)
 content_df[cat_cols + ['bundle_q_count', 'part']] = content_df[cat_cols + ['bundle_q_count', 'part']].astype('uint8')
 content_df[['wrong_et_med', 'right_et_med']] = content_df[['wrong_et_med', 'right_et_med']].astype('float32')
 content_df[['c_mean', 'c_std']] = content_df[['c_mean', 'c_std']].astype('float32')
 content_df[['content_c', 'tmed', 'che_sum']] =(
 content_df[['content_c', 'tmed', 'che_sum']] ).astype('uint16')
 content_df.drop(['part', 'bundle_id'], axis=1, inplace=True)
 data = data.merge(content_df, how='left', left_on='content_id', right_index=True)
 data.drop(['content_id'], axis=1, inplace=True)
 data.reset_index().to_feather("c_df.feather")
 del data, content_df
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating content_df for merge!")
 data = pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['timestamp', 'user_id'])
 data['response_time'] =(
 data.groupby("user_id")['timestamp'] 
.transform(lambda x: rt_func(x.values)))
 data.drop(['timestamp'], axis=1, inplace=True)
 gc.collect()
data['sessions'] =(data['response_time'] > SESSION_DURATION ).astype(bool)
 data['sessions'] = data.groupby('user_id')['sessions'].cumsum()
data['sess_event_count'] = data.groupby(['user_id', 'sessions'] ).cumcount()
data[['sessions', 'sess_event_count']] = data[['sessions', 'sess_event_count']].astype('uint16')
 data = data.loc[q_mask]
temp = pd.read_feather("./c_df.feather", columns=['index', 'bundle_q_count'])
 temp = temp.set_index("index")
 data['rt_per_task'] = data['response_time'] / temp['bundle_q_count'].values
data.drop(['user_id'], axis=1, inplace=True)
 data.reset_index().to_feather("response_time.feather")
 del data, temp
gc.collect()
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done creating reponse_time!")
 print("
Done creating all required intermediate files.
 ")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <== Starting merge operation!")
 for i in range(CHUNKS):
print(f"
Time Elapsed: {time.time() - start_time:10.2f} s | <= Beginning chunk {i + 1} merge!")
 front, rear = i * BATCH_SIZE,(i + 1)* BATCH_SIZE
data =(pd.read_feather(".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['timestamp', 'user_id', 'task_container_id',
'prior_question_elapsed_time', 'answered_correctly'])
.loc[q_mask] 
.iloc[front: rear])
 for name in ['c_df.feather', 'ts_up_recency.feather', "response_time.feather",
"concat_intermediate_2.feather", "concat_intermediate_4.feather",
"concat_intermediate_1.feather", "concat_intermediate_3.feather"]:
temp = pd.read_feather(name)
 temp.iloc[BATCH_SIZE:].reset_index(drop=True ).to_feather(name)
 temp = temp.iloc[:BATCH_SIZE].set_index("index")
 if name == 'c_df.feather':
temp.drop("bundle_q_count", axis=1, inplace=True)
 gc.collect()
time.sleep(3)
 assert len(data)== len(temp)== BATCH_SIZE
data = pd.concat([data, temp], axis=1)
 del temp
gc.collect()
time.sleep(2)
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done merging {name}!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Saving chunk {i + 1} onto Disk!")
 data.reset_index(drop=True, inplace=True)
 data.to_feather(f"model_train_c{i + 1}.feather")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | * Done saving chunk {i + 1} to Disk!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | <= Done Merging chunk {i + 1}!")
 del data
gc.collect()
time.sleep(3)
 print(f"
Time Elapsed: {time.time() - start_time:10.2f} s | <== Merge Complete!")
 print("Deleting unneeded intermediate files.
 ")
 ! rm c_df.feather concat_intermediate_*.feather ts_up_recency.feather
! rm response_time.feather
else:
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Found PreSaved File(s ).Load it manually if need be!")
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Copying Train File(s)back to Disk!")
 for i in range(CHUNKS):
! cp.. /input/riiid-final-model-inputs/model_train_c{i + 1}.feather./model_train_c{i + 1}.feather
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done copying Train File(s)back to Disk!" )
svr_model = make_pipeline(RobustScaler() , SVR(C=20, epsilon=0.005, gamma=0.0003))
rmse_cv(svr_model )<choose_model_class>
def return_random_slice(batch_size, nrows):

 front = np.random.choice(nrows - batch_size)
 rear = front + batch_size
return front, rear
xgb_model = XGBRegressor(learning_rate=0.01, max_depth=5, n_estimators=3000,
n_thread=-1, n_jobs=-1, objective='reg:squarederror')
 rmse_cv(xgb_model )<choose_model_class>
def return_chunk_indices(start, end, nrows, chunks=3):
'Maps the indices to chunk indices for data loading'
chunk_size =(nrows//chunks)
 indices = []
start_chunk = start // chunk_size
end_chunk =(end-1)// chunk_size
start_chunk_start = start - chunk_size * start_chunk
end_chunk_end = end - chunk_size * end_chunk
start_chunk_end = chunk_size if end_chunk != start_chunk else end_chunk_end
end_chunk_start = 0 if end_chunk != start_chunk else start_chunk_start
indices.append(( start_chunk+1, start_chunk_start, start_chunk_end))
if start_chunk != end_chunk:
for i in range(end_chunk - start_chunk - 1):
indices.append(( start_chunk + i + 2, 0, chunk_size))
indices.append(( end_chunk+1, end_chunk_start, end_chunk_end))
return indices
lgb_model = LGBMRegressor(objective='regression',
learning_rate=0.01, max_depth=5, num_leaves=4,
n_estimators=3000)
 rmse_cv(lgb_model )<choose_model_class>
%%time
ALL_FEATURES = [
'repeat_c', 'tagF', 'tagS', 'tagL', 'tagT', 'response_time', 'prior_question_elapsed_time',
'up_mean', 'up_count', 'uq_per_hr', 'uwrong_sum', 'lec_recent', 'pqet_mean', 'seen_ratio',
'tmed', 'up_recency', 'ts_recency_10', 'ts_recency_5', 'timestamp', 'task_container_id',
'content_c', 'che_sum', 'seen_exp_when_wrong', 'seen_exp_when_right', 'sessions', 'sess_event_count',
'pqetmr_10', 'uf_bundle', 'u_mean', 'ummr_10', 'rt_per_task', 'c_mean', 'c_std', 'h_mean', 'uph_mean',
'wrong_et_med', 'right_et_med'
]
CORE = ['repeat_c', 'tagF', 'tagS', 'tagL', 'response_time', 'up_mean', 'up_count', 'pqet_mean',
'seen_ratio', 'tmed', 'up_recency', 'ts_recency_5', 'content_c', 'pqetmr_10', 'u_mean',
'rt_per_task', 'c_mean', 'c_std', 'h_mean', 'ts_recency_10', 'uph_mean']
N_FEATURES = 30
BATCH_SIZE = int(2.5e7)
 STRIDE = int(1e7)
 HOLD_OUT = int(2.5e6)
 models = []
COL_SAMPLING = False
ALREADY_TRAINED = 8
TRAIN_FOR = 0
ITERATIVE_TRAIN = False
OOF_VALIDATE = False
N_ROUNDS = 3000
LOC1 = ".. /input/riiid-final-model-inputs/model_train_c1.feather"
LOC2 = ".. /input/riiid-final-model-inputs/model_train_c2.feather"
LOC3 = ".. /input/riiid-final-model-inputs/model_train_c3.feather"
SIZE1 = len(pd.read_feather(LOC1, columns=['answered_correctly']))
SIZE2 = len(pd.read_feather(LOC2, columns=['answered_correctly']))
SIZE3 = len(pd.read_feather(LOC3, columns=['answered_correctly']))
N_ROWS = SIZE1 + SIZE2 + SIZE3
CHUNKS =(( N_ROWS - BATCH_SIZE - HOLD_OUT)// STRIDE)+ 1
if TRAIN_FOR is None:
TRAIN_FOR = CHUNKS - ALREADY_TRAINED
assert TRAIN_FOR + ALREADY_TRAINED <= CHUNKS
if ALREADY_TRAINED:
OOF = np.load(".. /input/riiid-final-model-inputs/OOF.npy")
 for j in range(ALREADY_TRAINED):
model = lgb.Booster(model_file=f".. /input/riiid-final-model-inputs/trained_model_c{j+1}.txt")
 model.save_model(f"trained_model_c{j+1}.txt")
 models.append(model)
 print(f"Successfully loaded {len(models)} model(s)!")
 if TRAIN_FOR:
if not ALREADY_TRAINED:
print("Beginning Model Training from Scratch!")
 OOF = np.arange(N_ROWS)
 if ITERATIVE_TRAIN and ALREADY_TRAINED:
pre_model = model
else:
pre_model = None
print(f"{BATCH_SIZE} rows would be used for {CHUNKS} model(s)!", end=' ')
 print(f"{TRAIN_FOR} model(s)would be trained for {N_ROUNDS} rounds!")
 print(f"The models would {'BE' if ITERATIVE_TRAIN else 'NOT BE'} iteratively built!")
 print(f"Col sampling has been {'ENABLED' if COL_SAMPLING else 'DISABLED'}!")
 print(f"{HOLD_OUT + N_ROWS -(( CHUNKS - 1)* STRIDE + BATCH_SIZE)} rows goes straight to OOF!")
 print("Space Req.For One Chunk: {:.2f} GB({})".format(
(N_FEATURES + 1 if COL_SAMPLING else len(ALL_FEATURES)) * BATCH_SIZE * 64 / 8e+9, 'float64'))
for j in range(ALREADY_TRAINED, ALREADY_TRAINED + TRAIN_FOR):
front, rear = j * STRIDE, j * STRIDE + BATCH_SIZE
print(f"
Beginning chunk {j + 1}/{ALREADY_TRAINED + TRAIN_FOR}! Loading the Feather File to Memory => ")
 if COL_SAMPLING:
FEATURES =(CORE + np.random.choice(list(set(ALL_FEATURES)- set(CORE)) ,
size=(N_FEATURES - len(CORE)) ,
replace=False ).tolist())
 print(f"Selected Features({N_FEATURES}/{len(ALL_FEATURES)}): {list(set(FEATURES)- set(CORE)) }")
 cat_cols = np.intersect1d(['tagF', 'tagS', 'tagT', 'tagL'], FEATURES ).tolist()
COLUMNS = ['user_id', *FEATURES, 'answered_correctly']
if 'timestamp' not in COLUMNS:
COLUMNS += ['timestamp']
else:
FEATURES = ALL_FEATURES
COLUMNS = ['user_id', *FEATURES, 'answered_correctly']
cat_cols = ['tagF', 'tagS', 'tagT', 'tagL']
print(f"Slicing from {front} to {rear}!")
 data = pd.DataFrame()
for file, start, end in return_chunk_indices(front, rear, nrows=N_ROWS):
data = pd.concat([
data, 
(pd.read_feather(f".. /input/riiid-final-model-inputs/model_train_c{file}.feather", columns=COLUMNS)
.iloc[start: end])
 ])
 data.index = range(front, rear)
 gc.collect()
time.sleep(3)
 print("Generating train/val Masks for the data...")
 mask = generate_train_val(data, return_mask=True, tp=0.95, dyn_p=True ).values.astype(bool)
 print("Done generating masks! Saving to intermediate train/val files.. ")
 OOF = np.setdiff1d(OOF, data.index[mask])
 data.drop(['user_id'], axis=1, inplace=True)
 gc.collect()
time.sleep(3)
 data.loc[mask].reset_index(drop=True ).to_feather("train_intermediate.feather")
 data.loc[~mask].reset_index(drop=True ).to_feather("val_intermediate.feather")
 del data, mask
gc.collect()
time.sleep(3)
 print("Done saving! Beginning logic for LGB dataset creation.. ")
 chunk_per = 0.025
tr_data = pd.read_feather("train_intermediate.feather")
 tr_data = tr_data[FEATURES + ['answered_correctly']]
tr_result = np.empty(shape=(len(tr_data), len(FEATURES)+ 1), dtype='float64')
 step = int(len(tr_data)* chunk_per)
 end =(len(tr_data)// step)+ 1
for i in tqdm.tqdm(range(1, end + 1)) :
tr_result[(i - 1)* step: i * step] = tr_data.iloc[:step]
tr_data = tr_data.iloc[step:]
gc.collect()
time.sleep(3)
 del tr_data
gc.collect()
! rm./train_intermediate.feather
time.sleep(3)
 tr_data = lgb.Dataset(
 tr_result[:, :-1], label=tr_result[:, -1], feature_name=FEATURES,
categorical_feature=cat_cols, free_raw_data=not ITERATIVE_TRAIN)
 tr_data.construct()
del tr_result
gc.collect()
time.sleep(2)
 vl_data = pd.read_feather("./val_intermediate.feather")
 vl_data = vl_data[FEATURES + ['answered_correctly']]
vl_result = np.empty(shape=(len(vl_data), len(FEATURES)+ 1), dtype='float64')
 step = int(len(vl_data)* chunk_per)
 end =(len(vl_data)// step)+ 1
for i in tqdm.tqdm(range(1, end + 1)) :
vl_result[(i - 1)* step: i * step] = vl_data.iloc[:step]
vl_data = vl_data.iloc[step:]
gc.collect()
time.sleep(3)
 del vl_data
gc.collect()
! rm./val_intermediate.feather
time.sleep(3)
 vl_data = lgb.Dataset(
 vl_result[:, :-1], label=vl_result[:, -1], feature_name=FEATURES,
categorical_feature=cat_cols, reference=tr_data,
free_raw_data=not ITERATIVE_TRAIN)
 vl_data.construct()
del vl_result
gc.collect()
time.sleep(2)
 print("Done generating LGB datasets! Starting training.. ")
 model = lgb.train(
 params={'metric': 'auc', 'objective': 'binary',
'num_leaves': 200, 'learning_rate': 0.075},
train_set=tr_data,
feature_name=FEATURES,
categorical_feature=cat_cols,
num_boost_round=N_ROUNDS,
valid_sets=[tr_data, vl_data],
init_model=pre_model,
early_stopping_rounds=50,
verbose_eval=50 
)
 del tr_data, vl_data
gc.collect()
print(f"Done training chunk {j + 1}!")
 model.save_model(f"trained_model_c{j + 1}.txt")
 models.append(model)
 if ITERATIVE_TRAIN:
pre_model = model
if OOF_VALIDATE:
print(f"
We would be evaluating the model on {HOLD_OUT} rows out of {len(OOF)} OOF rows.")
 data = pd.read_feather(LOC3 ).iloc[-HOLD_OUT:]
actual = data['answered_correctly']
data.drop(['user_id', 'answered_correctly'], axis=1, inplace=True)
 preds = np.zeros(shape=(len(actual), len(models)))
 print("
Individual Model's OOF Scores:")
 for i in range(len(models)) :
FEATURES = models[i].feature_name()
cat_cols = np.intersect1d(['tagF', 'tagS', 'tagT', 'tagL'], FEATURES)
 preds[:, i] = models[i].predict(data[FEATURES], categorical_feature=cat_cols)
 print(f"\tModel
print(f"
The Ensemble's Score: {roc_auc_score(actual, preds.mean(1)) :.4f}")
 del data, actual, preds
gc.collect()
if ITERATIVE_TRAIN:
models = models[-1:]
np.save("OOF.npy", OOF)
 print(f"
Number of Models: {len(models)}
" )
stack_model = StackingCVRegressor([lasso_model, ridge_model, enet_model, gbdt_model, rf_model, svr_model, xgb_model, lgb_model],
meta_regressor=lgb_model,
use_features_in_secondary=True)
<compute_test_metric>
class ensemble(object):
def __init__(self, models, weights=None, treelite=True):
self.n_models = len(models)
 self.models = models
self.weights = [1 / self.n_models] * self.n_models if not weights else weights
self.cat_cols = ['tagF', 'tagS', 'tagL', 'tagT']
self.treelite = treelite
self.feats = ['repeat_c', 'tagF', 'tagS', 'tagL', 'tagT', 'response_time', 'prior_question_elapsed_time',
'up_mean', 'up_count', 'uq_per_hr', 'uwrong_sum', 'lec_recent', 'pqet_mean', 'seen_ratio',
'tmed', 'up_recency', 'ts_recency_10', 'ts_recency_5', 'timestamp', 'task_container_id',
'content_c', 'che_sum', 'seen_exp_when_wrong', 'seen_exp_when_right', 'sessions',
'sess_event_count', 'pqetmr_10', 'uf_bundle', 'u_mean', 'ummr_10', 'rt_per_task', 'c_mean',
'c_std', 'h_mean', 'uph_mean', 'wrong_et_med', 'right_et_med']
if not self.treelite:
assert len(np.setdiff1d(self.feats, self.models[0].feature_name())) == 0
def __getattr__(self, attr):

 return getattr(self.models[0], attr)
 def feature_name(self):
return self.feats
def predict(self, x, **kwargs):
if not self.treelite:
return self.predict_lgb(x[self.feats])
 else:
return self.predict_treelite(x[self.feats].values)
 def predict_treelite(self, x):
X = treelite_runtime.Batch.from_npy2d(x)
 pred = np.zeros(len(x))
for m, w in zip(self.models, self.weights):
pred += m.predict(X)* w
return pred
def predict_lgb(self, x):
pred = np.zeros(len(x))
for m, w in zip(self.models, self.weights):
pred += m.predict(x, categorical_feature=self.cat_cols)* w
return pred
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<train_model>
%%time
start_time = time.time()
if not os.path.exists(".. /input/col-sampled-train-dataset/lgb_pred_1.npy"):
models = [models[3], models[6]]
BATCH_SIZE = int(3.0e7)
 CHUNKS = list(range(0, N_ROWS, BATCH_SIZE))
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Predictions will be chunked for", len(CHUNKS), "Chunks!")
 print(CHUNKS)
 for j, front in enumerate(CHUNKS):
rear = front + BATCH_SIZE
print(f"
Beginning chunk {j+1}! Loading the Feather File to Memory => ")
 print(f"Slicing from {front} to {rear}!")
 data = pd.DataFrame()
for file, start, end in return_chunk_indices(front, rear, nrows=N_ROWS):
file = f".. /input/riiid-final-model-inputs/model_train_c{file}.feather"
if not os.path.exists(file):
continue
data = pd.concat([data, pd.read_feather(file ).iloc[start: end]])
 gc.collect()
time.sleep(2)
 data = data[LGB_COLS]
data_shape = data.shape
print(f"Time Elapsed: {time.time() - start_time:10.2f} s |", end=' ')
 print(f"Done loading! Data Dimensions: {data_shape}")
 print("Beginning logic for treelite batch dataset creation.. ")
 gc.collect()
time.sleep(2)
 chunk_per = 0.025
darray = np.empty(shape=(data_shape[0], data_shape[1]), dtype='float64')
 step = int(data_shape[0] * chunk_per)
 end =(data_shape[0] // step)+ 1
for i in tqdm.tqdm(range(1, end + 1)) :
darray[(i - 1)* step: i * step] = data.iloc[:step]
data = data.iloc[step:]
gc.collect()
time.sleep(3)
 del data
gc.collect()
time.sleep(3)
 darray = treelite_runtime.Batch.from_npy2d(darray)
 gc.collect()
time.sleep(3)
 lgb_pred = np.zeros(shape=data_shape[0], dtype='float32')
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Making Predictions using Treelite models =>")
 for k, model in enumerate(models):
lgb_pred = lgb_pred + model.predict(darray)
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done with model
lgb_pred = lgb_pred / len(models)
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done with Chunk {j+1}! Saving predictions to Disk!")
 np.save(f"lgb_pred_{j+1}.npy", lgb_pred)
 del darray, lgb_pred
gc.collect()
else:
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Copying LGB pred numpy arrays back to Disk!")
 for i in glob.glob(".. /input/col-sampled-train-dataset/lgb_pred_*"):
! cp {i} {i.split("/")[-1]}
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done copying to Disk!" )
lasso_model = lasso_model.fit(X_train, y_train)
 ridge_model = ridge_model.fit(X_train, y_train)
 enet_model = enet_model.fit(X_train, y_train)
 gbdt_model = gbdt_model.fit(X_train, y_train)
 rf_model = rf_model.fit(X_train, y_train)
 svr_model = svr_model.fit(X_train, y_train)
 xgb_model = xgb_model.fit(X_train, y_train)
 lgb_model = lgb_model.fit(X_train, y_train)
 stack_model = stack_model.fit(np.array(X_train), np.array(y_train))<predict_on_test>
%%time
start_time = time.time()
if not os.path.exists(".. /input/col-sampled-train-dataset/lgb2_pred_1.npy"):
models = [models[1], models[4]]
BATCH_SIZE = int(3.0e7)
 CHUNKS = list(range(0, N_ROWS, BATCH_SIZE))
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Predictions will be chunked for", len(CHUNKS), "Chunks!")
 print(CHUNKS)
 for j, front in enumerate(CHUNKS):
rear = front + BATCH_SIZE
print(f"
Beginning chunk {j+1}! Loading the Feather File to Memory => ")
 print(f"Slicing from {front} to {rear}!")
 data = pd.DataFrame()
for file, start, end in return_chunk_indices(front, rear, nrows=N_ROWS):
file = f".. /input/riiid-final-model-inputs/model_train_c{file}.feather"
if not os.path.exists(file):
continue
data = pd.concat([data, pd.read_feather(file ).iloc[start: end]])
 gc.collect()
time.sleep(2)
 data = data[LGB_COLS]
data_shape = data.shape
print(f"Time Elapsed: {time.time() - start_time:10.2f} s |", end=' ')
 print(f"Done loading! Data Dimensions: {data_shape}")
 print("Beginning logic for treelite batch dataset creation.. ")
 gc.collect()
time.sleep(2)
 chunk_per = 0.025
darray = np.empty(shape=(data_shape[0], data_shape[1]), dtype='float64')
 step = int(data_shape[0] * chunk_per)
 end =(data_shape[0] // step)+ 1
for i in tqdm.tqdm(range(1, end + 1)) :
darray[(i - 1)* step: i * step] = data.iloc[:step]
data = data.iloc[step:]
gc.collect()
time.sleep(3)
 del data
gc.collect()
time.sleep(3)
 darray = treelite_runtime.Batch.from_npy2d(darray)
 gc.collect()
time.sleep(3)
 lgb_pred = np.zeros(shape=data_shape[0], dtype='float32')
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Making Predictions using Treelite models =>")
 for k, model in enumerate(models):
lgb_pred = lgb_pred + model.predict(darray)
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done with model
lgb_pred = lgb_pred / len(models)
 print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done with Chunk {j+1}! Saving predictions to Disk!")
 np.save(f"lgb2_pred_{j+1}.npy", lgb_pred)
 del darray, lgb_pred
gc.collect()
else:
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Copying LGB pred numpy arrays back to Disk!")
 for i in glob.glob(".. /input/col-sampled-train-dataset/lgb_pred_*"):
! cp {i} {i.split("/")[-1]}
print(f"Time Elapsed: {time.time() - start_time:10.2f} s | Done copying to Disk!" )
def combine_models_predict(X):
return(( 0.1 * enet_model.predict(X)) + \ 
(0.1 * ridge_model.predict(X)) + \ 
(0.1 * lasso_model.predict(X)) + \ 
(0.15 * gbdt_model.predict(X)) + \ 
(0.15 * xgb_model.predict(X)) + \ 
(0.1 * lgb_model.predict(X)) + \ 
(0.075 * rf_model.predict(X)) + \ 
(0.075 * svr_model.predict(X)) + \ 
(0.15 * stack_model.predict(np.array(X)))
 )<compute_test_metric>
%%time
data = pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['content_id', 'content_type_id'])
 data = data.loc[~data['content_type_id']].iloc[:N_ROWS]
data = data.merge(
 ques[['question_id', 'part']].set_index("question_id"),
left_on=['content_id'], right_index=True, how='left')
 data.drop(['content_type_id'], axis=1, inplace=True)
 data.reset_index(drop=True, inplace=True)
 data.to_feather("./intermediate_train.feather")
 del data
gc.collect()
data = pd.DataFrame()
for LOC in [LOC1, LOC2, LOC3]:
data = pd.concat([
data, 
(pd.read_feather(LOC, columns=['user_id', 'task_container_id', 'content_c', 'c_mean', 'h_mean',
'answered_correctly', 'ummr_10']))
], axis=0)
 data.reset_index(drop=True, inplace=True)
 data['ummr_10_50'] = data['ummr_10'] > 0
data['h_mean_50'] = data['h_mean'] > 0.5
data['c_mean_50'] = data['c_mean'] > 0.5
data['c_mean_25'] = data['c_mean'] > 0.25
data['c_mean_75'] = data['c_mean'] > 0.75
data.drop(['ummr_10', 'c_mean', 'h_mean'], axis=1, inplace=True)
 data = pd.concat([
data, pd.read_feather("./intermediate_train.feather")
 ], axis=1)
 ! rm./intermediate_train.feather
lgb_pred = np.array([])
 for file in sorted(glob.glob(".. /input/col-sampled-train-dataset/lgb_pred_*")) :
lgb_pred = np.concatenate([lgb_pred, np.load(file)])
 lgb2_pred = np.array([])
 for file in sorted(glob.glob(".. /input/col-sampled-train-dataset/lgb2_pred_*")) :
lgb2_pred = np.concatenate([lgb2_pred, np.load(file)])
 assert len(lgb_pred)== len(lgb2_pred)
 lgb_pred =(lgb_pred + lgb2_pred)/ 2
del lgb2_pred
gc.collect()
data['lgb_pred'] = lgb_pred
data['lgb_75'] = data['lgb_pred'] > 0.75
data['lgb_50'] = data['lgb_pred'] > 0.5
data['lgb_25'] = data['lgb_pred'] > 0.25
del lgb_pred
gc.collect()
print(data.shape)
 print(data.columns )
print('RMSLE score on train data:')
 print(rmsle(y_train, combine_models_predict(X_train)) )<predict_on_test>
def df_to_dt_format(df):
for i in df.columns:
org = str(df[i].dtype)
 converted = org.lstrip("u")
 if org != converted:
converted = converted[:3] + str(int(converted.lstrip("int")) * 2)
 df[i] = df[i].astype(converted )
log_result = combine_models_predict(X_test)
 result = np.expm1(log_result )<prepare_output>
df_to_dt_format(data)
 data = dt.Frame(data )
sub = pd.DataFrame()
sub['Id'] = test['Id']
sub['SalePrice'] = result
sub.head()<save_to_csv>
FTRL_COLS = [
'user_id', 'task_container_id', 'content_c', 'ummr_10_50', 'h_mean_50',
'c_mean_50', 'c_mean_25', 'c_mean_75', 'content_id', 'part', 'lgb_pred', 'lgb_75',
'lgb_50', 'lgb_25'
]
INTERACTIONS = None
sub.to_csv('submission.csv',index=False )<load_from_csv>
%%time
ftrl = Ftrl(
 nepochs=1, interactions=INTERACTIONS,
alpha=0.005, double_precision=True, 
)
 ftrl.fit(data[:int(9.5e7), FTRL_COLS], data[:int(9.5e7), ['answered_correctly']] )
df_trainval=pd.read_csv("/kaggle/input/titanic/train.csv")
 df_test=pd.read_csv("/kaggle/input/titanic/test.csv" )<create_dataframe>
%%time
preds = ftrl.predict(data[int(9.5e7):, FTRL_COLS] ).to_pandas()
actual = data[int(9.5e7):, 'answered_correctly'].to_numpy()
print("Baseline Score to beat: {:.4f}".format(roc_auc_score(
 actual, data[int(9.5e7):, 'lgb_pred'].to_pandas() 
)))
 roc_auc_score(actual, preds )
df_trainval_C=df_trainval.copy()<data_type_conversions>
model = ensemble(models, treelite=False)
 model, type(model.models[0] )
print("% Of Males Survived:",(((((df_trainval["Sex"]=="male")&(df_trainval["Survived"]==1)).astype(int)).sum())/(df_trainval["Sex"]=="male" ).astype(int ).sum())*100,"%")
 print("% Of Females Survived:",(((((df_trainval["Sex"]=="female")&(df_trainval["Survived"]==1)).astype(int)).sum())/(df_trainval["Sex"]=="female" ).astype(int ).sum())*100,"%")
 fig, ax = plt.subplots(figsize=(8, 5))
sns.countplot(df_trainval['Survived'],hue=df_trainval["Sex"])
 plt.title('Survived labels values')
 plt.show()<data_type_conversions>
if not os.path.exists(".. /input/treelite-converted-ensemble-8-models-25m/lgb_pred_final.npy"):
temp = pd.read_feather(LOC3 ).iloc[-(N_ROWS - int(9.5e7)) :][model.feature_name() ]
lgb_pred = model.predict(temp)
 del temp
gc.collect()
else:
lgb_pred = np.load(".. /input/treelite-converted-ensemble-8-models-25m/lgb_pred_final.npy" )
print("% Of Males Survived from class 1:",(((((df_trainval["Sex"]=="male")&(df_trainval["Survived"]==1)&(df_trainval["Pclass"]==1)).astype(int)).sum())/(((df_trainval["Sex"]=="male")&(df_trainval["Pclass"]==1)).astype(int)).sum())*100,"%")
 print("% Of Males Survived from class 2:",(((((df_trainval["Sex"]=="male")&(df_trainval["Survived"]==1)&(df_trainval["Pclass"]==2)).astype(int)).sum())/(((df_trainval["Sex"]=="male")&(df_trainval["Pclass"]==2)).astype(int)).sum())*100,"%")
 print("% Of Males Survived from class 3:",(((((df_trainval["Sex"]=="male")&(df_trainval["Survived"]==1)&(df_trainval["Pclass"]==3)).astype(int)).sum())/(((df_trainval["Sex"]=="male")&(df_trainval["Pclass"]==3)).astype(int)).sum())*100,"%")
 print("% Of Females Survived from class 1:",(((((df_trainval["Sex"]=="female")&(df_trainval["Survived"]==1)&(df_trainval["Pclass"]==1)).astype(int)).sum())/(((df_trainval["Sex"]=="female")&(df_trainval["Pclass"]==1)).astype(int)).sum())*100,"%")
 print("% Of Females Survived from class 2:",(((((df_trainval["Sex"]=="female")&(df_trainval["Survived"]==1)&(df_trainval["Pclass"]==2)).astype(int)).sum())/(((df_trainval["Sex"]=="female")&(df_trainval["Pclass"]==2)).astype(int)).sum())*100,"%")
 print("% Of Females Survived from class 3:",(((((df_trainval["Sex"]=="female")&(df_trainval["Survived"]==1)&(df_trainval["Pclass"]==3)).astype(int)).sum())/(((df_trainval["Sex"]=="female")&(df_trainval["Pclass"]==3)).astype(int)).sum())*100,"%")
 sns.catplot(x='Survived',hue="Pclass",col='Sex',kind="count",data=df_trainval)
 plt.show()<count_values>
data[int(9.5e7):, 'lgb_pred'] = lgb_pred
data[int(9.5e7):, 'lgb_75'] = data[int(9.5e7):, dt.f.lgb_pred > 0.75]
data[int(9.5e7):, 'lgb_50'] = data[int(9.5e7):, dt.f.lgb_pred > 0.50]
data[int(9.5e7):, 'lgb_25'] = data[int(9.5e7):, dt.f.lgb_pred > 0.25]
preds = ftrl.predict(data[int(9.5e7):, FTRL_COLS])
 print("Baseline Score to beat: {:.4f}".format(roc_auc_score(
 actual, data[int(9.5e7):, 'lgb_pred'].to_pandas() 
)))
 roc_auc_score(data[int(9.5e7):, 'answered_correctly'].to_pandas() , preds )
df_trainval["Parch"].value_counts()<data_type_conversions>
ol_ftrl = deepcopy(ftrl)
 ol_ftrl.alpha = 0.005
print("% of people survived with having 0 Parch: ",(((((df_trainval["Parch"]==0)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["Parch"]==0 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 1 Parch: ",(((((df_trainval["Parch"]==1)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["Parch"]==1 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 2 Parch: ",(((((df_trainval["Parch"]==2)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["Parch"]==2 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 3 Parch: ",(((((df_trainval["Parch"]==3)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["Parch"]==3 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 4 Parch: ",(((((df_trainval["Parch"]==4)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["Parch"]==4 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 5 Parch: ",(((((df_trainval["Parch"]==5)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["Parch"]==5 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 6 Parch: ",(((((df_trainval["Parch"]==6)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["Parch"]==6 ).astype(int ).sum())) *100," %")
 sns.catplot(x='Survived',hue="Parch",kind="count",data=df_trainval )<count_values>
%%time
ol_preds = []
preds = []
BATCH_SIZE = 1000
for front in range(int(9.5e7), N_ROWS, BATCH_SIZE):
pred = ol_ftrl.predict(data[front:front+BATCH_SIZE, FTRL_COLS] ).to_list() [0]
ol_preds.append(pred)
 pred = ftrl.predict(data[front:front+BATCH_SIZE, FTRL_COLS] ).to_list() [0]
preds.append(pred)
 ol_ftrl.fit(data[front:front+BATCH_SIZE, FTRL_COLS], data[front:front+BATCH_SIZE, 'answered_correctly'] )
df_trainval["SibSp"].value_counts()<data_type_conversions>
actual = data[int(9.5e7):, 'answered_correctly'].to_numpy()
print("Baseline Score to beat: {:.4f}".format(roc_auc_score(
 actual, data[int(9.5e7):, 'lgb_pred'].to_pandas() 
)))
 print("
Model Score Comparison: Online: {:.4f} | Offline: {:.4f}".format(
 roc_auc_score(actual, np.concatenate(ol_preds)) ,
roc_auc_score(actual, np.concatenate(preds)) 
))
print("% of people survived with having 0 Siblings: ",(((((df_trainval["SibSp"]==0)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["SibSp"]==0 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 1 Siblings: ",(((((df_trainval["SibSp"]==1)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["SibSp"]==1 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 2 Siblings: ",(((((df_trainval["SibSp"]==2)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["SibSp"]==2 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 3 Siblings: ",(((((df_trainval["SibSp"]==3)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["SibSp"]==3 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 4 Siblings: ",(((((df_trainval["SibSp"]==4)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["SibSp"]==4 ).astype(int ).sum())) *100," %")
 print("% of people survived with having 5 Siblings: ",(((((df_trainval["SibSp"]==5)&(df_trainval["Survived"]==1)).astype(int ).sum())/(df_trainval["SibSp"]==5 ).astype(int ).sum())) *100," %")
 sns.catplot(x='Survived',hue="SibSp",kind="count",data=df_trainval )<data_type_conversions>
%%time
TREELITE = False
if TREELITE:
models = []
for model in sorted(glob.glob(".. /input/treelite-converted-ensemble-8-models-25m/tl_*.so")) :
models.append(treelite_runtime.Predictor(model, verbose=False, nthread=1))
else:
models = []
for file in sorted(glob.glob(".. /input/riiid-final-model-inputs/trained_model_*.txt")) :
models.append(lgb.Booster(model_file=file))
model = ensemble(models, treelite=TREELITE)
 model, model.n_models, type(model.models[0] )
print("% Of male Survived with boarding at Southampton :",(((((df_trainval["Sex"]=="male")&(df_trainval["Survived"]==1)&(df_trainval["Embarked"]=="S")).astype(int)).sum())/(((df_trainval["Sex"]=="male")&(df_trainval["Embarked"]=="S")).astype(int)).sum())*100,"%")
 print("% Of female Survived with boarding at Southampton :",(((((df_trainval["Sex"]=="female")&(df_trainval["Survived"]==1)&(df_trainval["Embarked"]=="S")).astype(int)).sum())/(((df_trainval["Sex"]=="female")&(df_trainval["Embarked"]=="S")).astype(int)).sum())*100,"%")
 print("% Of male Survived with boarding at Cherbourg :",(((((df_trainval["Sex"]=="male")&(df_trainval["Survived"]==1)&(df_trainval["Embarked"]=="C")).astype(int)).sum())/(((df_trainval["Sex"]=="male")&(df_trainval["Embarked"]=="C")).astype(int)).sum())*100,"%")
 print("% Of female Survived with boarding at Cherbourg :",(((((df_trainval["Sex"]=="female")&(df_trainval["Survived"]==1)&(df_trainval["Embarked"]=="C")).astype(int)).sum())/(((df_trainval["Sex"]=="female")&(df_trainval["Embarked"]=="C")).astype(int)).sum())*100,"%")
 print("% Of female Survived with boarding at Queenstown :",(((((df_trainval["Sex"]=="male")&(df_trainval["Survived"]==1)&(df_trainval["Embarked"]=="Q")).astype(int)).sum())/(((df_trainval["Sex"]=="male")&(df_trainval["Embarked"]=="Q")).astype(int)).sum())*100,"%")
 print("% Of female Survived with boarding at Queenstown :",(((((df_trainval["Sex"]=="female")&(df_trainval["Survived"]==1)&(df_trainval["Embarked"]=="Q")).astype(int)).sum())/(((df_trainval["Sex"]=="female")&(df_trainval["Embarked"]=="Q")).astype(int)).sum())*100,"%")
 sns.catplot(x='Survived',col="Embarked",hue="Sex",kind="count",data=df_trainval)
 plt.show()<count_values>
load_from_file = True
cat_cols = ['tagF', 'tagS', 'tagT', 'tagL']
df_trainval["Sex"].value_counts()<categorify>
start_time = time.time()
if not load_from_file:
pq_shifted = pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'task_container_id', 'content_type_id',
'prior_question_elapsed_time', 'prior_question_had_explanation'])
 q_mask = pq_shifted['content_type_id'] == 0
pq_shifted = pq_shifted.loc[q_mask]
pq_shifted['prior_question_had_explanation'] =(
 pq_shifted['prior_question_had_explanation'] 
.fillna(False ).astype('uint8'))
temp =(pq_shifted.groupby(['user_id', 'task_container_id'])
 [['prior_question_elapsed_time', 'prior_question_had_explanation']] 
.mean())
 temp = temp.groupby("user_id" ).shift(-1)
 temp.columns = ['pqet_shifted', 'pqhe_shifted']
pq_shifted.drop(
 ['content_type_id', 'prior_question_elapsed_time', 'prior_question_had_explanation'],
axis=1, inplace=True)
 gc.collect()
pq_shifted = pq_shifted.merge(temp, left_on=['user_id', 'task_container_id'], right_index=True, how='left')
 pq_shifted.drop(['user_id'], axis=1, inplace=True)
 pq_shifted['pqet_shifted'] = pq_shifted['pqet_shifted'].astype('float32')
 pq_shifted['pqhe_shifted'] = pq_shifted['pqhe_shifted'].fillna(False ).astype(bool)
 pq_shifted.reset_index().to_feather("./pq_shifted.feather")
 content_df =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['content_id', 'answered_correctly'] ).loc[q_mask])
 content_df = pd.concat([content_df, pq_shifted], axis=1)
 del temp, pq_shifted
gc.collect()
et_med = content_df.groupby(["content_id", 'answered_correctly'])['pqet_shifted'].median()
et_med = et_med.unstack()
et_med.columns = ['wrong_et_med', 'right_et_med']
che_sum = content_df.groupby(['content_id'])['pqhe_shifted'].sum().rename("che_sum")
 tmed = content_df.groupby('content_id')['task_container_id'].median().rename("tmed")
 content_df =(content_df.groupby('content_id')['answered_correctly'].agg(
 content_c='count', c_mean='mean', c_std='std'))
content_df = pd.concat([content_df, et_med, tmed, che_sum], axis=1)
 del et_med, tmed, che_sum
gc.collect()
content_df = content_df.merge(
 ql.loc[~ql.content_type_id, ['content_id', *cat_cols, 'part', 'bundle_q_count', 'bundle_id']] 
.set_index("content_id"), on='content_id', how='left')
 else:
content_df = pd.read_csv(".. /input/riiid-final-model-inputs/content-df.csv", index_col=0)
 content_df[cat_cols + ['bundle_q_count', 'part']] = content_df[cat_cols + ['bundle_q_count', 'part']].astype('uint8')
 content_df[['wrong_et_med', 'right_et_med']] = content_df[['wrong_et_med', 'right_et_med']].astype('float32')
 content_df[['c_mean', 'c_std']] = content_df[['c_mean', 'c_std']].astype('float32')
 content_df[['content_c', 'tmed', 'che_sum', 'bundle_id']] =(
 content_df[['content_c', 'tmed', 'che_sum', 'bundle_id']] ).astype('uint16')
 print(f"Time Elapsed: {(time.time() - start_time):6.2f} Sec")
 print(f"Memory Usage: {content_df.memory_usage(deep=True ).sum() /(2**20):6.2f}", 'MB')
 print(f"Length : {len(content_df):7}")
 content_df.sample(5 )
df_trainval["Sex_labels"],Sex_uniques=pd.factorize(df_trainval["Sex"])
 df_trainval.head()<count_values>
start_time = time.time()
LEC_RECENT_ROLL = 10
SESSION_DURATION = 15 * 60 * 1000
if not load_from_file:
user_df =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'answered_correctly', 'timestamp', 'content_id',
'prior_question_elapsed_time', 'prior_question_had_explanation']))
user_df = user_df.merge(
 content_df[['part', 'c_mean', 'bundle_id']], left_on='content_id',
right_index=True, how='left')
 uf_bundle = user_df.groupby("user_id")['bundle_id'].first().rename("uf_bundle")
 user_df.drop(['bundle_id', 'content_id'], axis=1, inplace=True)
 user_df['sessions'] = user_df.groupby("user_id")['timestamp'].transform(lambda x: rt_func(x.values))
user_df['sessions'] =(user_df['sessions'] > SESSION_DURATION ).astype('uint8')
 sess = pd.concat([
user_df.groupby('user_id')['sessions'].sum().rename("sessions"),
user_df.groupby('user_id')['sessions'].agg(
 lambda x: np.argmax(x.values[::-1])if x.any() else len(x)- 1 ).rename("sess_event_count")
 ], axis=1)
 user_ts = user_df.groupby(['user_id', 'part'])['timestamp'].max().unstack().fillna(0)
 user_ts.columns = [f'ts_max_{i}' for i in range(1, 8)]
user_ts = pd.concat([
user_df.groupby("user_id")['timestamp'].agg(ts_max='max', uic='count'),
user_ts
], axis=1)
 user_df.drop(['sessions', 'timestamp'], axis=1, inplace=True)
 lec_recent = user_df[['user_id', 'answered_correctly']].groupby("user_id" ).tail(LEC_RECENT_ROLL)
 lec_recent['content_type_id'] = lec_recent['answered_correctly'].map({-1: 1, 0: 0, 1: 0})
 lec_recent.drop("answered_correctly", axis=1, inplace=True)
 lec_recent =(
 lec_recent.groupby('user_id')['content_type_id'] 
.apply(lambda x: np.where(x)[0][-1] + 1 if x.any() else 0)
.rename("lec_recent")
)
 user_df = user_df[user_df.answered_correctly != -1]
user_df['prior_question_had_explanation'] = user_df['prior_question_had_explanation'].astype(bool ).fillna(False)
 pq_sum = user_df.groupby("user_id")[['prior_question_had_explanation', 'prior_question_elapsed_time']].sum()
user_df.drop(['prior_question_had_explanation', 'prior_question_elapsed_time'], axis=1, inplace=True)
 gc.collect()
seen_exp = pd.read_feather("./pq_shifted.feather", columns=['index', 'pqhe_shifted'] ).set_index('index')
 user_df = pd.concat([user_df, seen_exp], axis=1)
 seen_exp = user_df.groupby(['user_id', 'answered_correctly'])['pqhe_shifted'].sum().unstack()
seen_exp.columns = ['seen_exp_when_wrong', 'seen_exp_when_right']
seen_exp.fillna(0, inplace=True)
 user_df.drop(['pqhe_shifted'], axis=1, inplace=True)
 u_crct_sum = user_df.groupby("user_id")['answered_correctly'].sum().rename("ucs")
 up_org_sum = user_df.groupby(['user_id', 'part'])['answered_correctly'].agg('sum' ).unstack()
up_org_sum = up_org_sum.astype("float32" ).fillna(0)
 user_df['answered_correctly'] = modify_ac(user_df['answered_correctly'].values, user_df['c_mean'].values)
 user_df.drop(["c_mean"], axis=1, inplace=True)
 gc.collect()
user_df = user_df.groupby(['user_id', 'part'])['answered_correctly'].agg(['sum', 'count'] ).unstack()
user_df = user_df.astype('float32' ).fillna(0)
 user_df['u_sum'] = user_df['sum'].sum(1)
 user_df['u_count'] = user_df['count'].sum(1)
 user_df = pd.concat([
user_ts, sess, lec_recent, up_org_sum,
user_df, uf_bundle, u_crct_sum,
pq_sum, seen_exp
], axis=1)
 del user_ts, pq_sum, u_crct_sum, lec_recent
del uf_bundle, sess, seen_exp
gc.collect()
user_df.columns =(
 ['ts_max', 'uic'] + [f'ts_max_{i}' for i in range(1, 8)] +
['sess_max', 'sess_event_sum'] + ['lec_recent'] +
[f'ups{i}' for i in range(1, 8)] +
[f's{i}' for i in range(1, 8)] +
[f'c{i}' for i in range(1, 8)] +
['us', 'uc'] + ['uf_bundle'] +
['ucs'] + ['pqhe_sum', 'pqet_sum'] +
['seen_exp_when_wrong', 'seen_exp_when_right'])
 else:
user_df = pd.read_csv(".. /input/riiid-final-model-inputs/user-df.csv", index_col=0)
 user_df['lec_recent'] = user_df['lec_recent'].astype('uint8')
 user_df[[f'ups{i}' for i in range(1, 8)]] = user_df[[f'ups{i}' for i in range(1, 8)]].astype("uint16")
 user_df[[f'c{i}' for i in range(1, 8)]] = user_df[[f'c{i}' for i in range(1, 8)]].astype('uint16')
 user_df[['uic', 'uc', 'ucs', 'pqhe_sum']] = user_df[['uic', 'uc', 'ucs', 'pqhe_sum']].astype("uint16")
 user_df[['sess_event_sum', 'sess_max']] = user_df[['sess_event_sum', 'sess_max']].astype("uint16")
 user_df[['seen_exp_when_wrong', 'seen_exp_when_right', 'uf_bundle']] =(
 user_df[['seen_exp_when_wrong', 'seen_exp_when_right', 'uf_bundle']].astype("uint16"))
print(f"Time Elapsed: {(time.time() - start_time):7.2f} Sec")
 print(f"Memory Usage: {user_df.memory_usage(deep=True ).sum() /(2**20):7.2f}", 'MB')
 print(f"Length : {len(user_df):8}")
 user_df.sample(5 ).T
df_trainval["Pclass"].value_counts()<data_type_conversions>
max_q = ques.question_id.max() + 1
def to_ba(indices, max_q=max_q):
'Function to convert indices to bitarray'
ba = np.zeros(max_q, dtype=bool)
 ba[indices] = 1
return bitarray(list(ba))
def parallelize(data, func, num_of_processes=8):
data_split = np.array_split(data, num_of_processes)
 pool = Pool(num_of_processes)
 data = pd.concat(pool.map(func, data_split))
pool.close()
pool.join()
return data
def run_on_subset(func, data_subset):
return data_subset.apply(func)
 def parallelize_on_rows(data, func, num_of_processes=8):
return parallelize(data, partial(run_on_subset, func), num_of_processes)
 start_time = time.time()
if not load_from_file:
repeat_c =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'content_id', 'answered_correctly']))
repeat_c = repeat_c[repeat_c['answered_correctly'] != -1]
repeat_c = repeat_c.groupby("user_id")['content_id'].unique()
repeat_c = parallelize_on_rows(repeat_c, to_ba)
 else:
repeat_c = pd.read_pickle(".. /input/riiid-final-model-inputs/repeat-c.pkl")
 print(f"Time Elapsed: {(time.time() - start_time):6.2f} Sec")
 print(f"Memory Usage: {sys.getsizeof(repeat_c)/(2 ** 20):6.2f} MB")
 print(f"{len(repeat_c)} users have been extracted!")
 repeat_c.sample(3 )
Age_mean=df_trainval["Age"].mean()
df_trainval["Age"].fillna(Age_mean,inplace=True )<feature_engineering>
start_time = time.time()
ROLL_WINDOW = 10
UROLL_NULL_FILL = -2
if not load_from_file:
u_roll =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'answered_correctly', 'content_id']))
u_roll = u_roll[u_roll.answered_correctly != -1]
u_roll = u_roll.groupby("user_id" ).tail(ROLL_WINDOW)
 u_roll = u_roll.merge(content_df['c_mean'], left_on='content_id', right_index=True, how='left')
 u_roll['answered_correctly'] = modify_ac(u_roll['answered_correctly'].values, u_roll['c_mean'].values)
 u_roll.drop(['c_mean', 'content_id'], axis=1, inplace=True)
 gc.collect()
u_roll = u_roll.groupby("user_id")['answered_correctly'].apply(
 lambda x: np.concatenate([np.repeat(UROLL_NULL_FILL, ROLL_WINDOW - len(x)) , x.values])
)
 u_roll = parallelize_on_rows(u_roll, pd.Series)
 gc.collect()
else:
u_roll = pd.read_csv(".. /input/riiid-final-model-inputs/u_roll.csv", index_col=0)
 u_roll = u_roll.astype("float32")
 print(f"Time Elapsed: {(time.time() - start_time):7.2f} Sec")
 print(f"Memory Usage: {u_roll.memory_usage(deep=True ).sum() /(2**20):7.2f}", 'MB')
 print(f"Length : {len(u_roll):7}")
 u_roll.sample(5 )
a,age_bins=pd.qcut(df_trainval["Age"],7,retbins=True)
 df_trainval["Age_label"],Age_unique=pd.factorize(a)
 df_trainval.head()<count_values>
start_time = time.time()
ROLL_WINDOW_PQET = 10
PQET_ROLL_NULL_FILL = -1
if not load_from_file:
pqet_roll =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'prior_question_elapsed_time', 'answered_correctly']))
pqet_roll = pqet_roll[pqet_roll.answered_correctly != -1]
pqet_roll.drop(['answered_correctly'], axis=1, inplace=True)
 pqet_roll = pqet_roll.groupby("user_id" ).tail(ROLL_WINDOW_PQET)
 pqet_roll = pqet_roll.groupby("user_id")['prior_question_elapsed_time'].apply(
 lambda x: np.concatenate([np.repeat(PQET_ROLL_NULL_FILL, ROLL_WINDOW_PQET - len(x)) , x.values])
)
 pqet_roll = parallelize_on_rows(pqet_roll, pd.Series)
 gc.collect()
pqet_roll = pqet_roll.fillna(PQET_ROLL_NULL_FILL)
 else:
pqet_roll = pd.read_csv(".. /input/riiid-final-model-inputs/pqet_roll.csv", index_col=0)
 pqet_roll = pqet_roll.astype("float32")
 print(f"Time Elapsed: {(time.time() - start_time):7.2f} Sec")
 print(f"Memory Usage: {pqet_roll.memory_usage(deep=True ).sum() /(2**20):7.2f}", 'MB')
 print(f"Length : {len(pqet_roll):7}")
 pqet_roll.sample(5 )
df_trainval["Age_label"].value_counts()<data_type_conversions>
start_time = time.time()
TS_RECENCY_PERIOD = 10
TS_ROLL_NULL_FILL = np.nan
if not load_from_file:
ts_roll =(pd.read_feather(
 ".. /input/riiid-train-data-multiple-formats/riiid_train.feather",
columns=['user_id', 'timestamp']))
ts_roll = ts_roll.groupby("user_id" ).tail(TS_RECENCY_PERIOD)
 ts_roll = ts_roll.groupby("user_id")['timestamp'].apply(
 lambda x: np.concatenate([np.repeat(TS_ROLL_NULL_FILL, TS_RECENCY_PERIOD - len(x)) , x.values])
)
 ts_roll = parallelize_on_rows(ts_roll, pd.Series)
 gc.collect()
ts_roll = ts_roll.fillna(TS_ROLL_NULL_FILL)
 else:
ts_roll = pd.read_csv(".. /input/riiid-final-model-inputs/ts_roll.csv", index_col=0)
 print(f"Time Elapsed: {(time.time() - start_time):7.2f} Sec")
 print(f"Memory Usage: {ts_roll.memory_usage(deep=True ).sum() /(2**20):7.2f}", 'MB')
 print(f"Length : {len(ts_roll):7}")
 ts_roll.sample(5 )
age_cat=[]
for i in range(7):
i1=age_bins[i]
i2=age_bins[i+1]
age_cat.append(str(i)+" : "+i1.astype(str)+"-"+i2.astype(str)) 
<count_values>
%%time
user_df.to_csv("user-df.csv")
 content_df.to_csv("content-df.csv")
 u_roll.to_csv("u_roll.csv")
 repeat_c.to_pickle("repeat-c.pkl")
 pqet_roll.to_csv("pqet_roll.csv")
 ts_roll.to_csv("ts_roll.csv")
 if os.path.exists(".. /input/riiid-final-model-inputs/pq_shifted.feather"):
! cp.. /input/riiid-final-model-inputs/pq_shifted.feather./pq_shifted.feather
with open("sample-batches.pkl", 'wb')as f:
pickle.dump(batches, f )
df_trainval["Parch"].value_counts()<feature_engineering>
def insert_lecture(batches):
'A simple function to randomly insert a lecture in between, for debugging purposes!'
temp = deepcopy(batches)
 i = np.random.choice(len(temp))
j = np.random.choice(len(temp[i][0]))
print(f"Lecture inserted at {i+1} batch at {j} index!")
 temp[i][0].iloc[j, 4] = 1
temp[i][0].iloc[j, 3] = np.random.choice(lectures['lecture_id'].values)
 temp[i][0].iloc[j, 6] = np.nan
temp[i][0].iloc[j, 7] = np.nan
if len(temp)> i + 1:
pgac = eval(temp[i + 1][0].iloc[0]['prior_group_answers_correct'])
 pgr = eval(temp[i + 1][0].iloc[0]['prior_group_responses'])
 pgac[j] = -1
pgr[j] = -1
temp[i + 1][0].iloc[0, -2] = str(pgac)
 temp[i + 1][0].iloc[0, -1] = str(pgr)
 return temp
df_trainval["(Sex_labels+1)/Parch"]=(df_trainval["Sex_labels"]+1)/df_trainval["Pclass"]
df_trainval["Parch+SibSp+1"]=df_trainval["SibSp"]+df_trainval["Parch"]+1
df_trainval["Pclass/Parch"]=df_trainval["Parch"]/df_trainval["Pclass"]
df_trainval["Parch+SibSp+sex_label"]=df_trainval["SibSp"]+df_trainval["Parch"]+df_trainval["Sex_labels"]
df_trainval.head()<count_values>
SUBMIT = False
if not SUBMIT:
print("Validation Mode.")
 iter_test = iter(insert_lecture(batches))
op = []
else:
print("Prediction Mode.")
 iter_test = env.iter_test()
<feature_engineering><EOS> 
%%time
i, prev_test = 0, tuple()
for i, batch in enumerate(iter_test):
if len(prev_test):
processed_batch = post_process(prev_test[0], batch[0])
 ftrl.fit(dt.Frame(prev_test[1])[:, FTRL_COLS],
dt.Frame(processed_batch.loc[q_mask, 'answered_correctly']))
for _, ts, user, content, ans, pqhe, pqet in(
 processed_batch[[
'timestamp', 'user_id', 'content_id', 'answered_correctly',
'prior_question_had_explanation', 'prior_question_elapsed_time']].itertuples()):
org_ans = ans
if org_ans == -1:
part = lectures.loc[lectures.lecture_id == content, 'part'].values[0]
c_mean = None
else:
part = content_df.at[content, 'part']
c_mean = content_df.at[content, 'c_mean']
ans =(1 - c_mean)if ans == 1 else(- c_mean)
 pqet = pqet if not pd.isna(pqet)else np.nan
pqhe = int(pqhe)if not pd.isna(pqhe)else 0
if user not in user_df.index:
user_df.loc[user] = [0] * user_df.shape[1]
ts_roll.loc[user] =([TS_ROLL_NULL_FILL] *(TS_RECENCY_PERIOD - 1)) + [ts]
repeat_c.at[user] = to_ba([content])
 u_roll.loc[user] =([UROLL_NULL_FILL] *(ROLL_WINDOW - 1)) + [ans]
pqet_roll.loc[user] = [PQET_ROLL_NULL_FILL] * ROLL_WINDOW_PQET
user_df.at[user, 'uf_bundle'] = content_df.at[content, 'bundle_id']
else:
ts_roll.loc[user] = ts_roll.loc[user].values.tolist() [1:] + [ts]
if org_ans != -1:
u_roll.loc[user] = u_roll.loc[user].values.tolist() [1:] + [ans]
temp = repeat_c.at[user]
temp[content] = True
repeat_c.at[user] = temp
pqet_roll.loc[user] = pqet_roll.loc[user].values.tolist() [1:] + [pqet]
user_df.at[user, 'ts_max'] = ts
user_df.at[user, f'ts_max_{int(part)}'] = ts
user_df.at[user, 'uic'] = user_df.at[user, 'uic'] + 1
temp =(ts - user_df.at[user, 'ts_max'])> SESSION_DURATION
user_df.at[user, 'sess_max'] =(user_df.at[user, 'sess_max'] + 1 if temp else user_df.at[user, 'sess_max'])
 user_df.at[user, 'sess_event_sum'] = 0 if temp else user_df.at[user, 'sess_event_sum'] + 1
if org_ans == -1:
user_df.at[user, 'lec_recent'] = LEC_RECENT_ROLL
else:
temp = user_df.at[user, 'lec_recent']
user_df.at[user, 'lec_recent'] =(temp -1)if temp > 0 else 0
user_df.at[user, 'uc'] = user_df.at[user, 'uc'] + 1
user_df.at[user, 'us'] = user_df.at[user, 'us'] + ans
user_df.at[user, f'c{part}'] = user_df.at[user, f'c{part}'] + 1
user_df.at[user, f's{part}'] = user_df.at[user, f's{part}'] + ans
user_df.at[user, f'ups{part}'] = user_df.at[user, f'ups{part}'] + org_ans
user_df.at[user, 'ucs'] = user_df.at[user, 'ucs'] + org_ans
user_df.at[user, 'pqhe_sum'] = user_df.at[user, 'pqhe_sum'] + pqhe
if np.isnan(pqet):
user_df.at[user, 'pqet_sum'] = np.nan
elif np.isnan(user_df.at[user, 'pqet_sum']):
user_df.at[user, 'pqet_sum'] = pqet
else:
user_df.at[user, 'pqet_sum'] = user_df.at[user, 'pqet_sum'] + pqet
if pqhe:
if org_ans == 1:
user_df.at[user, 'seen_exp_when_right'] = user_df.at[user, 'seen_exp_when_right'] + 1
else:
user_df.at[user, 'seen_exp_when_wrong'] = user_df.at[user, 'seen_exp_when_wrong'] + 1
q_mask = batch[0]['content_type_id'] == 0
pred = batch[0].loc[q_mask, [
'row_id', 'timestamp', 'task_container_id', 'content_id',
'user_id', 'prior_question_elapsed_time']].reset_index()
pred = pd.concat([
pred, content_df.reindex(pred['content_id'] ).reset_index(drop=True)],
axis=1)
 pred[f'ummr_{ROLL_WINDOW}'] = u_roll.reindex(pred['user_id'] ).apply(
 lambda x: float(np.ma.masked_values(x, UROLL_NULL_FILL ).mean()), axis=1 ).values
pred[f'pqetmr_{ROLL_WINDOW_PQET}'] = pqet_roll.reindex(pred['user_id'] ).apply(
 lambda x: float(np.ma.masked_values(x, PQET_ROLL_NULL_FILL ).mean()), axis=1 ).values
pred['ts_recency_10'] = pred['timestamp'] - ts_roll.reindex(pred['user_id'] ).iloc[:, 0].values
pred['ts_recency_5'] = pred['timestamp'] - ts_roll.reindex(pred['user_id'] ).iloc[:, 4].values
temp = user_df.reindex(pred['user_id'] ).reset_index(drop=True)
 temp = pd.concat([temp, pred['part']], axis=1)
 temp['up_count'] = temp.apply(lambda x: x[f"c{int(x['part'])}"] , axis=1)
 temp['up_mean'] = temp.apply(lambda x: x[f"s{int(x['part'])}"] / x["up_count"], axis=1)
 temp['uph_mean'] = temp.apply(lambda x: x[f"ups{int(x['part'])}"] / x["up_count"], axis=1)
 temp['up_recency'] = temp.apply(lambda x: x[f"ts_max_{int(x['part'])}"], axis=1)
 temp['up_count'] = temp['up_count'] / temp['uc'].values
temp['uwrong_sum'] = temp['uc'] - temp['ucs'].values
temp['u_mean'] =(temp['us'] / temp['uc'].values)
 temp['h_mean'] =(temp['ucs'] / temp['uc'].values)
 temp['pqet_mean'] = temp['pqet_sum'] /(temp['uc'].values - 1)
 temp['seen_ratio'] = temp['pqhe_sum'] / temp['uc'].values
temp['lec_recent'] = temp['lec_recent'].fillna(0 ).astype(bool)
 temp = temp[['ts_max', 'up_count', 'up_mean', 'u_mean', 'uwrong_sum', 'lec_recent',
'h_mean', 'pqet_mean', 'seen_ratio', 'uic', 'uph_mean', 'uf_bundle',
'seen_exp_when_right', 'seen_exp_when_wrong', 'up_recency',
'sess_max', 'sess_event_sum']]
pred = pd.concat([pred, temp], axis=1 ).set_index("group_num")
 pred['uq_per_hr'] = pred['uic'] /(pred['timestamp'] / 60000)
 pred['h_mean'] =(2 * pred['h_mean'] * pred['c_mean'])/(pred['h_mean'] + pred['c_mean'])
 pred['uph_mean'] =(2 * pred['uph_mean'] * pred['c_mean'])/(pred['uph_mean'] + pred['c_mean'])
 pred['repeat_c'] = repeat_c.reindex(pred['user_id'] ).values
pred['repeat_c'] =(pred[['repeat_c', 'content_id']].apply(
 lambda x: x['repeat_c'][x['content_id']] if type(x['repeat_c'])!= float
else False, axis=1))
pred['ts_max'] = pred['ts_max'].fillna(0)
 pred['up_recency'] = pred['timestamp'] - pred['up_recency']
pred['response_time'] = pred['timestamp'] - pred['ts_max']
pred['rt_per_task'] = pred['response_time'] / pred['bundle_q_count']
pred['sessions'] =(pred['response_time'] > SESSION_DURATION ).astype(bool)
 pred['sess_event_count'] = np.where(pred['sessions'], 0, pred['sess_event_sum'] + 1)
 pred['sessions'] = pred['sess_max'] + pred['sessions']
pred['uf_bundle'] = pred['uf_bundle'].fillna(pred['bundle_id'])
 pred[['seen_exp_when_wrong', 'seen_exp_when_right', 'up_mean', 'up_count',
'up_recency', 'seen_ratio', 'rt_per_task', 'response_time', 'sessions',
'sess_event_count', 'uwrong_sum']] =(
 pred[['seen_exp_when_wrong', 'seen_exp_when_right', 'up_mean', 'up_count',
'up_recency', 'seen_ratio', 'rt_per_task', 'response_time', 'sessions',
'sess_event_count', 'uwrong_sum']].fillna(0))
pred['lgb_pred'] = model.predict(pred)
 pred['lgb_75'] = pred['lgb_pred'] > 0.75
pred['lgb_50'] = pred['lgb_pred'] > 0.5
pred['lgb_25'] = pred['lgb_pred'] > 0.25
pred['ummr_10_50'] = pred['ummr_10'] > 0
pred['h_mean_50'] = pred['h_mean'] > 0.5
pred['c_mean_50'] = pred['c_mean'] > 0.5
pred['c_mean_25'] = pred['c_mean'] > 0.25
pred['c_mean_75'] = pred['c_mean'] > 0.75
pred['pqet_proximity'] =(( pred['prior_question_elapsed_time'] - pred['right_et_med'] ).abs() < 
(pred['prior_question_elapsed_time'] - pred['wrong_et_med'] ).abs())
 df_to_dt_format(pred)
 pred['answered_correctly'] = ftrl.predict(dt.Frame(pred)[:, FTRL_COLS] ).to_list() [0]
if SUBMIT:
env.predict(pred[['row_id', 'answered_correctly']])
 else:
op.append(pred)
 prev_test =(batch[0].copy(deep=True), pred)
 else:
print(f"Sucessfully completed after {i + 1} iterations!" )