Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<count_unique_values>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
num_uniq = all_df.select_dtypes(include=['category'] ).nunique().sort_values()
index = num_uniq.index
num_uniq<count_unique_values>
import numpy as np
import random
import pandas as pd
import joblib
num_uniq = all_df.select_dtypes(include=['int64'] ).nunique().sort_values()
index = num_uniq.index
num_uniq<train_model>
_ = np.seterr(divide='ignore', invalid='ignore' )
model_lgb = lgb.LGBMRegressor(num_leaves=10,
max_depth=5,
learning_rate=0.1,
random_state=2)
 model_lgb.fit(train_df.drop(['Id', 'SalePrice'], axis=1), train_df.SalePrice )<compute_test_metric>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
rmsle_cv(model_lgb, train_df.drop('SalePrice', axis=1), train_df.SalePrice )<create_dataframe>
print('start read train data...')
 train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
df = pd.DataFrame(model_lgb.feature_importances_,
index=train_df.drop(['Id', 'SalePrice'], axis=1 ).columns,
columns=['importance'] ).sort_values('importance', ascending=False)
 df[df.importance > 10]<drop_column>
print('start handle lecture data...' )
imp_index = pd.Index([
'GrLivArea',
'TotalBsmtSF',
'LotArea',
'Neighborhood',
'OverallQual',
'OverallCond',
'GarageCars',
'MasVnrArea',
'YearBuilt',
'KitchenQual',
'YearRemodAdd',
'SalePrice',
] )<count_unique_values>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
train_df.loc[:, imp_index].nunique()<create_dataframe>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
c_train_df = train_df.copy()
c_test_df = test_df.copy()
c_all_df = all_df.copy()<drop_column>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
imp_index = pd.Index([
'GrLivArea',
'TotalBsmtSF',
'LotArea',
'Neighborhood',
'OverallQual',
'OverallCond',
'GarageCars',
'MasVnrArea',
'YearBuilt',
'KitchenQual',
'YearRemodAdd',
'SalePrice',
] )<sort_values>
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
doubtful_train_df = train_df[ 
(train_df.SalePrice > 700000)\
|(( train_df.GrLivArea > 4000)&(train_df.SalePrice < 200000))
|(( train_df.LotArea > 100000))
]
doubtful_train_df.loc[:, imp_index].sort_values('SalePrice', ascending=False )<drop_column>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )
c_all_df = c_all_df.drop(
 c_all_df[c_all_df.Id.isin(
 train_df[ 
(train_df.SalePrice > 700000)\
|(( train_df.GrLivArea > 4000)&(train_df.SalePrice < 200000))
].Id)].index
)<feature_engineering>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )
c_all_df['YearsSinceBuilt'] = c_all_df.YrSold.astype('int64')- c_all_df.YearBuilt
c_all_df.loc[c_all_df.YearsSinceBuilt <= 0, 'YearsSinceBuilt'] = 0<feature_engineering>
del(train_lectures)
 gc.collect()
c_all_df['YearsSinceRemod'] = c_all_df.YrSold.astype('int64')- c_all_df.YearRemodAdd
c_all_df.loc[c_all_df.YearRemodAdd == 1950, 'YearsSinceRemod'] = \
c_all_df.loc[c_all_df.YearRemodAdd == 1950, 'YearsSinceBuilt']
c_all_df.loc[c_all_df.YearsSinceRemod <= 0, 'YearsSinceRemod'] = 0<feature_engineering>
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg=user_lecture_agg.astype('int16' )
c_all_df['TotalSF'] = c_all_df['TotalBsmtSF'] + c_all_df['1stFlrSF'] + c_all_df['2ndFlrSF']<feature_engineering>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_interaction_timestamp_mean'] = train_df['timestamp']/cum['cumcount']
train_df['user_lecture_sum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_sum=train_df.user_lecture_sum.astype('int16')
 train_df.user_interaction_count=train_df.user_interaction_count.astype('int16')
 train_df['user_interaction_timestamp_mean']=train_df['user_interaction_timestamp_mean']/(1000*3600)
 train_df.user_interaction_timestamp_mean=train_df.user_interaction_timestamp_mean.astype('float32')
c_all_df['TotalBathrooms'] = \
c_all_df['FullBath'] +(0.5 * c_all_df['HalfBath'])+ \
c_all_df['BsmtFullBath'] +(0.5 * c_all_df['BsmtHalfBath'] )<feature_engineering>
del cum
gc.collect()
c_all_df['TotalPorchSf'] = \
c_all_df['OpenPorchSF'] + c_all_df['3SsnPorch'] + \
c_all_df['EnclosedPorch'] + c_all_df['ScreenPorch'] + c_all_df['WoodDeckSF']<feature_engineering>
print('start handle train_df...' )
<categorify>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True )
<drop_column>
content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'] )
c_all_df = c_all_df.drop(columns=['Street', 'PoolQC', 'Utilities'],axis=1 )<categorify>
content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']
c_all_df.LotShape.replace(to_replace=['IR3', 'IR2', 'IR1', 'Reg'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.LandContour.replace(to_replace=['Low', 'Bnk', 'HLS', 'Lvl'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.LandSlope.replace(to_replace=['Sev', 'Mod', 'Gtl'], value=[0, 1, 2], inplace=True)
 c_all_df.ExterQual.replace(to_replace=['Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.ExterCond.replace(to_replace=['Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.BsmtQual.replace(to_replace=['None', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.BsmtCond.replace(to_replace=['None', 'Po', 'Fa', 'TA', 'Gd'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.BsmtExposure.replace(to_replace=['None', 'No', 'Mn', 'Av', 'Gd'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.BsmtFinType1.replace(to_replace=['None', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], value=[0, 1, 2, 3, 4, 5, 6], inplace=True)
 c_all_df.BsmtFinType2.replace(to_replace=['None', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ'], value=[0, 1, 2, 3, 4, 5, 6], inplace=True)
 c_all_df.HeatingQC.replace(to_replace=['Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.Electrical.replace(to_replace=['Mix', 'FuseP', 'FuseF', 'FuseA', 'SBrkr'], value=[0, 1, 2, 3, 4], inplace=True)
 c_all_df.KitchenQual.replace(to_replace=['Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.Functional.replace(to_replace=['Sev', 'Maj2', 'Maj1', 'Mod', 'Min2', 'Min1', 'Typ'], value=[0, 1, 2, 3, 4, 5, 6], inplace=True)
 c_all_df.FireplaceQu.replace(to_replace=['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4, 5], inplace=True)
 c_all_df.GarageFinish.replace(to_replace=['None', 'Unf', 'RFn', 'Fin'], value=[0, 1, 2, 3], inplace=True)
 c_all_df.GarageQual.replace(to_replace=['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4, 5], inplace=True)
 c_all_df.GarageCond.replace(to_replace=['None', 'Po', 'Fa', 'TA', 'Gd', 'Ex'], value=[0, 1, 2, 3, 4, 5], inplace=True)
 c_all_df.PavedDrive.replace(to_replace=['N', 'P', 'Y'], value=[0, 1, 2], inplace=True)
 c_all_df.Fence.replace(to_replace=['None', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv'], value=[0, 1, 2, 3, 4], inplace=True )<feature_engineering>
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16' )
c_all_df.MSSubClass = c_all_df.MSSubClass.astype('int64')
 c_all_df.loc[c_all_df['MSSubClass']==60, 'MSSubClass'] = 0
c_all_df.loc[(c_all_df['MSSubClass']==20)|(c_all_df['MSSubClass']==120), 'MSSubClass'] = 1
c_all_df.loc[c_all_df['MSSubClass']==75, 'MSSubClass'] = 2
c_all_df.loc[(c_all_df['MSSubClass']==40)|(c_all_df['MSSubClass']==70)|(c_all_df['MSSubClass']==80), 'MSSubClass'] = 3
c_all_df.loc[(c_all_df['MSSubClass']==50)|(c_all_df['MSSubClass']==85)|(c_all_df['MSSubClass']==90)|(c_all_df['MSSubClass']==160)|(c_all_df['MSSubClass']==190), 'MSSubClass'] = 4
c_all_df.loc[(c_all_df['MSSubClass']==30)|(c_all_df['MSSubClass']==45)|(c_all_df['MSSubClass']==180), 'MSSubClass'] = 5
c_all_df.loc[(c_all_df['MSSubClass']==150), 'MSSubClass'] = 6<categorify>
print('start handle attempt_no...' )
c_all_df.GarageCars.replace(to_replace=[0, 1, 2, 3, 4, 5], value=[0, 1, 2, 3, 3, 3], inplace=True )<sort_values>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'] ).astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
skewness = c_all_df.select_dtypes(include=['int64'] ).apply(lambda x: skew(x))
skew_index = skewness[abs(skewness)>= 0.75].index
skewness[skew_index].sort_values(ascending=False )<feature_engineering>
print('start handle timestamp...')
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )
lam = 0.15
for column in skew_index:
c_all_df[column] = boxcox1p(c_all_df[column], lam )<count_missing_values>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
max_timestamp_u.user_id=max_timestamp_u.user_id.astype('int32' )
(all_df.isnull().sum() == 0 ).all()<merge>
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
max_timestamp_u2 = train_df[['user_id','lagtime']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u2.columns = ['user_id', 'max_time_stamp2']
max_timestamp_u2.user_id=max_timestamp_u2.user_id.astype('int32' )
c_train_df = pd.merge(c_all_df, train_df[['Id', 'SalePrice']], how='inner', on='Id')
 c_test_df = c_all_df[c_all_df.Id.isin(test_Id)]<import_modules>
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
lagtime_mean=train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True )
<prepare_x_and_y>
train_df['lagtime']=train_df['lagtime']/(1000*3600)
 train_df.lagtime=train_df.lagtime.astype('float32' )
c_train_X = c_train_df.drop(['Id', 'SalePrice'], axis=1)
 c_train_y = c_train_df.SalePrice
c_test_X = c_test_df.drop('Id', axis=1)

d_c_train_X = pd.get_dummies(c_train_X)
 d_c_test_X = pd.get_dummies(c_test_X )<train_on_grid>
def tune_hyperparameters(model, param_grid, train_X, train_y):
grid = GridSearchCV(
 model, param_grid,
scoring='neg_mean_squared_log_error',
cv=5, n_jobs=-1, 
)
 grid.fit(train_X, train_y)
 best_params = grid.best_params_
best_score = np.round(np.sqrt(-1 * grid.best_score_), 5)
 return best_params, best_score<train_model>
train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id=max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2']=train_df['timestamp']-train_df['lagtime2']
lagtime_mean2=train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True)
model = make_pipeline(
 RobustScaler() ,
LinearRegression() , 
)
 lr_model = MyEstimator(model)
 lr_model.fit(d_c_train_X, c_train_y)
 lr_eval = evaluation_model(lr_model, d_c_train_X, c_train_y, d_c_test_X)
 print(lr_eval )<train_on_grid>
train_df['lagtime2']=train_df['lagtime2']/(1000*3600)
 train_df.lagtime2=train_df.lagtime2.astype('float32' )
model = make_pipeline(
 RobustScaler() ,
LassoCV(
 alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10), 
), 
)
 lasso_cv_model = MyEstimator(model)
 lasso_cv_model.fit(d_c_train_X, c_train_y)
 lasso_cv_eval = evaluation_model(lasso_cv_model, d_c_train_X, c_train_y, d_c_test_X)
 print(lasso_cv_eval)
 opt_alpha = lasso_cv_model.model.steps[1][1].alpha_
print(f'
opt_alpha: {opt_alpha}' )<train_on_grid>
train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3']=train_df['timestamp']-train_df['lagtime3']
lagtime_mean3=train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3']=train_df['lagtime3']/(1000*3600)
 train_df.lagtime3=train_df.lagtime3.astype('float32' )
model = make_pipeline(
 RobustScaler() ,
RidgeCV(
 alphas=(0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1, 0.3, 1, 3, 5, 10), 
), 
)
 ridge_cv_model = MyEstimator(model)
 ridge_cv_model.fit(d_c_train_X, c_train_y)
 ridge_cv_eval = evaluation_model(ridge_cv_model, d_c_train_X, c_train_y, d_c_test_X)
 print(ridge_cv_eval)
 opt_alpha = ridge_cv_model.model.steps[1][1].alpha_
print(f'
opt_alpha: {opt_alpha}' )<train_on_grid>
model = make_pipeline(
 RobustScaler() ,
ElasticNetCV(
 alphas=(0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007),
l1_ratio=(0.8, 0.85, 0.9, 0.95, 0.99, 1), 
), 
)
 elsnt_cv_model = MyEstimator(model)
 elsnt_cv_model.fit(d_c_train_X, c_train_y)
 elsnt_cv_eval = evaluation_model(elsnt_cv_model, d_c_train_X, c_train_y, d_c_test_X)
 print(elsnt_cv_eval)
 opt_alpha = elsnt_cv_model.model.steps[1][1].alpha_
opt_l1_ratio = elsnt_cv_model.model.steps[1][1].l1_ratio_
print(f'
opt_alpha: {opt_alpha} opt_l1_ratio: {opt_l1_ratio}' )<train_model>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16' )
model = make_pipeline(
 RobustScaler() ,
SVR() , 
)
 svr_model = MyEstimator(model)
 param_grid = {'model__svr__C': [1, 10, 20],
'model__svr__epsilon': [0.001, 0.01, 0.1],
'model__svr__gamma': [0.0001, 0.001, 0.01]}
best_params, best_score = \
tune_hyperparameters(svr_model, param_grid, d_c_train_X, c_train_y)
 svr_model.set_params(**best_params)
 svr_model.fit(d_c_train_X, c_train_y)
 svr_eval = evaluation_model(svr_model, d_c_train_X, c_train_y, d_c_test_X)
 print(svr_eval)
 print(f'
grid best_params: {best_params}' )<train_model>
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
model = make_pipeline(
 RobustScaler() ,
KernelRidge() , 
)
 kr_model = MyEstimator(model)
 param_grid = {'model__kernelridge__alpha': [0.01, 0.1, 0.5, 1],
'model__kernelridge__kernel': ['linear', 'polynomial'],
'model__kernelridge__degree': [1, 1.5, 2, 3],
'model__kernelridge__coef0': [3, 4, 5]}
best_params, best_score = \
tune_hyperparameters(kr_model, param_grid, d_c_train_X, c_train_y)
 kr_model.set_params(**best_params)
 kr_model.fit(d_c_train_X, c_train_y)
 kr_eval = evaluation_model(kr_model, d_c_train_X, c_train_y, d_c_test_X)
 print(kr_eval)
 print(f'
grid best_params: {best_params}' )<train_model>
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )
model = make_pipeline(
 RobustScaler() ,
KNeighborsRegressor() , 
)
 knn_model = MyEstimator(model)
 param_grid = {'model__kneighborsregressor__n_neighbors': [3, 5, 7]}
best_params, best_score = \
tune_hyperparameters(knn_model, param_grid, d_c_train_X, c_train_y)
 knn_model.set_params(**best_params)
 knn_model.fit(d_c_train_X, c_train_y)
 knn_eval = evaluation_model(knn_model, d_c_train_X, c_train_y, d_c_test_X)
 print(knn_eval)
 print(f'
grid best_params: {best_params}' )<train_model>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16')
model = ExtraTreesRegressor(n_jobs=-1, n_estimators=100)
 et_model = MyEstimator(model)
 param_grid = {'model__max_depth': [1, 3, 5]}
best_params, best_score = \
tune_hyperparameters(et_model, param_grid, d_c_train_X, c_train_y)
 et_model.set_params(**best_params)
 et_model.fit(d_c_train_X, c_train_y)
 et_eval = evaluation_model(et_model, d_c_train_X, c_train_y, d_c_test_X)
 print(et_eval)
 print(f'
grid best_params: {best_params}' )<choose_model_class>
del cum
gc.collect()
model = XGBRegressor(learning_rate=0.01, n_estimators=3000,
max_depth=3,
nthread=-1,
objective='reg:squarederror',
min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
scale_pos_weight=1,
reg_alpha=0.00006)
 xgb_model = MyEstimator(model)
 xgb_model.fit(d_c_train_X, c_train_y)
 xgb_eval = evaluation_model(xgb_model, d_c_train_X, c_train_y, d_c_test_X)
 print(xgb_eval )<choose_model_class>
model = LGBMRegressor(learning_rate=0.01, n_estimators=3000,
num_leaves=5,
objective='regression',
max_bin=55, bagging_fraction=0.8,
bagging_freq=5, feature_fraction=0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=6, min_sum_hessian_in_leaf=11)
 lgb_model = MyEstimator(model)
 lgb_model.fit(d_c_train_X, c_train_y)
 lgb_eval = evaluation_model(lgb_model, d_c_train_X, c_train_y, d_c_test_X)
 print(lgb_eval )<define_variables>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
models = [ 
('lr_model', lr_eval), 
('lasso_cv_model', lasso_cv_eval), 
('ridge_cv_model', ridge_cv_eval), 
('elsnt_cv_model', elsnt_cv_eval), 
('svr_model', svr_eval), 
('kr_model', kr_eval), 
('knn_model', knn_eval), 
('et_model', et_eval), 
('xgb_model', xgb_eval), 
('lgb_model', lgb_eval),
]
for mname, meval in models:
print(f'[{mname}]')
 print(meval )<train_on_grid>
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16' )
stack_cv_model = StackingCVRegressor(
 regressors=(ridge_cv_model, lasso_cv_model, elsnt_cv_model,
svr_model, kr_model, xgb_model, lgb_model),
meta_regressor=lgb_model,
cv=5,
n_jobs=-1,
use_features_in_secondary=True)
 stack_cv_model.fit(np.array(d_c_train_X), np.array(c_train_y))
stack_cv_eval = evaluation_model(stack_cv_model, np.array(d_c_train_X), np.array(c_train_y), np.array(d_c_test_X))
print(stack_cv_eval )<save_to_csv>
del cum
gc.collect()
def output_submission_file(test_X, filename='submission.csv'):
prediction = blend_models_predict(test_X)
 df = pd.DataFrame({'Id': test_Id.values, 'SalePrice': prediction})
 print(f'{df.shape}')
 print(f'{df.head(5)}')
 df.to_csv(filename, index=False)
 df.to_csv(filename + '.gz', index=False, compression='gzip' )<import_modules>
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
import pandas as pd
import numpy as np<import_modules>
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )
import pandas as pd
import numpy as np<load_from_csv>
content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'] )
test = pd.read_csv('.. /input/titanic/test.csv')
 train = pd.read_csv('.. /input/titanic/train.csv' )<prepare_output>
print('start questions data...' )
testPassengerIds = test['PassengerId']
testPassengerIds.head()<drop_column>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}
)
train.drop(['PassengerId', 'Name', 'Ticket'], inplace = True, axis = 1)
 test.drop(['PassengerId', 'Name', 'Ticket'], inplace = True, axis = 1 )<count_missing_values>
bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'] )
train.isnull().any()<count_missing_values>
questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8' )
test.isnull().any()<feature_engineering>
questions_df['tags'].fillna('188', inplace=True )
train['Cabin'].fillna(0, inplace = True)
 def getCabin(value):
val_dict = {
'A' : 6,
'B' : 5,
'C' : 4,
'D' : 3,
'E' : 2,
'F' : 1,
'T' : 1
}
return val_dict.get(str(value)[0], 0)
 train['Cabin'] = train["Cabin"].apply(getCabin)
 test['Cabin'] = test['Cabin'].apply(getCabin )<categorify>
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result
train['Embarked'].fillna(train['Embarked'].mode().item() , inplace = True)
 test['Embarked'].fillna(train['Embarked'].mode().item() , inplace = True )<categorify>
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform )
train = pd.get_dummies(train, drop_first=True)
 test = pd.get_dummies(test, drop_first=True )<count_missing_values>
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )
train.isnull().any() | test.isnull().any()<feature_engineering>
questions_df.drop(columns=['tags'], inplace=True )
def correctedLog(value):
return np.log(1 + value)
 train['Status'] = train['Pclass'] + train['Fare'].apply(correctedLog)
 test['Status'] = test['Pclass'] + test['Fare'].apply(correctedLog )<feature_engineering>
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
train['RootAgeTimesClass'] = train['Age'].apply(np.sqrt)* train['Pclass']
test['RootAgeTimesClass'] = test['Age'].apply(np.sqrt)* test['Pclass']<feature_engineering>
train['FamilySize'] = train['SibSp'] + train['Parch'] + 1
test['FamilySize'] = train['SibSp'] + train['Parch'] + 1<feature_engineering>
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )
train['Young'] = train['Age'] <= train['Age'].mean()
test['Young'] = test["Age"] <= train['Age'].mean()<feature_engineering>
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
train['YoungMale'] = train['Young'] & train['Sex_male']
test['YoungMale'] = test["Young"] & test['Sex_male']<prepare_x_and_y>
del content_explation_agg
X = train.iloc[:, 1: ]
y = train.iloc[:, 0]
y.shape, X.shape<feature_engineering>
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )
train['Fare'], maxlog = boxcox(train['Fare'] + 1)
 test['Fare'] = boxcox(test['Fare'] + 1, lmbda = maxlog )<import_modules>
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score<split>
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()
X_train, X_val, y_train, y_val = train_test_split(X, y, random_state =55, test_size = 0.2, shuffle = True)
 y_train.shape, y_val.shape<set_options>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )
np.random.seed(0 )<choose_model_class>
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )
paramDict = {
'n_estimators' : [5, 10, 25, 50, 75, 100, 200, 500],
'max_depth' : [4, 8, 10, 15, 20, 50],
}
model = RandomForestClassifier(n_jobs = 8)
 clf = GridSearchCV(estimator=model, param_grid=paramDict, n_jobs=10 )<train_model>
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
clf.fit(X_train, y_train )<find_best_params>
clf.best_params_, clf.best_score_<predict_on_test>
del content_agg
del bundle_agg
del part_agg
gc.collect()
f1_score(clf.predict(X_val), y_val )<save_to_csv>
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
finalModel = RandomForestClassifier(**clf.best_params_)
 finalModel.fit(X, y)
 y_preds = finalModel.predict(test)
 file_name = "Submission_16_08_6.csv"
y_pred_series = pd.Series(y_preds.flatten() , name = 'Survived')
 file = pd.concat([testPassengerIds, y_pred_series], axis = 1)
 file.to_csv(file_name, index = False);<import_modules>
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 200,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df))
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
del train_df_clf
del valid_df
gc.collect()
gender_submission = pd.read_csv(".. /input/titanic/gender_submission.csv")
 test_raw = pd.read_csv(".. /input/titanic/test.csv", index_col='PassengerId')
 train_raw = pd.read_csv(".. /input/titanic/train.csv",index_col='PassengerId' )<prepare_x_and_y>
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.Booster(model_file='.. /input/savemodel/lgbm_model.txt')
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect()
X=train_raw.drop('Survived', axis=1)
 y=train_raw['Survived']
X_cb=pd.concat([X, test_raw], axis=0)
 test=test_raw
train_idx=list(train_raw.index)
 test_idx=list(test_raw.index )<sort_values>
MAX_SEQ = 128
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 nn_model = SAKTModel(n_skill, embed_dim=128)
 try:
nn_model.load_state_dict(torch.load(".. /input/saktttmodel/sakt_model.pt"))
except:
nn_model.load_state_dict(torch.load(".. /input/saktttmodel/sakt_model.pt", map_location='cpu'))
nn_model.to(device)
 nn_model.eval()
X_cb.isnull().sum().sort_values(ascending=False )<groupby>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
class_fare=test.groupby(['Pclass','Embarked'])['Fare'].mean()
test['Fare'].fillna(class_fare[3]['S'], inplace=True )<count_values>
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
X['Embarked'].value_counts()<correct_missing_values>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()
X['Embarked'].fillna(X['Embarked'].mode() [0], inplace=True )<feature_engineering>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()
X[['Last','First']]=X['Name'].str.split(',', expand=True)
 X[['Title', 'FName']]=X['First'].str.split('.', n=1, expand=True)
 X['Title']=X['Title'].str.strip()
test[['Last','First']]=test['Name'].str.split(',', expand=True)
 test[['Title', 'FName']]=test['First'].str.split('.', n=1, expand=True)
 test['Title']=test['Title'].str.strip()<feature_engineering>
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
X['Adult']=''
for i in list(X.index):
if X['Age'].loc[i]>8:
X['Adult'].loc[i]=1
elif X['Age'].loc[i]<=8:
X['Adult'].loc[i]=0
else:
if X['Title'].loc[i]=='Master':
X['Adult'].loc[i]=0
else:
X['Adult'].loc[i]=1
test['Adult']=''
for i in list(test.index):
if test['Age'].loc[i]>8:
test['Adult'].loc[i]=1
elif test['Age'].loc[i]<=8:
test['Adult'].loc[i]=0
else:
if test['Title'].loc[i]=='Master':
test['Adult'].loc[i]=0
else:
test['Adult'].loc[i]=1<merge>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
X['Family_Size']=X['SibSp']+X['Parch']+1
X["With_Family"]=X.apply(lambda row: 1 if row['Family_Size']>1 else 0, axis=1)
 test['Family_Size']=test['SibSp']+test['Parch']+1
test["With_Family"]=test.apply(lambda row: 1 if row['Family_Size']>1 else 0, axis=1)
 count=X['Ticket'].value_counts().to_frame().reset_index()
count.columns=['Ticket','Person_on_Ticket']
X=pd.merge(X,count,how='left',on='Ticket' ).set_index(X.index)
 count_t=test['Ticket'].value_counts().to_frame().reset_index()
count_t.columns=['Ticket','Person_on_Ticket']
test=pd.merge(test,count_t,how='left',on='Ticket' ).set_index(test.index )<drop_column>
X.drop(['Name','Age','Ticket','First','FName','Cabin'], axis=1, inplace=True)
 test.drop(['Name','Age','Ticket','First','FName','Cabin'], axis=1, inplace=True )<categorify>
iter_test = env.iter_test()
prior_test_df = None
prev_test_df1 = None
X=pd.get_dummies(X)
 test=pd.get_dummies(test)
 X.head()<drop_column>
N=[0.4,0.6]
<train_model><EOS> 
%%time
for(test_df, sample_prediction_df)in iter_test:
test_df1=test_df.copy()
if(prev_test_df1 is not None):
prev_test_df1['answered_correctly'] = eval(test_df1['prior_group_answers_correct'].iloc[0])
 prev_test_df1 = prev_test_df1[prev_test_df1.content_type_id == False]
prev_group = prev_test_df1[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df1 = test_df1.copy()
test_df1 = test_df1[test_df1.content_type_id == False]
test_dataset = TestDataset(group, test_df1, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = nn_model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prior_test_df = test_df.copy()
question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
o2=sub_preds / len(clfs)
 test_df[target]=0.5*np.array(outs)+0.5*np.array(o2)
 env.predict(test_df[['row_id', target]] )