Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<normalization>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
forest_reg = RandomForestRegressor(n_estimators=40, random_state=42,
min_samples_leaf=3, max_features=0.5, n_jobs=-1, oob_score=True)
 modelfit(forest_reg, train_dummies_prepared, train_labels )<normalization>
import numpy as np
import random
import pandas as pd
import joblib
tree_ada = DecisionTreeRegressor(random_state = 42,max_depth = 4)
 ada_reg = AdaBoostRegressor(
 tree_reg, n_estimators=3000, random_state=42,learning_rate=0.009, loss='square')
 modelfit(ada_reg, train_dummies_prepared, train_labels )<normalization>
_ = np.seterr(divide='ignore', invalid='ignore' )
params = {'n_estimators': 500, 'max_depth': 4, 'min_samples_split': 2,
'learning_rate': 0.01, 'loss': 'ls'}
params_1 = {'n_estimators': 3000, 'learning_rate': 0.05, 'max_depth' : 4,
'max_features': 'sqrt', 'min_samples_leaf': 15, 'min_samples_split': 10,
'loss': 'huber', 'random_state': 42}
train_dummies_prepared = num_pipeline.fit_transform(train_dummies)
 gdb_model = GradientBoostingRegressor(**params_1)
 modelfit(gdb_model, train_dummies_prepared, train_labels )<train_model>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
def indices_of_top_k(arr, k):
return np.sort(np.argpartition(np.array(arr), -k)[-k:])
 class TopFeatureSelector(BaseEstimator, TransformerMixin):
def __init__(self, feature_importances, k):
self.feature_importances = feature_importances
self.k = k
def fit(self, X, y=None):
self.feature_indices_ = indices_of_top_k(self.feature_importances, self.k)
 return self
def transform(self, X):
return X[:, self.feature_indices_]<choose_model_class>
print('start read train data...')
 train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
params = {'objective': 'reg:linear',
'eta': 0.01,
'max_depth': 6,
'subsample': 0.6,
'colsample_bytree': 0.7,
'eval_metric': 'rmse',
'seed': 42,
'silent': True,
}
model_xgb = xgb.XGBRegressor(**params,n_estimators=3000,learning_rate=0.05,verbose_eval=True )<prepare_x_and_y>
print('start handle lecture data...' )
fold_val_pred = []
fold_err = []
fold_importance = []
record = dict()
kfold = list(KFold(10,shuffle = True, random_state = 42 ).split(train_dummies))
def xgb_model(train, params, train_label, fold, verbose, pipeline):
for i,(trn, val)in enumerate(fold):
print(i+1, "fold.RMSE")
 trn_x = train.loc[trn, :]
trn_y = train_label[trn]
val_x = train.loc[val, :]
val_y = train_label[val]
fold_val_pred = []
start = datetime.now()
model = xgb.train(params
, xgb.DMatrix(trn_x, trn_y)
 , 3500
, [(xgb.DMatrix(trn_x, trn_y), 'train'),(xgb.DMatrix(val_x, val_y), 'valid')]
, verbose_eval=verbose
, early_stopping_rounds=500
, callbacks = [xgb.callback.record_evaluation(record)])
 best_idx = np.argmin(np.array(record['valid']['rmse']))
val_pred = model.predict(xgb.DMatrix(val_x), ntree_limit=model.best_ntree_limit)
 fold_val_pred.append(val_pred*0.2)
 fold_err.append(record['valid_0']['rmse'][best_idx])
 fold_importance.append(model.feature_importance('gain'))
print("xgb model.", "{0:.5f}".format(result['error']), '(' + str(int(( datetime.now() -start ).seconds/60)) + 'm)' )<normalization>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
modelfit(model_xgb, train_dummies, train_labels )<define_search_space>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
pipe = Pipeline([ 
('pca', PCA(5)) , 
('xgb', xgb.XGBClassifier())
 ])
 param_grid = {
'pca__n_components': [3, 5, 7, 10,15],
'xgb__n_estimators': [500,1000,2000,3500],
'xgb__learning_rate': [0.09,0.1,0.05,0.001]
} 
<choose_model_class>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
lgb_model = lgb.LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.005,
n_estimators=10000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 modelfit(lgb_model, train_dummies, train_labels )<train_model>
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1 )<choose_model_class>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )
models_total =(model_kernel, model_lasso, model_elastic, gdb_model, lgb_model)
 model_ada_gbd =(ada_reg,gdb_model)
 stack_gen = StackingCVRegressor(regressors= models_total,
meta_regressor=model_xgb,
use_features_in_secondary=True )<concatenate>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )
modelfit(stack_gen, train_dummies_prepared, train_labels )<categorify>
del(train_lectures)
 gc.collect()
test_dummies_prepared = num_pipeline.fit_transform(test_dummies)
<predict_on_test>
user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'])
 user_lecture_agg=user_lecture_agg.astype('int16' )
prediction_some_regressors = stack_gen.predict(test_dummies_prepared )<save_to_csv>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['user_interaction_count'] = cum['cumcount']
train_df['user_interaction_timestamp_mean'] = train_df['timestamp']/cum['cumcount']
train_df['user_lecture_sum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_sum=train_df.user_lecture_sum.astype('int16')
 train_df.user_interaction_count=train_df.user_interaction_count.astype('int16')
 train_df['user_interaction_timestamp_mean']=train_df['user_interaction_timestamp_mean']/(1000*3600)
 train_df.user_interaction_timestamp_mean=train_df.user_interaction_timestamp_mean.astype('float32')
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['SalePrice'] = np.expm1(prediction_some_regressors)
 sub.to_csv("stack_regression.csv", index=False )<set_options>
del cum
gc.collect()
%matplotlib inline
sns.set(style="whitegrid")
 warnings.filterwarnings('ignore')
 pd.set_option('max_colwidth',80 )<load_from_csv>
print('start handle train_df...' )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_combine = df_train.append(df_test, sort=False )<create_dataframe>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True )
feat_desc = pd.DataFrame({'Description': ['Passenger ID',
'Whether the passenger was survived or not',
'The ticket class that the passenger bought',
'The passenger name',
'The gender of the passenger',
'The age of the passenger',
'The number of siblings/spouses that the passenger has aboard the Titanic',
'The number of parents/children that the passenger has aboard the Titanic',
'The ticket number of the passenger',
'The ticket fare that the passenger paid',
'The cabin number that the passenger boarded',
'The passenger port of embarkation'],
'Values': [df_train[i].unique() for i in df_train.columns],
'Number of unique values': [len(df_train[i].unique())for i in df_train.columns]},
index = df_train.columns)
 feat_desc<concatenate>
content_explation_agg=train_df[["content_id","prior_question_had_explanation",target]].groupby(["content_id","prior_question_had_explanation"])[target].agg(['mean'] )
df_train.set_index('PassengerId', inplace=True)
 df_test.set_index('PassengerId', inplace=True)
 df_combine = df_train.append(df_test, sort=False )<count_missing_values>
content_explation_agg=content_explation_agg.unstack()
content_explation_agg=content_explation_agg.reset_index()
content_explation_agg.columns = ['content_id', 'content_explation_false_mean','content_explation_true_mean']
pd.DataFrame({'Number of Missing Values(Training)': df_train.isna().sum() ,
'% of Missing Values(Training)':(df_train.isna().sum() /df_train.shape[0] * 100 ).round(2),
'Number of Missing Values(Test)': df_test.isna().sum().round(0),
'% of Missing Values(Test)':(df_test.isna().sum() /df_test.shape[0] * 100 ).round(2)} )<drop_column>
content_explation_agg.content_id=content_explation_agg.content_id.astype('int16')
 content_explation_agg.content_explation_false_mean=content_explation_agg.content_explation_false_mean.astype('float16')
 content_explation_agg.content_explation_true_mean=content_explation_agg.content_explation_true_mean.astype('float16' )
df_train['Cabin'].fillna('Z', inplace=True)
 df_test['Cabin'].fillna('Z', inplace=True )<data_type_conversions>
print('start handle attempt_no...' )
df_combine['Cabin'] = df_train['Cabin'].str.get(0)
 df_combine.groupby('Cabin')['Pclass'].value_counts().to_frame('Count' )<filter>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'] ).astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
df_train.loc[df_train['Embarked'].isna() ]<feature_engineering>
print('start handle timestamp...')
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )
df_train.loc[df_train['Embarked'].isna() , 'Embarked'] = 'S'<filter>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
max_timestamp_u.user_id=max_timestamp_u.user_id.astype('int32' )
df_test.loc[df_test['Fare'].isna() ]<feature_engineering>
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
max_timestamp_u2 = train_df[['user_id','lagtime']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u2.columns = ['user_id', 'max_time_stamp2']
max_timestamp_u2.user_id=max_timestamp_u2.user_id.astype('int32' )
df_test.loc[df_test['Fare'].isna() ] = df_train['Fare'].mean()<choose_model_class>
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
lagtime_mean=train_df['lagtime'].mean()
train_df['lagtime'].fillna(lagtime_mean, inplace=True )
df_age = df_train.loc[~df_train['Age'].isna() ]
reg = LinearRegression()
reg.fit(df_age[['SibSp', 'Pclass']], df_age['Age'])
 pred_age_train = pd.Series(reg.predict(df_train[['SibSp', 'Pclass']]), index=df_train.index)
 pred_age_test = pd.Series(reg.predict(df_test[['SibSp', 'Pclass']]), index=df_test.index)
 df_train['Age'].fillna(pred_age_train, inplace=True)
 df_test['Age'].fillna(pred_age_test, inplace=True )<filter>
train_df['lagtime']=train_df['lagtime']/(1000*3600)
 train_df.lagtime=train_df.lagtime.astype('float32' )
df_train.loc[df_train['Age'] < 0]<filter>
df_test.loc[df_test['Age'] < 0]<feature_engineering>
train_df['lagtime2'] = train_df.groupby('user_id')['timestamp'].shift(2)
 max_timestamp_u3 = train_df[['user_id','lagtime2']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u3.columns = ['user_id', 'max_time_stamp3']
max_timestamp_u3.user_id=max_timestamp_u3.user_id.astype('int32')
 train_df['lagtime2']=train_df['timestamp']-train_df['lagtime2']
lagtime_mean2=train_df['lagtime2'].mean()
train_df['lagtime2'].fillna(lagtime_mean2, inplace=True)
df_train.loc[df_train['Age'] < 0, 'Age'] = df_train['Age'].mean()
df_test.loc[df_test['Age'] < 0, 'Age'] = df_train['Age'].mean()<data_type_conversions>
train_df['lagtime2']=train_df['lagtime2']/(1000*3600)
 train_df.lagtime2=train_df.lagtime2.astype('float32' )
df_train['Age'] = df_train['Age'].round().astype('int')
 df_test['Age'] = df_test['Age'].round().astype('int' )<feature_engineering>
train_df['lagtime3'] = train_df.groupby('user_id')['timestamp'].shift(3)
 train_df['lagtime3']=train_df['timestamp']-train_df['lagtime3']
lagtime_mean3=train_df['lagtime3'].mean()
train_df['lagtime3'].fillna(lagtime_mean3, inplace=True)
 train_df['lagtime3']=train_df['lagtime3']/(1000*3600)
 train_df.lagtime3=train_df.lagtime3.astype('float32' )
df_train['Title'] = df_train['Name'].str.split(',', expand=True)[1].str.split('.' ).str.get(0)
 df_test['Title'] = df_test['Name'].str.split(',', expand=True)[1].str.split('.' ).str.get(0 )<count_values>
df_train['Title'].value_counts().to_frame('Number of Passengers' ).T<feature_engineering>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('float16' )
df_train['SibSp+Parch'] = df_train['SibSp'] + df_train['Parch']
df_test['SibSp+Parch'] = df_test['SibSp'] + df_test['Parch']<feature_engineering>
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
df_train['IsAlone'] = df_train['SibSp+Parch'].map(lambda x: 1 if x == 0 else 0)
 df_test['IsAlone'] = df_test['SibSp+Parch'].map(lambda x: 1 if x == 0 else 0 )<concatenate>
delta_prior_question_elapsed_time_mean=train_df['delta_prior_question_elapsed_time'].mean()
train_df['delta_prior_question_elapsed_time'].fillna(delta_prior_question_elapsed_time_mean, inplace=True)
 train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )
train_size = df_train.shape[0]
test_size = df_test.shape[0]
df_combine = df_train.append(df_test, sort=False)
 df_combine['Last_Name'] = df_combine['Name'].str.split(',', expand=True)[0]<groupby>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 user_agg = train_df.groupby('user_id')['lag'].agg(['sum', 'count'] ).astype('int16')
 cum['cumsum'].fillna(0, inplace=True)
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_count'] = cum['cumsum']
train_df['user_uncorrect_count'] = cum['cumcount']-cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correctness'].fillna(0.67, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_count=train_df.user_correct_count.astype('int16')
 train_df.user_uncorrect_count=train_df.user_uncorrect_count.astype('int16')
fare_df = df_combine.loc[df_combine['SibSp+Parch'] > 0, ['Last_Name', 'Fare', 'SibSp+Parch']]
fare_diff =(fare_df.groupby(['Last_Name', 'SibSp+Parch'])['Fare'].aggregate('max')- fare_df.groupby(['Last_Name', 'SibSp+Parch'])['Fare'].aggregate('min')).value_counts()
print('Percentage of families with the same fare: {:.2f}%'.format(fare_diff[0]/fare_diff.sum() *100))<data_type_conversions>
del cum
gc.collect()
train_temp_df = df_combine.iloc[:train_size]
family_group_df = train_temp_df.loc[train_temp_df['SibSp+Parch']>0,
['Last_Name', 'Fare', 'SibSp+Parch', 'Survived']].groupby(['Last_Name', 'Fare'])
 family_df = pd.DataFrame(data=family_group_df.size() , columns=['Size_in_training_dataset'])
 family_df['Survived_Total'] = family_group_df['Survived'].sum().astype('int')
 family_df['SibSp+Parch'] = family_group_df['SibSp+Parch'].mean().astype('int')
 all_survived =(family_df['Size_in_training_dataset'] == family_df['Survived_Total'] ).sum() /len(family_df)*100
print('Families with the whole members survived: {:.1f}%'.format(all_survived))
all_not_survived =(family_df['Survived_Total']==0 ).sum() /len(family_df)*100
print('Families with the whole members not survived: {:.1f}%'.format(all_not_survived))<feature_engineering>
df_combine['FamilySurvival'] = 0.5
for _, grp_df in df_combine[['Survived', 'Last_Name', 'Fare']].groupby(['Last_Name', 'Fare']):
if len(grp_df)> 1:
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
if smax == 1:
df_combine.loc[ind, 'FamilySurvival'] = 1
elif smin == 0:
df_combine.loc[ind, 'FamilySurvival'] = 0<data_type_conversions>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
train_temp_df = df_combine.iloc[:train_size]
ticket_group_df = train_temp_df.groupby('Ticket')
 ticket_df = pd.DataFrame(data=ticket_group_df.size() , columns=['Size_in_training_dataset'])
 ticket_df['Survived_Total'] = ticket_group_df['Survived'].sum().astype('int')
 ticket_df['Not_Family'] = ticket_group_df['Last_Name'].unique().apply(len)
 ticket_df = ticket_df.loc[(ticket_df['Size_in_training_dataset'] > 1)&(ticket_df['Not_Family'] > 1)]
print('Number of groups in training set that is not family: {}'.format(len(ticket_df)))
 all_survived =(ticket_df['Size_in_training_dataset'] == ticket_df['Survived_Total'] ).sum() /len(ticket_df)*100
print('Families with the whole members survived: {:.1f}%'.format(all_survived))
all_not_survived =(ticket_df['Survived_Total'] == 0 ).sum() /len(ticket_df)*100
print('Families with the whole members not survived: {:.1f}%'.format(all_not_survived))<groupby>
cum = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['cumsum', 'cumcount'])
 cum['cumcount']=cum['cumcount']+1
train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_true_count'] = cum['cumsum']
train_df['explanation_false_count'] = cum['cumcount']-cum['cumsum']
train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_true_count=train_df.explanation_true_count.astype('int16')
 train_df.explanation_false_count=train_df.explanation_false_count.astype('int16' )
for grp, grp_df in df_combine.groupby('Ticket'):
if len(grp_df)> 1:
for ind, row in grp_df.iterrows() :
if(row['FamilySurvival'])== 0 or(row['FamilySurvival'] == 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
if smax == 1:
df_combine.loc[ind, 'FamilySurvival'] = 1
elif smin == 0:
df_combine.loc[ind, 'FamilySurvival'] = 0
df_train['FamilySurvival'] = df_combine.iloc[:train_size]['FamilySurvival']
df_test['FamilySurvival'] = df_combine.iloc[train_size:]['FamilySurvival']<feature_engineering>
del cum
gc.collect()
df_combine['RealFare'] = 0
for _, grp_df in df_combine.groupby(['Ticket']):
grp_size = len(grp_df)
 for ind, row in grp_df.iterrows() :
real_fare = row['Fare']/grp_size
df_combine.loc[ind, 'RealFare'] = real_fare
df_train['Fare'] = df_combine.iloc[:train_size]['RealFare']
df_test['Fare'] = df_combine.iloc[train_size:]['RealFare']<create_dataframe>
content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
df_train.dtypes.to_frame(name='Data type' )<categorify>
train_df['task_container_uncor_count'] = train_df['task_container_id'].map(task_container_agg['count']-task_container_agg['sum'] ).astype('int32')
 train_df['task_container_cor_count'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )
df_train.replace({'male': 1, 'female': 0}, inplace=True)
 df_test.replace({'male': 1, 'female': 0}, inplace=True )<categorify>
content_elapsed_time_agg=train_df.groupby('content_id')['prior_question_elapsed_time'].agg(['mean'])
 content_had_explanation_agg=train_df.groupby('content_id')['prior_question_had_explanation'].agg(['mean'] )
df_train.replace({'S': 0, 'C': 1, 'Q': 2}, inplace=True)
 df_test.replace({'S': 0, 'C': 1, 'Q': 2}, inplace=True )<drop_column>
print('start questions data...' )
df_train.drop(columns=['Name', 'Ticket'], inplace=True)
 df_test.drop(columns=['Name', 'Ticket'], inplace=True )<create_dataframe>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'}
)
df_train['Title'].value_counts().to_frame('Number of Passengers' ).T<count_values>
bundle_agg = questions_df.groupby('bundle_id')['question_id'].agg(['count'] )
df_test['Title'].value_counts().to_frame('Number of Passengers' ).T<feature_engineering>
questions_df['content_sub_bundle'] = questions_df['bundle_id'].map(bundle_agg['count'] ).astype('int8' )
df_train['Title'] = df_train['Title'].str.strip().map(lambda x: x if x == 'Mr' or x == 'Miss' or x == 'Mrs' or x == 'Master' else 'Other')
 df_test['Title'] = df_test['Title'].str.strip().map(lambda x: x if x == 'Mr' or x == 'Miss' or x == 'Mrs' or x == 'Master' else 'Other' )<categorify>
questions_df['tags'].fillna('188', inplace=True )
df_train = df_train.join(pd.get_dummies(df_train['Title'], prefix='Title'), how='outer')
 df_test = df_test.join(pd.get_dummies(df_test['Title'], prefix='Title'), how='outer' )<drop_column>
def gettags(tags,num):
tags_splits=tags.split(" ")
 result=''
for t in tags_splits:
x=int(t)
 if(x<32*(num+1)and x>=32*num):
result=result+' '+t
return result
df_train.drop(columns=['Title'], inplace=True)
 df_test.drop(columns=['Title'], inplace=True )<feature_engineering>
for num in range(0,6):
questions_df["tags"+str(num)] = questions_df["tags"].apply(lambda row: gettags(row,num))
le = LabelEncoder()
le.fit(np.unique(questions_df['tags'+str(num)].values))
questions_df['tags'+str(num)]=questions_df[['tags'+str(num)]].apply(le.transform )
df_train['Age_binned'] = pd.cut(df_train['Age'], np.arange(0, 85, 5), include_lowest=True)
 df_test['Age_binned'] = pd.cut(df_test['Age'], np.arange(0, 85, 5), include_lowest=True )<data_type_conversions>
questions_df_dict = {
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
}
questions_df = questions_df.astype(questions_df_dict )
df_train['Age_binned'] = pd.cut(df_train['Age'], [0, 5, 30, 60, 80], labels=[0, 1, 2, 3], retbins=False, include_lowest=True)
 df_train['Age_binned'] = df_train['Age_binned'].astype('int')
 df_test['Age_binned'] = pd.cut(df_test['Age'], [0, 5, 30, 60, 80], labels=[0, 1, 2, 3], retbins=False, include_lowest=True)
 df_test['Age_binned'] = df_test['Age_binned'].astype('int' )<drop_column>
questions_df.drop(columns=['tags'], inplace=True )
df_train.drop(columns='Age', inplace=True)
 df_test.drop(columns='Age', inplace=True )<categorify>
questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
df_train['SibSp'] = df_train['SibSp'].map(lambda x: 3 if x == 4 or x == 5 or x == 8 else x)
 df_test['SibSp'] = df_test['SibSp'].map(lambda x: 3 if x == 4 or x == 5 or x == 8 else x )<feature_engineering>
df_train['Parch'] = df_train['Parch'].map(lambda x: x if x == 0 else 1)
 df_test['Parch'] = df_test['Parch'].map(lambda x: x if x == 0 else 1 )<categorify>
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )
df_train['SibSp+Parch'] = df_train['SibSp+Parch'].map(lambda x: 1 if x == 1 or x == 2 or x == 3 else x)
 df_test['SibSp+Parch'] = df_test['SibSp+Parch'].map(lambda x: 1 if x == 1 or x == 2 or x == 3 else x)
 df_train['SibSp+Parch'] = df_train['SibSp+Parch'].map(lambda x: 2 if x == 4 or x == 5 or x == 6 or x == 7 or x == 10 else x)
 df_test['SibSp+Parch'] = df_test['SibSp+Parch'].map(lambda x: 2 if x == 4 or x == 5 or x == 6 or x == 7 or x == 10 else x )<data_type_conversions>
questions_df = pd.merge(questions_df, content_explation_agg, on='content_id', how='left',right_index=True)
df_train['Fare_binned'] = pd.cut(df_train['Fare'], bins=[0,25,75,513], labels=[0, 1, 2], retbins=False, include_lowest=True)
 df_train['Fare_binned'] = df_train['Fare_binned'].astype('int')
 df_test['Fare_binned'] = pd.cut(df_test['Fare'], bins=[0,25,75,513], labels=[0, 1, 2], retbins=False, include_lowest=True)
 df_test['Fare_binned'] = df_test['Fare_binned'].astype('int' )<drop_column>
del content_explation_agg
df_train.drop(columns='Fare', inplace=True)
 df_test.drop(columns='Fare', inplace=True )<feature_engineering>
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16')
 questions_df['content_uncorrect_count'] = questions_df['content_id'].map(content_agg['count']-content_agg['sum'] ).astype('int32')
 questions_df['content_correct_count'] = questions_df['content_id'].map(content_agg['sum'] ).astype('int32' )
df_train['Cabin'] = df_train['Cabin'].str.get(0 )<feature_engineering>
questions_df['content_elapsed_time_mean'] = questions_df['content_id'].map(content_elapsed_time_agg['mean'])
 questions_df.content_elapsed_time_mean=questions_df.content_elapsed_time_mean.astype('float16')
 questions_df['content_had_explanation_mean'] = questions_df['content_id'].map(content_had_explanation_agg['mean'])
 questions_df.content_had_explanation_mean=questions_df.content_had_explanation_mean.astype('float16' )
df_test['HaveCabin'] = df_test['Cabin'].str.get(0)
 df_train['HaveCabin'] = df_train['Cabin'].map(lambda x: 0 if x == 'Z' else 1)
 df_test['HaveCabin'] = df_test['HaveCabin'].map(lambda x: 0 if x == 'Z' else 1 )<drop_column>
del content_elapsed_time_agg
del content_had_explanation_agg
gc.collect()
df_train.drop(columns=['Cabin'], inplace=True)
 df_test.drop(columns=['Cabin'], inplace=True )<define_variables>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )
estimator = [('Logistic Regression', LogisticRegression),('Ridge Classifier', RidgeClassifier), 
('SGD Classifier', SGDClassifier),('Passive Aggressive Classifier', PassiveAggressiveClassifier), 
('SVC', SVC),('Linear SVC', LinearSVC),('Nu SVC', NuSVC), 
('K-Neighbors Classifier', KNeighborsClassifier), 
('Gaussian Naive Bayes', GaussianNB),('Multinomial Naive Bayes', MultinomialNB), 
('Bernoulli Naive Bayes', BernoulliNB),('Complement Naive Bayes', ComplementNB), 
('Decision Tree Classifier', DecisionTreeClassifier), 
('Random Forest Classifier', RandomForestClassifier),('AdaBoost Classifier', AdaBoostClassifier), 
('Gradient Boosting Classifier', GradientBoostingClassifier),('Bagging Classifier', BaggingClassifier), 
('Extra Trees Classifier', ExtraTreesClassifier),('XGBoost', XGBClassifier)]
X_train = df_train.drop(columns='Survived')
 y_train = df_train['Survived']
comparison_cols = ['Algorithm', 'Training Time(Avg)', 'Accuracy(Avg)', 'Accuracy(3xSTD)']
comparison_df = pd.DataFrame(columns=comparison_cols)
 cv_split = StratifiedShuffleSplit(n_splits=10, test_size=0.3, random_state=0)
 for idx, est in enumerate(estimator):
cv_results = cross_validate(est[1]() , X_train, y_train, cv=cv_split)
 comparison_df.loc[idx, 'Algorithm'] = est[0]
comparison_df.loc[idx, 'Training Time(Avg)'] = cv_results['fit_time'].mean()
comparison_df.loc[idx, 'Accuracy(Avg)'] = cv_results['test_score'].mean()
comparison_df.loc[idx, 'Accuracy(3xSTD)'] = cv_results['test_score'].std() * 3
comparison_df.set_index(keys='Algorithm', inplace=True)
 comparison_df.sort_values(by='Accuracy(Avg)', ascending=False, inplace=True)
 fig, ax = plt.subplots(figsize=(12,10))
y_pos = np.arange(len(comparison_df))
ax.barh(y_pos, comparison_df['Accuracy(Avg)'], xerr=comparison_df['Accuracy(3xSTD)'], color='skyblue')
 ax.set_yticks(y_pos)
 ax.set_yticklabels(comparison_df.index)
 ax.set_xlabel('Accuracy Score(Average)')
 ax.set_title('Performance Comparison After Simple Modelling', size=13)
 ax.set_xlim(0, 1)
 plt.show()<define_search_space>
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )
estimator = [('Logistic Regression', LogisticRegression),('Ridge Classifier', RidgeClassifier),('SVC', SVC), 
('Linear SVC', LinearSVC),('Nu SVC', NuSVC),('Random Forest Classifier', RandomForestClassifier), 
('AdaBoost Classifier', AdaBoostClassifier), 
('Gradient Boosting Classifier', GradientBoostingClassifier), 
('Bagging Classifier', BaggingClassifier),('XGBoost', XGBClassifier)
 ]
index = [est[0] for est in estimator]
grid_params = {'SVC': {'C': np.arange(1,21,1), 'gamma': [0.005, 0.01, 0.015, 0.02], 'random_state': [0]},
'Ridge Classifier': {'alpha': [0.001, 0.0025, 0.005], 'random_state': [0]},
'Nu SVC': {'nu': [0.5], 'gamma': [0.001, 0.01, 0.1, 1], 'random_state': [0]},
'Gradient Boosting Classifier': {'learning_rate': [0.001, 0.005, 0.01, 0.015], 'random_state': [0],
'max_depth': [1,2,3,4,5], 'n_estimators': [300, 350, 400, 450, 500]},
'Linear SVC': {'C': [1, 5, 10], 'random_state': [0]},
'Logistic Regression': {'C': np.arange(2,7.5,0.25),
'solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],
'random_state': [0]},
'AdaBoost Classifier': {'learning_rate': np.arange(0.05, 0.21, 0.01), 'n_estimators': [50, 75, 100, 125, 150],
'random_state': [0]},
'Random Forest Classifier': {'n_estimators': [200, 250, 300, 350], 'max_depth': [1,2,3,4,5,6],
'criterion': ['gini', 'entropy'], 'random_state': [0]},
'Bagging Classifier': {'n_estimators': np.arange(200, 300, 10), 'random_state': [0]},
'XGBoost': {'learning_rate': [0.001, 0.005, 0.01, 0.015], 'random_state': [0],
'max_depth': [1,2,3,4,5], 'n_estimators': [300, 350, 400, 450, 500]}
}
best_params_df = pd.DataFrame(columns=['Optimized Hyperparameters', 'Accuracy'], index=index)
 for idx, est in enumerate(estimator):
best_clf = GridSearchCV(est[1]() , param_grid=grid_params[est[0]], cv=cv_split, scoring='accuracy', n_jobs=12)
 best_clf.fit(X_train, y_train)
 best_params_df.loc[est[0], 'Optimized Hyperparameters'] = [best_clf.best_params_]
best_params_df.loc[est[0], 'Accuracy'] = best_clf.best_score_ 
<train_on_grid>
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
xg = GradientBoostingClassifier(learning_rate=0.005, max_depth=2, n_estimators=450, random_state=0)
 selector = RFECV(xg, step=1, cv=cv_split, scoring='accuracy', n_jobs=8)
 selector = selector.fit(X_train, y_train )<create_dataframe>
pd.DataFrame([X_train.columns, selector.ranking_], index=['Features', 'Ranking'] ).T.sort_values(by='Ranking' )<find_best_model_class>
del content_agg
del bundle_agg
del part_agg
gc.collect()
selected_features = ['Pclass', 'Sex', 'SibSp+Parch', 'FamilySurvival', 'Title_Mr', 'Fare_binned']
gb = GradientBoostingClassifier(learning_rate=0.005, max_depth=2, n_estimators=450, random_state=0)
 gb.fit(X_train[selected_features], y_train)
 gb_acc_score = cross_val_score(gb, X_train[selected_features], y_train, cv=cv_split, scoring='accuracy')
 print('The performance of the model using the selected features: {:.2f}%'.format(gb_acc_score.mean() *100))<save_to_csv>
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
gb = GradientBoostingClassifier(learning_rate=0.005, max_depth=2, n_estimators=450, random_state=0)
 gb.fit(X_train[selected_features], y_train)
 test_Survived = pd.DataFrame(gb.predict(df_test[selected_features]), columns=['Survived'], index=np.arange(892,1310,1))
test_Survived = test_Survived.reset_index()
test_Survived.rename(columns={'index': 'PassengerID'}, inplace=True)
 test_Survived.to_csv("gb.csv",index=False )<set_options>
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 200,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df))
warnings.filterwarnings('ignore')
 %matplotlib inline<load_from_csv>
del train_df_clf
del valid_df
gc.collect()
train_data = pd.read_csv(".. /input/train.csv")
 train_data.columns<load_from_csv>
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect()
test = pd.read_csv(".. /input/test.csv")
 IDtest = test["PassengerId"] 
<drop_column>
MAX_SEQ = 100
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 nn_model = SAKTModel(n_skill, embed_dim=128)
 try:
nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt"))
except:
nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt", map_location='cpu'))
nn_model.to(device)
 nn_model.eval()
train_data.drop(['PassengerId','Ticket'], axis=1, inplace = True)
<count_missing_values>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
train_data.isnull().sum()<data_type_conversions>
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
train_data['Age'].fillna(train_data['Age'].mean() , inplace = True)
 test['Age'].fillna(test['Age'].mean() , inplace = True)
 train_data["Embarked"] = train_data["Embarked"].fillna("C")
 test["Embarked"] = test["Embarked"].fillna("C")
 train_data['Fare'].fillna(train_data['Fare'].median() , inplace = True)
 test['Fare'].fillna(test['Fare'].median() , inplace = True)
 train_data.Cabin.fillna("N", inplace=True)
 test.Cabin.fillna("N", inplace=True )<count_missing_values>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()
train_data.isnull().sum()<feature_engineering>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()
train_title = [i.split(",")[1].split(".")[0].strip() for i in train_data["Name"]]
train_data["Title"] = pd.Series(train_title)
 train_data["Title"].head()<feature_engineering>
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
test_title = [i.split(",")[1].split(".")[0].strip() for i in test["Name"]]
test["Title"] = pd.Series(test_title)
 test["Title"].head()<categorify>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
train_data["Title"] = train_data["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 train_data["Title"] = train_data["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 train_data["Title"] = train_data["Title"].astype(int )<categorify>
test["Title"] = test["Title"].replace(['Lady', 'the Countess','Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 test["Title"] = test["Title"].map({"Master":0, "Miss":1, "Ms" : 1 , "Mme":1, "Mlle":1, "Mrs":1, "Mr":2, "Rare":3})
 test["Title"] = test["Title"].astype(int )<feature_engineering>
iter_test = env.iter_test()
prior_test_df = None
prev_test_df1 = None
train_data["Family_size"] = train_data["SibSp"] + train_data["Parch"] + 1
test["Family_size"] = test["SibSp"] + test["Parch"] + 1 
<feature_engineering>
N=[0.4,0.6]
<categorify><EOS> 
%%time
for(test_df, sample_prediction_df)in iter_test:
test_df1=test_df.copy()
if(prev_test_df1 is not None):
prev_test_df1['answered_correctly'] = eval(test_df1['prior_group_answers_correct'].iloc[0])
 prev_test_df1 = prev_test_df1[prev_test_df1.content_type_id == False]
prev_group = prev_test_df1[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df1 = test_df1.copy()
test_df1 = test_df1[test_df1.content_type_id == False]
test_dataset = TestDataset(group, test_df1, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = nn_model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prior_test_df = test_df.copy()
question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
o2=sub_preds / len(clfs)
 test_df[target]=0.5*np.array(outs)+0.5*np.array(o2)
 env.predict(test_df[['row_id', target]] )