Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<train_on_grid>
env = riiideducation.make_env()
n_estimators = [10, 15, 20, 25, 50, 75, 100, 150]
max_samples = [1, 2, 3, 5, 7, 10, 15, 20, 25, 30, 50]
max_features = [1, 3, 5, 7]
hyperparams = {'n_estimators': n_estimators, 'max_samples': max_samples, 'max_features': max_features}
gd=GridSearchCV(estimator = BaggingClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<choose_model_class>
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)
ran = RandomForestClassifier(n_estimators=50,
max_depth=3,
max_features=7,
min_samples_leaf=8,
min_samples_split=6,
random_state=1)
 knn = KNeighborsClassifier(algorithm='auto',
leaf_size=3,
n_neighbors=10,
weights='uniform')
 log = LogisticRegression(C=21.544346900318832,
penalty='l2')
 xgb = XGBClassifier(learning_rate=0.0001,
n_estimators=10,
random_state=1)
 gbc = GradientBoostingClassifier(learning_rate=0.0005,
n_estimators=1250,
random_state=1)
 svc = SVC(C=50, gamma=0.01, probability=True)
 ext = ExtraTreesClassifier(max_depth=3,
max_features=7,
min_samples_leaf=8,
min_samples_split=4,
n_estimators=25,
random_state=1)
 ada = AdaBoostClassifier(learning_rate=0.5,
n_estimators=25,
random_state=1)
 gpc = GaussianProcessClassifier(max_iter_predict=1)
 bag = BaggingClassifier(max_features=7, max_samples=50, n_estimators=20,random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v3 = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores_v3.append(acc.mean() )<prepare_output>
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'}
)
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Original Score': scores,
'Score with feature selection': scores_v2,
'Score with tuned parameters': scores_v3})
 result_df = results.sort_values(by='Score with tuned parameters', ascending=False ).reset_index(drop=True)
 result_df.head(11 )<choose_model_class>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
grid_hard = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbour', knn), 
('Bagging Classifier', bag)], voting = 'hard')
 grid_hard_cv = model_selection.cross_validate(grid_hard, X_train, y_train, cv = 10, return_train_score=True)
 grid_hard.fit(X_train, y_train)
 print("Hard voting on train set score mean: {:.2f}".format(grid_hard_cv['train_score'].mean() *100))
print("Hard voting on test set score mean: {:.2f}".format(grid_hard_cv['test_score'].mean() *100)) 
<choose_model_class>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
grid_soft = VotingClassifier(estimators = [('Random Forest', ran), 
('Logistic Regression', log), 
('XGBoost', xgb), 
('Gradient Boosting', gbc), 
('Extra Trees', ext), 
('AdaBoost', ada), 
('Gaussian Process', gpc), 
('SVC', svc), 
('K Nearest Neighbour', knn), 
('Bagging Classifier', bag)], voting = 'soft')
 grid_soft_cv = model_selection.cross_validate(grid_soft, X_train, y_train, cv = 10, return_train_score=True)
 grid_soft.fit(X_train, y_train)
 print("Soft voting on train set score mean: {:.2f}".format(grid_soft_cv['train_score'].mean() *100))
print("Soft voting on test set score mean: {:.2f}".format(grid_soft_cv['test_score'].mean() *100)) 
<save_to_csv>
train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
predictions = grid_hard.predict(X_test)
 submission = pd.concat([pd.DataFrame(passId), pd.DataFrame(predictions)], axis = 'columns')
 submission.columns = ["PassengerId", "Survived"]
submission.to_csv('titanic_submission.csv', header = True, index = False )<set_options>
user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()
%load_ext autoreload
%autoreload 2
%matplotlib inline<set_options>
for column in user_lecture_stats_part.columns:
bool_column = column + '_boolean'
user_lecture_stats_part[bool_column] =(user_lecture_stats_part[column] > 0 ).astype(int )
sns.set(style="darkgrid")
 warnings.filterwarnings('ignore' )<define_variables>
del(train_lectures )
PATH = ".. /input/cleaned-data/"<load_from_csv>
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )
df_train=pd.read_csv(f'{PATH}train_clean.csv')
 df_test=pd.read_csv(f'{PATH}test_clean.csv' )<train_model>
train[(train.content_type_id == False)].task_container_id.nunique()
print('Number of Training Examples = {}'.format(df_train.shape[0]))
print('Number of Test Examples = {}
'.format(df_test.shape[0]))
print('Training X Shape = {}'.format(df_train.shape))
print('Training y Shape = {}
'.format(df_train['SalePrice'].shape[0]))
print('Test X Shape = {}'.format(df_test.shape))
print('Test y Shape = {}
'.format(df_test.shape[0]))<define_variables>
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2
ntrain = df_train.shape[0]
ntest = df_test.shape[0]<concatenate>
group3['avg_questions_seen'] = group3.avg_questions.cumsum()
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Test Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
df_all.shape<prepare_x_and_y>
group3.iloc[0].avg_questions_seen
y_train_full = df_train.SalePrice.values
df_all.drop(['SalePrice'], axis=1, inplace=True)
<sort_values>
results_u_final = train.loc[train.content_type_id == False, ['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_final.columns = ['answered_correctly_user']
results_u2_final = train.loc[train.content_type_id == False, ['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_final.columns = ['explanation_mean_user']
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
skewness = df_all.select_dtypes(include=numerics ).apply(lambda x: skew(x))
skew_index = skewness[abs(skewness)>= 0.85].index
skewness[skew_index].sort_values(ascending=False )<feature_engineering>
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )
lam = 0.15
for column in skew_index:
df_all[column] = boxcox1p(df_all[column], lam)
<categorify>
results_q_final = train.loc[train.content_type_id == False, ['question_id','answered_correctly']].groupby(['question_id'] ).agg(['mean'])
 results_q_final.columns = ['quest_pct']
df_all=pd.get_dummies(df_all )<split>
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']
X_test=df_all.iloc[ntrain:]
X_train_full=df_all.iloc[:ntrain]
X_train, X_valid, y_train, y_valid = train_test_split(pd.get_dummies(X_train_full), y_train_full )<drop_column>
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )
all_features = df_all.keys()
df_all = df_all.drop(df_all.loc[:,(df_all==0 ).sum() >=(df_all.shape[0]*0.984)],axis=1)
 df_all = df_all.drop(df_all.loc[:,(df_all==1 ).sum() >=(df_all.shape[0]*0.984)],axis=1)
 remain_features = df_all.keys()
remov_features = [st for st in all_features if st not in remain_features]
print(len(remov_features), 'features were removed:', remov_features )<normalization>
question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )
scaler = RobustScaler()
df_all = pd.DataFrame(scaler.fit_transform(df_all))<split>
question2.quest_pct = round(question2.quest_pct,5 )
X_test=df_all.iloc[ntrain:]
X_train_full=df_all.iloc[:ntrain]
X_train, X_valid, y_train, y_valid = train_test_split(pd.get_dummies(X_train_full), y_train_full )<compute_test_metric>
train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True )
def rmse(x,y): return math.sqrt(((x-y)**2 ).mean())
 def print_score(m,X_train=X_train, X_valid=X_valid, y_train=y_train, y_valid=y_valid):
res = [rmse(m.predict(X_train), y_train), rmse(m.predict(X_valid), y_valid),
m.score(X_train, y_train), m.score(X_valid, y_valid)]
if hasattr(m, 'oob_score_'): res.append(m.oob_score_)
 print(res )<train_on_grid>
validation = train.groupby('user_id' ).tail(5)
 train = train[~train.index.isin(validation.index)]
len(train)+ len(validation )
print('ElasticNet Lasso')
 def lasso_score(X,y):
lasso = ElasticNet(random_state=1)
 param = {'l1_ratio' : [0],
'alpha' : [0.017]}
lasso = GridSearchCV(lasso, param, cv=5, scoring='neg_mean_squared_error')
 lasso.fit(X,y)
 print('Lasso:', np.sqrt(lasso.best_score_*-1))
return lasso
lasso_score(X_train, y_train)
 print('XGBoost')
 m_xgb = XGBRegressor(n_estimators=160, learning_rate=0.05)
 m_xgb.fit(X_train, y_train)
 print_score(m_xgb,X_train, X_valid, y_train, y_valid)
 print('Random Forest')
 m_rf = RandomForestRegressor(n_estimators=160, min_samples_leaf=1, max_features=0.5, n_jobs=-1, oob_score=True)
 m_rf.fit(X_train, y_train)
 print_score(m_rf,X_train, X_valid, y_train, y_valid)
<compute_train_metric>
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X_train_full=X_train_full):
rmse = np.sqrt(-cross_val_score(model, X_train_full, y_train_full,
scoring="neg_mean_squared_error",
cv=kfolds))
return(rmse)
 alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
ridge = make_pipeline(RobustScaler() ,
RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() ,
LassoCV(max_iter=1e7, alphas=alphas2,
random_state=42, cv=kfolds))
elasticnet = make_pipeline(RobustScaler() ,
ElasticNetCV(max_iter=1e7, alphas=e_alphas,
cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() ,
SVR(C= 20, epsilon= 0.008, gamma=0.0003,))
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =42)
 lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1, 
)
 xgboost = XGBRegressor(learning_rate=0.01, n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet,
gbr, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True)
 print('TEST score on CV')
 score = cv_rmse(ridge)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(elasticnet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(svr)
 print("SVR score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(lightgbm)
 print("Lightgbm score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(gbr)
 print("GradientBoosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = cv_rmse(xgboost)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
X = train.groupby('user_id' ).tail(18)
 train = train[~train.index.isin(X.index)]
len(X)+ len(train)+ len(validation )
print('StackingCVRegressor')
 stack_gen_model = stack_gen.fit(np.array(X_train_full), np.array(y_train_full))
elastic_model_full_data = elasticnet.fit(X_train_full, y_train_full)
 lasso_model_full_data = lasso.fit(X_train_full, y_train_full)
 ridge_model_full_data = ridge.fit(X_train_full, y_train_full)
 svr_model_full_data = svr.fit(X_train_full, y_train_full)
 gbr_model_full_data = gbr.fit(X_train_full, y_train_full)
 xgb_model_full_data = xgboost.fit(X_train_full, y_train_full)
 lgb_model_full_data = lightgbm.fit(X_train_full, y_train_full )<predict_on_test>
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']
def blend_models_predict(X_train_full):
return(( 0.1 * elastic_model_full_data.predict(X_train_full)) + \ 
(0.05 * lasso_model_full_data.predict(X_train_full)) + \ 
(0.1 * ridge_model_full_data.predict(X_train_full)) + \ 
(0.1 * svr_model_full_data.predict(X_train_full)) + \ 
(0.1 * gbr_model_full_data.predict(X_train_full)) + \ 
(0.15 * xgb_model_full_data.predict(X_train_full)) + \ 
(0.1 * lgb_model_full_data.predict(X_train_full)) + \ 
(0.3 * stack_gen_model.predict(np.array(X_train_full))))<compute_test_metric>
del(train )
print('RMSLE score on train data:')
 print(rmsle(y_train_full, blend_models_predict(X_train_full)) )<load_from_csv>
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left")
 X = pd.merge(X, user_lecture_stats_part, on=['user_id'], how="left" )
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_test)) )<save_to_csv>
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left")
 validation = pd.merge(validation, user_lecture_stats_part, on=['user_id'], how="left" )
submission.to_csv("submission.csv", index=False)
 print('Save submission', datetime.now() , )<set_options>
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )
plt.style.use('bmh')
 sns.set_style({'axes.grid':False})
 %matplotlib inline

 init_notebook_mode(connected = True)

wrn.filterwarnings('ignore', category = DeprecationWarning)
 wrn.filterwarnings('ignore', category = FutureWarning)
 wrn.filterwarnings('ignore', category = UserWarning )<load_from_csv>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )<concatenate>
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )
merged = pd.concat([train, test], axis=0, sort=True)
 display(merged.dtypes.value_counts())
 print('Dimensions of data:', merged.shape )<data_type_conversions>
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1
merged.loc[:,['MSSubClass', 'OverallQual', 'OverallCond', 'MoSold', 'YrSold']] = merged.loc[:,['MSSubClass', 'OverallQual', 'OverallCond', 'MoSold', 'YrSold']].astype('object' )<count_values>
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )
merged.dtypes.value_counts()<drop_column>
X = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]
X_val = X_val[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]
train.drop(train[train.GrLivArea>4000].index, inplace = True)
 train.reset_index(drop = True, inplace = True )<drop_column>
X['answered_correctly_user'].fillna(0.65, inplace=True)
 X['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X['quest_pct'].fillna(content_mean, inplace=True)
 X['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X['part_1'].fillna(0, inplace = True)
 X['part_2'].fillna(0, inplace = True)
 X['part_3'].fillna(0, inplace = True)
 X['part_4'].fillna(0, inplace = True)
 X['part_5'].fillna(0, inplace = True)
 X['part_6'].fillna(0, inplace = True)
 X['part_7'].fillna(0, inplace = True)
 X['type_of_concept'].fillna(0, inplace = True)
 X['type_of_intention'].fillna(0, inplace = True)
 X['type_of_solving_question'].fillna(0, inplace = True)
 X['type_of_starter'].fillna(0, inplace = True)
 X['part_1_boolean'].fillna(0, inplace = True)
 X['part_2_boolean'].fillna(0, inplace = True)
 X['part_3_boolean'].fillna(0, inplace = True)
 X['part_4_boolean'].fillna(0, inplace = True)
 X['part_5_boolean'].fillna(0, inplace = True)
 X['part_6_boolean'].fillna(0, inplace = True)
 X['part_7_boolean'].fillna(0, inplace = True)
 X['type_of_concept_boolean'].fillna(0, inplace = True)
 X['type_of_intention_boolean'].fillna(0, inplace = True)
 X['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X['type_of_starter_boolean'].fillna(0, inplace = True )
train.drop(train[train.TotalBsmtSF>3000].index, inplace = True)
 train.reset_index(drop = True, inplace = True )<drop_column>
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X_val['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X_val['part_1'].fillna(0, inplace = True)
 X_val['part_2'].fillna(0, inplace = True)
 X_val['part_3'].fillna(0, inplace = True)
 X_val['part_4'].fillna(0, inplace = True)
 X_val['part_5'].fillna(0, inplace = True)
 X_val['part_6'].fillna(0, inplace = True)
 X_val['part_7'].fillna(0, inplace = True)
 X_val['type_of_concept'].fillna(0, inplace = True)
 X_val['type_of_intention'].fillna(0, inplace = True)
 X_val['type_of_solving_question'].fillna(0, inplace = True)
 X_val['type_of_starter'].fillna(0, inplace = True)
 X_val['part_1_boolean'].fillna(0, inplace = True)
 X_val['part_2_boolean'].fillna(0, inplace = True)
 X_val['part_3_boolean'].fillna(0, inplace = True)
 X_val['part_4_boolean'].fillna(0, inplace = True)
 X_val['part_5_boolean'].fillna(0, inplace = True)
 X_val['part_6_boolean'].fillna(0, inplace = True)
 X_val['part_7_boolean'].fillna(0, inplace = True)
 X_val['type_of_concept_boolean'].fillna(0, inplace = True)
 X_val['type_of_intention_boolean'].fillna(0, inplace = True)
 X_val['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X_val['type_of_starter_boolean'].fillna(0, inplace = True )
train.drop(train[train.YearBuilt<1900].index, inplace = True)
 train.reset_index(drop = True, inplace = True )<prepare_x_and_y>
params = {
'objective': 'binary',
'boosting' : 'gbdt',
'max_bin': 800,
'learning_rate': 0.0175,
'num_leaves': 80
}
lgb_train = lgb.Dataset(X, y, categorical_feature = ['part', 'prior_question_had_explanation_enc'])
 lgb_eval = lgb.Dataset(X_val, y_val, categorical_feature = ['part', 'prior_question_had_explanation_enc'], reference=lgb_train )
y_train = train.SalePrice

 train.drop('SalePrice', axis = 1, inplace = True)

df_merged = pd.concat([train, test], axis = 0)

df_merged.shape<data_type_conversions>
model = lgb.train(
 params, lgb_train,
valid_sets=[lgb_train, lgb_eval],
verbose_eval=50,
num_boost_round=10000,
early_stopping_rounds=12
)
df_merged.loc[:,['MSSubClass', 'OverallQual', 'OverallCond', 'MoSold', 'YrSold']] = df_merged.loc[:,['MSSubClass', 'OverallQual', 'OverallCond', 'MoSold', 'YrSold']].astype('object')
 df_merged.dtypes.value_counts()<count_missing_values>
y_pred = model.predict(X_val)
 y_true = np.array(y_val)
 roc_auc_score(y_true, y_pred )
missing_columns = df_merged.columns[df_merged.isnull().any() ].values

 total_missing_columns = np.count_nonzero(df_merged.isnull().sum())
 print('We have ' , total_missing_columns , 'features with missing values and those features(with missing values)are:
' , missing_columns )<categorify>
import matplotlib.pyplot as plt
import seaborn as sns
to_impute_by_mode = df_merged.loc[:, ['Electrical', 'MSZoning','Utilities','Exterior1st','Exterior2nd','KitchenQual','Functional', 'SaleType']]
for i in to_impute_by_mode.columns:
df_merged[i].fillna(df_merged[i].mode() [0], inplace = True )<categorify>
iter_test = env.iter_test()
<feature_engineering><EOS> 
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how="left")
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['part_1_boolean'].fillna(0, inplace = True)
 test_df['part_2_boolean'].fillna(0, inplace = True)
 test_df['part_3_boolean'].fillna(0, inplace = True)
 test_df['part_4_boolean'].fillna(0, inplace = True)
 test_df['part_5_boolean'].fillna(0, inplace = True)
 test_df['part_6_boolean'].fillna(0, inplace = True)
 test_df['part_7_boolean'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df['answered_correctly'] = model.predict(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )