Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<categorify>
import numpy as np
import pandas as pd
import gc
import riiideducation
from sklearn.metrics import roc_auc_score
from sklearn.feature_selection import RFE
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import KFold
from lightgbm import LGBMClassifier
from xgboost import XGBClassifier
from sklearn.linear_model import LogisticRegression
for df in [train, test]:
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'Neighborhood', 'BldgType', 'HouseStyle', 'MasVnrType', 'FireplaceQu', 'Fence', 'PoolQC', 'MiscFeature', 'Alley'):
df[col] = df[col].fillna('None' )<data_type_conversions>
env = riiideducation.make_env()
train['MasVnrArea'] = train['MasVnrArea'].fillna(0)
 test['MasVnrArea'] = test['MasVnrArea'].fillna(0 )<sort_values>
train = pd.read_csv(
 '/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'
}
)
missing = train.isnull().sum().sort_values(ascending=False)
 missing = missing[missing>0]
missing<categorify>
questions_df = pd.read_csv(
 '/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={
'question_id': 'int16',
'part': 'int8'}
)
train['Electrical'] = train['Electrical'].fillna(train['Electrical'].mode() [0])
 test['Electrical'] = test['Electrical'].fillna(test['Electrical'].mode() [0] )<data_type_conversions>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
train['LotFrontage'] = train.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))
test['LotFrontage'] = test.groupby('Neighborhood')['LotFrontage'].apply(lambda x: x.fillna(x.median()))<sort_values>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
missing = train.isnull().sum().sort_values(ascending=False)
 missing = missing[missing>0]
missing<sort_values>
train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
missing = test.select_dtypes(include='object' ).isnull().sum().sort_values(ascending=False)
 missing = missing[missing>0]
missing<sort_values>
user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()
missing = test.select_dtypes(exclude='object' ).isnull().sum().sort_values(ascending=False)
 missing[missing>0]<categorify>
for column in user_lecture_stats_part.columns:
bool_column = column + '_boolean'
user_lecture_stats_part[bool_column] =(user_lecture_stats_part[column] > 0 ).astype(int )
train['MSZoning'] = train.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))
test['MSZoning'] = test.groupby('MSSubClass')['MSZoning'].transform(lambda x: x.fillna(x.mode() [0]))<drop_column>
del train_lectures
gc.collect()
test = test.drop(['Utilities'], axis=1)
 train = train.drop(['Utilities'], axis=1 )<data_type_conversions>
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )
for df in [train, test]:
for col in('Functional', 'SaleType', 'Exterior2nd', 'Exterior1st', 'KitchenQual'):
df[col] = df[col].fillna(df[col].mode() [0] )<sort_values>
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2
missing = test.select_dtypes(include='object' ).isnull().sum().sort_values(ascending=False)
 missing = missing[missing>0]
missing<data_type_conversions>
group3['avg_questions_seen'] = group3.avg_questions.cumsum()
for df in [train, test]:
for col in('MasVnrArea', 'BsmtFullBath', 'BsmtHalfBath', 'GarageCars', 'TotalBsmtSF',
'BsmtUnfSF', 'BsmtFinSF2', 'BsmtFinSF1'):
df[col] = df[col].fillna(0 )<sort_values>
results_u_final = train.loc[train.content_type_id == False, ['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_final.columns = ['answered_correctly_user']
results_u2_final = train.loc[train.content_type_id == False, ['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_final.columns = ['explanation_mean_user']
missing = test.select_dtypes(exclude='object' ).isnull().sum().sort_values(ascending=False)
 missing[missing>0]<define_variables>
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )
sf = [col for col in train.columns if 'SF' in col]
sf<feature_engineering>
results_q_final = train.loc[train.content_type_id == False, ['question_id','answered_correctly']].groupby(['question_id'] ).agg(['mean'])
 results_q_final.columns = ['quest_pct']
train['TotalSF'] = train['TotalBsmtSF'] + train['2ndFlrSF']
test['TotalSF'] = test['TotalBsmtSF'] + test['2ndFlrSF']<define_variables>
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']
bath = [col for col in train.columns if 'Bath' in col]
bath<feature_engineering>
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )
train['TotalBath'] = train['BsmtFullBath'] + train['BsmtHalfBath'] + train['FullBath'] + train['HalfBath']
test['TotalBath'] = test['BsmtFullBath'] + test['BsmtHalfBath'] + test['FullBath'] + test['HalfBath']<define_variables>
question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )
porch = [col for col in train.columns if 'Porch' in col]
porch<feature_engineering>
question2.quest_pct = round(question2.quest_pct, 5 )
train['PorchSF'] = train['OpenPorchSF'] + train['EnclosedPorch'] + train['3SsnPorch'] + train['ScreenPorch'] + train['WoodDeckSF']
test['PorchSF'] = test['OpenPorchSF'] + test['EnclosedPorch'] + test['3SsnPorch'] + test['ScreenPorch'] + test['WoodDeckSF']<define_variables>
train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True )
pool = [col for col in train.columns if 'Pool' in col]
pool<feature_engineering>
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']
train['HasBsmt'] = train['TotalBsmtSF'].apply(lambda x: 1 if x>0 else 0)
 train['HasPool'] = train['PoolArea'].apply(lambda x: 1 if x>0 else 0)
 train['Has2ndFlr'] = train['2ndFlrSF'].apply(lambda x: 1 if x>0 else 0)
 train['HasFirePlace'] = train['Fireplaces'].apply(lambda x: 1 if x>0 else 0)
 train['HasGarage'] = train['GarageCars'].apply(lambda x: 1 if x>0 else 0)
 test['HasBsmt'] = test['TotalBsmtSF'].apply(lambda x: 1 if x>0 else 0)
 test['HasPool'] = test['PoolArea'].apply(lambda x: 1 if x>0 else 0)
 test['Has2ndFlr'] = test['2ndFlrSF'].apply(lambda x: 1 if x>0 else 0)
 test['HasFirePlace'] = test['Fireplaces'].apply(lambda x: 1 if x>0 else 0)
 test['HasGarage'] = test['GarageCars'].apply(lambda x: 1 if x>0 else 0 )<data_type_conversions>
X = train.groupby('user_id' ).tail(18)
 train = train[~train.index.isin(X.index)]
len(X)+ len(train)+ len(validation )
train['MasVnrArea'] = train['MasVnrArea'].astype(int)
 test['MasVnrArea'] = test['MasVnrArea'].astype(int )<categorify>
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']
label_enc_variables = ['FireplaceQu', 'LotShape', 'OverallCond', 'ExterQual',
'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1',
'BsmtFinType2', 'HeatingQC', 'KitchenQual', 'GarageFinish', 'GarageQual',
'GarageCond', 'PavedDrive', 'PoolQC', 'Fence']
label_encoder = LabelEncoder()
for col in label_enc_variables:
train[col] = label_encoder.fit_transform(train[col])
 test[col] = label_encoder.transform(test[col] )<prepare_x_and_y>
del(train)
 gc.collect()
X = train.drop(['SalePrice'], axis=1)
 y = train['SalePrice']<data_type_conversions>
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left")
 X = pd.merge(X, user_lecture_stats_part, on=['user_id'], how="left" )
num_cols = [col for col in X.columns if X[col].dtype!='object']
X.update(X[num_cols].fillna(0))
test.update(test[num_cols].fillna(0))
cat_cols = [col for col in X.columns if X[col].dtype=='object']
X.update(X[cat_cols].fillna('None'))
test.update(test[cat_cols].fillna('None'))<import_modules>
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left")
 validation = pd.merge(validation, user_lecture_stats_part, on=['user_id'], how="left" )
from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer
from xgboost import XGBRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from mlxtend.regressor import StackingCVRegressor
from sklearn.pipeline import make_pipeline
from lightgbm import LGBMRegressor
import lightgbm as lgb
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import Ridge, RidgeCV, ElasticNetCV
from sklearn.preprocessing import RobustScaler
from sklearn.svm import SVR<split>
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )
X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=0.8, test_size=0.2, random_state=2 )<define_variables>
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )
numerical_cols = [col for col in X_train.columns
if X_train[col].dtype in ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']]
categorical_cols= [col for col in X_train.columns
if X_train[col].nunique() <=30 and X_train[col].dtype=='object']
final_cols = numerical_cols + categorical_cols
X_train = X_train[final_cols].copy()
X_valid = X_valid[final_cols].copy()
test = test[final_cols].copy()<categorify>
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1
X_train = pd.get_dummies(X_train)
 X_valid = pd.get_dummies(X_valid)
 test = pd.get_dummies(test)
 X_train, X_valid = X_train.align(X_valid, join='left', axis=1)
 X_train, test = X_train.align(test, join='left', axis=1 )<compute_train_metric>
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )
estimators = [2000, 2500, 3000, 3500]
for n in estimators:
my_dict = dict()
model = XGBRegressor(n_estimators=n, learning_rate=0.01, colsample_bytree=0.45, max_depth=3)
 model.fit(X_train, y_train)
 preds = model.predict(X_valid)
 mae = mean_absolute_error(rev_y(preds), rev_y(y_valid))
my_dict[f"n_est {n}"] = mae
print(my_dict )<train_model>
X = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]
X_val = X_val[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]
my_dict = dict()
n=3500
model = XGBRegressor(n_estimators=n, learning_rate=0.01, colsample_bytree=0.45, max_depth=3,
gamma=0, subsample=0.4, reg_alpha=0, reg_lambda=1, objective='reg:squarederror')
 model.fit(X_train, y_train)
 preds = model.predict(X_valid)
 mae = mean_absolute_error(rev_y(preds), rev_y(y_valid))
my_dict[f"n_est {n}"] = mae
print(my_dict )<choose_model_class>
X['answered_correctly_user'].fillna(0.65, inplace=True)
 X['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X['quest_pct'].fillna(content_mean, inplace=True)
 X['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X['part_1'].fillna(0, inplace = True)
 X['part_2'].fillna(0, inplace = True)
 X['part_3'].fillna(0, inplace = True)
 X['part_4'].fillna(0, inplace = True)
 X['part_5'].fillna(0, inplace = True)
 X['part_6'].fillna(0, inplace = True)
 X['part_7'].fillna(0, inplace = True)
 X['type_of_concept'].fillna(0, inplace = True)
 X['type_of_intention'].fillna(0, inplace = True)
 X['type_of_solving_question'].fillna(0, inplace = True)
 X['type_of_starter'].fillna(0, inplace = True)
 X['part_1_boolean'].fillna(0, inplace = True)
 X['part_2_boolean'].fillna(0, inplace = True)
 X['part_3_boolean'].fillna(0, inplace = True)
 X['part_4_boolean'].fillna(0, inplace = True)
 X['part_5_boolean'].fillna(0, inplace = True)
 X['part_6_boolean'].fillna(0, inplace = True)
 X['part_7_boolean'].fillna(0, inplace = True)
 X['type_of_concept_boolean'].fillna(0, inplace = True)
 X['type_of_intention_boolean'].fillna(0, inplace = True)
 X['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X['type_of_starter_boolean'].fillna(0, inplace = True )
n = 3500
my_dict = dict()
xgb = XGBRegressor(n_estimators=n,
learning_rate=0.01,
colsample_bytree=0.45,
max_depth=3,
gamma=0,
subsample=0.4,
reg_alpha=0,
reg_lambda=1,
objective='reg:squarederror')
 lgb = LGBMRegressor(objective='regression',
num_leaves=6,
learning_rate=0.01,
n_estimators=n,
max_bin=200,
bagging_fraction=0.8,
bagging_freq=4,
bagging_seed=8,
feature_fraction=0.2,
feature_fraction_seed=8,
min_sum_hessian_in_leaf = 11,
verbose=-1,
random_state=42)
 ridge_alphas = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=ridge_alphas))
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003))
gbr = GradientBoostingRegressor(n_estimators=n,
learning_rate=0.01,
max_depth=4,
max_features='sqrt',
min_samples_leaf=15,
min_samples_split=10,
loss='huber',
random_state=42)
 stack = StackingCVRegressor(regressors=(ridge, gbr, xgb, elasticnet, svr, lgb),
meta_regressor=xgb,
use_features_in_secondary=True )<define_variables>
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X_val['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X_val['part_1'].fillna(0, inplace = True)
 X_val['part_2'].fillna(0, inplace = True)
 X_val['part_3'].fillna(0, inplace = True)
 X_val['part_4'].fillna(0, inplace = True)
 X_val['part_5'].fillna(0, inplace = True)
 X_val['part_6'].fillna(0, inplace = True)
 X_val['part_7'].fillna(0, inplace = True)
 X_val['type_of_concept'].fillna(0, inplace = True)
 X_val['type_of_intention'].fillna(0, inplace = True)
 X_val['type_of_solving_question'].fillna(0, inplace = True)
 X_val['type_of_starter'].fillna(0, inplace = True)
 X_val['part_1_boolean'].fillna(0, inplace = True)
 X_val['part_2_boolean'].fillna(0, inplace = True)
 X_val['part_3_boolean'].fillna(0, inplace = True)
 X_val['part_4_boolean'].fillna(0, inplace = True)
 X_val['part_5_boolean'].fillna(0, inplace = True)
 X_val['part_6_boolean'].fillna(0, inplace = True)
 X_val['part_7_boolean'].fillna(0, inplace = True)
 X_val['type_of_concept_boolean'].fillna(0, inplace = True)
 X_val['type_of_intention_boolean'].fillna(0, inplace = True)
 X_val['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X_val['type_of_starter_boolean'].fillna(0, inplace = True )
Scores = {}<compute_test_metric>
params = {
'num_leaves': 31,
'n_estimators': 200,
'max_depth': 8,
'min_child_samples': 356,
'learning_rate': 0.2982483634778906,
'min_data_in_leaf': 82,
'bagging_fraction': 0.6545628633239445,
'feature_fraction': 0.9164482379289846,
'random_state': 666
}
full_model = LGBMClassifier(**params)
 full_model.fit(X, y)
 preds = full_model.predict_proba(X_val)[:,1]
print('LGB roc auc', roc_auc_score(y_val, preds))
full_xgb = XGBClassifier(random_state=666)
 full_xgb.fit(X, y)
 preds = full_xgb.predict_proba(X_val)[:,1]
print('XGB roc auc', roc_auc_score(y_val, preds))
full_lr = LogisticRegression(random_state=666)
 full_lr.fit(X, y)
 preds = full_lr.predict_proba(X_val)[:,1]
print('LR roc auc', roc_auc_score(y_val, preds))
def rmse(model, X, y):
scores = np.sqrt(-1 * cross_val_score(model, X, y,
cv=10,
scoring='neg_mean_squared_error'))
return scores
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))<compute_test_metric>
import optuna
from optuna.samplers import TPESampler
scores = rmse(xgb, X_train, y_train)
 print("XGBoost Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['XGB'] = scores.mean()<compute_test_metric>
rfe = RFE(estimator=DecisionTreeClassifier(random_state=666), n_features_to_select=14)
 rfe.fit(X, y)
 X = rfe.transform(X)
 X_val = rfe.transform(X_val )
scores = rmse(lgb, X_train, y_train)
 print("Light Gradient Boosting Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['LGB'] = scores.mean()<compute_test_metric>
sampler = TPESampler(seed=666)
 def create_model(trial):
num_leaves = trial.suggest_int("num_leaves", 2, 31)
 n_estimators = trial.suggest_int("n_estimators", 20, 300)
 max_depth = trial.suggest_int('max_depth', 3, 9)
 min_child_samples = trial.suggest_int('min_child_samples', 100, 1200)
 learning_rate = trial.suggest_uniform('learning_rate', 0.0001, 0.99)
 min_data_in_leaf = trial.suggest_int('min_data_in_leaf', 5, 90)
 bagging_fraction = trial.suggest_uniform('bagging_fraction', 0.0001, 1.0)
 feature_fraction = trial.suggest_uniform('feature_fraction', 0.0001, 1.0)
 model = LGBMClassifier(
 num_leaves=num_leaves,
n_estimators=n_estimators,
max_depth=max_depth,
min_child_samples=min_child_samples,
min_data_in_leaf=min_data_in_leaf,
learning_rate=learning_rate,
feature_fraction=feature_fraction,
random_state=666 
)
 return model
def objective(trial):
model = create_model(trial)
 model.fit(X, y)
 preds = model.predict_proba(X_val)[:,1]
score = roc_auc_score(y_val, preds)
 return score
params = {
'num_leaves': 28,
'n_estimators': 295,
'max_depth': 8,
'min_child_samples': 1178,
'learning_rate': 0.2379173491475032,
'min_data_in_leaf': 35,
'bagging_fraction': 0.8389723511600549,
'feature_fraction': 0.9606189400533491,
'random_state': 666
}
model = LGBMClassifier(**params)
 model.fit(X, y)
 preds = model.predict_proba(X_val)[:,1]
roc_auc_score(y_val, preds )
scores = rmse(ridge, X_train, y_train)
 print("Ridge Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['Ridge'] = scores.mean()<compute_test_metric>
X = pd.DataFrame(X)
 X_val = pd.DataFrame(X_val)
 y = pd.DataFrame(y)
 y_val = pd.DataFrame(y_val )
scores = rmse(svr, X_train, y_train)
 print("Support Vector Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['SVR'] = scores.mean()<compute_test_metric>
models = []
preds = []
for n,(tr, te)in enumerate(KFold(n_splits=5, random_state=666, shuffle=True ).split(y)) :
print(f'Fold {n}')
 model = LGBMClassifier(**params)
 model.fit(X.values[tr], y.values[tr])
 pred = model.predict_proba(X_val)[:, 1]
preds.append(pred)
 print('Fold roc auc:', roc_auc_score(y.values[te], model.predict_proba(X.values[te])[:, 1]))
models.append(model )
scores = rmse(gbr, X_train, y_train)
 print("Gradient Boosting Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['GBR'] = scores.mean()<compute_test_metric>
predictions = preds[0]
for i in range(1, 5):
predictions += preds[i]
predictions /= 5
print('ROC AUC', roc_auc_score(y_val, predictions))
scores = rmse(elasticnet, X_train, y_train)
 print("ElasticNet Regressor
")
 print("Root Mean Square Error(RMSE)", str(scores.mean()))
print("Error Standard Deviation", str(scores.std()))
Scores['ElasticNet'] = scores.mean()<train_model>
iter_test = env.iter_test()
<train_model><EOS> 
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how="left")
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['part_1_boolean'].fillna(0, inplace = True)
 test_df['part_2_boolean'].fillna(0, inplace = True)
 test_df['part_3_boolean'].fillna(0, inplace = True)
 test_df['part_4_boolean'].fillna(0, inplace = True)
 test_df['part_5_boolean'].fillna(0, inplace = True)
 test_df['part_6_boolean'].fillna(0, inplace = True)
 test_df['part_7_boolean'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 full_preds = full_model.predict_proba(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])[:, 1]
full_preds_xgb = full_xgb.predict_proba(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])[:, 1]
full_preds_lr = full_lr.predict_proba(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])[:, 1]
X_test = rfe.transform(test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']])
 preds = [model.predict_proba(X_test)[:,1] for model in models]
predictions = preds[0]
for i in range(1, 5):
predictions += preds[i]
predictions /= 5
test_df['answered_correctly'] = predictions * 0.65 + full_preds * 0.15 + full_preds_xgb * 0.15 + full_preds_lr * 0.05
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )