Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<categorify>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1
<categorify>
_ = np.seterr(divide='ignore', invalid='ignore' )
lbl = LabelEncoder()
lbl.fit(list(train_data['Embarked'].values))
train_data['Embarked'] = lbl.transform(list(train_data['Embarked'].values))
lbl.fit(list(test['Embarked'].values))
test['Embarked'] = lbl.transform(list(test['Embarked'].values))<data_type_conversions>
%matplotlib inline
train_data['FareBin'] = pd.qcut(train_data['Fare'], 4)
 train_data['AgeBin'] = pd.cut(train_data['Age'].astype(int), 5)
 test['FareBin'] = pd.qcut(test['Fare'], 4)
 test['AgeBin'] = pd.cut(test['Age'].astype(int), 5 )<categorify>
SEED = 123
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED )
train_data['AgeBin_Code'] = lbl.fit_transform(train_data['AgeBin'])
 train_data['FareBin_Code'] = lbl.fit_transform(train_data['FareBin'])
 test['AgeBin_Code'] = lbl.fit_transform(test['AgeBin'])
 test['FareBin_Code'] = lbl.fit_transform(test['FareBin'] )<categorify>
data_types_dict = {
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id':'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'bool'
}
target = 'answered_correctly'
def encode(x): return 1 if x == 'female' else 0
train_data['enc_sex'] = train_data.Sex.apply(encode)
 test['enc_sex'] = test.Sex.apply(encode )<feature_engineering>
train_df = dt.fread('.. /input/riiid-test-answer-prediction/train.csv', columns=set(data_types_dict.keys())).to_pandas()
train_data["has_cabin"] = [0 if i == 'N'else 1 for i in train_data.Cabin]
test["has_cabin"] = [0 if i == 'N'else 1 for i in test.Cabin]<feature_engineering>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
def detect_outliers(train_data,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(train_data[col], 25)
 Q3 = np.percentile(train_data[col],75)
 IQR = Q3 - Q1
outlier_step = 1.5 * IQR
outlier_list_col = train_data[(train_data[col] < Q1 - outlier_step)|(train_data[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train_data,2,["Age","SibSp","Parch","Fare"])
<filter>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
train_data.loc[Outliers_to_drop]<drop_column>
train_lectures = train_df[train_df.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
train_data = train_data.drop(Outliers_to_drop, axis = 0 ).reset_index(drop=True )<prepare_x_and_y>
user_lecture_stats_part = train_lectures.groupby('user_id',as_index = False)[part_lectures_columns + types_of_lectures_columns].sum()
y_train = train_data["Survived"]
X_train = data.drop(labels = ["Survived"],axis = 1 )<concatenate>
lecturedata_types_dict = {
'user_id': 'int32',
'part_1': 'int8',
'part_2': 'int8',
'part_3': 'int8',
'part_4': 'int8',
'part_5': 'int8',
'part_6': 'int8',
'part_7': 'int8',
'type_of_concept': 'int8',
'type_of_intention': 'int8',
'type_of_solving_question': 'int8',
'type_of_starter': 'int8'
}
user_lecture_stats_part = user_lecture_stats_part.astype(lecturedata_types_dict )
test = test.select_dtypes(include=[np.number] ).interpolate().dropna()
test = test[X_train.columns] 
<normalization>
for column in user_lecture_stats_part.columns:
if(column !='user_id'):
user_lecture_stats_part[column] =(user_lecture_stats_part[column] > 0 ).astype('int8' )
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 test = sc.transform(test )<choose_model_class>
train_lectures[train_lectures.user_id==5382]
kfold = StratifiedKFold(n_splits=10 )<define_search_space>
user_lecture_stats_part[user_lecture_stats_part.user_id==5382]
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [4],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsExtC.fit(X_train,y_train)
 ExtC_best = gsExtC.best_estimator_
gsExtC.best_score_<define_search_space>
del(train_lectures )
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsRFC.fit(X_train,y_train)
 RFC_best = gsRFC.best_estimator_
gsRFC.best_score_<choose_model_class>
cum = train_df.groupby('user_id')['content_type_id'].agg(['cumsum', 'cumcount'])
 train_df['user_lecture_cumsum'] = cum['cumsum']
train_df['user_lecture_lv'] = cum['cumsum'] / cum['cumcount']
train_df.user_lecture_lv=train_df.user_lecture_lv.astype('float16')
 train_df.user_lecture_cumsum=train_df.user_lecture_cumsum.astype('int8')
 user_lecture_agg = train_df.groupby('user_id')['content_type_id'].agg(['sum', 'count'] )
DTC = DecisionTreeClassifier()
adaDTC = AdaBoostClassifier(DTC, random_state=7)
 ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"algorithm" : ["SAMME","SAMME.R"],
"n_estimators" :[30],
"learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsadaDTC.fit(X_train,y_train)
 ada_best = gsadaDTC.best_estimator_
gsadaDTC.best_score_<train_on_grid>
train_df['prior_question_had_explanation'].fillna(False, inplace=True)
 train_df = train_df.astype(data_types_dict)
 train_df = train_df[train_df[target] != -1].reset_index(drop=True)
 prior_question_elapsed_time_mean=train_df['prior_question_elapsed_time'].mean()
train_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True )
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1, 10, 50, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(X_train,y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_<choose_model_class>
max_timestamp_u = train_df[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsGBC.fit(X_train,y_train)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_<train_model>
train_df['lagtime'] = train_df.groupby('user_id')['timestamp'].shift()
train_df['lagtime']=train_df['timestamp']-train_df['lagtime']
train_df['lagtime'].fillna(0, inplace=True)
 train_df.lagtime=train_df.lagtime.astype('int32')
votingC = VotingClassifier(estimators=[('rfc', RFC_best),('extc', ExtC_best),('svm',SVMC_best), 
('gbc',GBC_best)], voting='soft', n_jobs=4)
 votingC = votingC.fit(X_train, y_train)
<save_to_csv>
lagtime_agg = train_df.groupby('user_id')['lagtime'].agg(['mean'])
 train_df['lagtime_mean'] = train_df['user_id'].map(lagtime_agg['mean'])
 train_df.lagtime_mean=train_df.lagtime_mean.astype('int32' )
test_Survived = pd.Series(votingC.predict(test), name="Survived")
 Submission = pd.concat([IDtest,test_Survived],axis=1)
 Submission.to_csv("submission.csv",index=False )<load_from_csv>
user_prior_question_elapsed_time = train_df[['user_id','prior_question_elapsed_time']].groupby(['user_id'] ).tail(1)
 user_prior_question_elapsed_time.columns = ['user_id', 'prior_question_elapsed_time']
df_train = pd.read_csv(r".. /input/train.csv")
 df_test = pd.read_csv(r".. /input/test.csv")
 pd.set_option('display.width', 700)
 df_train.head(5 )<feature_engineering>
train_df['delta_prior_question_elapsed_time'] = train_df.groupby('user_id')['prior_question_elapsed_time'].shift()
train_df['delta_prior_question_elapsed_time']=train_df['prior_question_elapsed_time']-train_df['delta_prior_question_elapsed_time']
train_df['delta_prior_question_elapsed_time'].fillna(0, inplace=True )
count_nans = len(df_train)- df_train.count()
df_count_nans = count_nans.to_frame()
df_count_nans.columns=["train_nan_count"]
df_count_nans["%_train_nans"]=(df_count_nans["train_nan_count"]/df_train.shape[0])* 100
df_count_nans["test_nan_count"] = len(df_test)- df_test.count()
df_count_nans["%_test_nans"]=(df_count_nans["test_nan_count"]/df_test.shape[0])* 100
df_count_nans.sort_values("train_nan_count", ascending=False, inplace=True)
 df_count_nans.query('train_nan_count > 0 or test_nan_count > 0' )<prepare_x_and_y>
train_df.delta_prior_question_elapsed_time=train_df.delta_prior_question_elapsed_time.astype('int32' )
y_train = df_train.SalePrice.values
print(y_train)
 df_train.drop("SalePrice", inplace=True, axis=1 )<concatenate>
train_df['timestamp']=train_df['timestamp']/(1000*3600)
 train_df.timestamp=train_df.timestamp.astype('int16')
df_all_data = pd.concat([df_train, df_test])
 df_all_data.reset_index(inplace=True, drop=True)
 print(df_all_data.shape)
 df_all_data.columns
df_all_data.head()<drop_column>
train_df['lag'] = train_df.groupby('user_id')[target].shift()
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['user_correctness'] = cum['cumsum'] / cum['cumcount']
train_df['user_correct_cumsum'] = cum['cumsum']
train_df['user_correct_cumcount'] = cum['cumcount']
train_df.drop(columns=['lag'], inplace=True)
 train_df['user_correct_cumsum'].fillna(0, inplace=True)
 train_df.user_correctness=train_df.user_correctness.astype('float16')
 train_df.user_correct_cumcount=train_df.user_correct_cumcount.astype('int16')
 train_df.user_correct_cumsum=train_df.user_correct_cumsum.astype('int16' )
df_all_data.drop(["PoolQC", "Alley", "MiscFeature", "Fence"], axis=1, inplace=True )<categorify>
train_df.prior_question_had_explanation=train_df.prior_question_had_explanation.astype('int8')
 train_df['lag'] = train_df.groupby('user_id')['prior_question_had_explanation'].shift()
df_all_data["FireplaceQu"].fillna("None", inplace=True)
 df_all_data[["GarageCond", "GarageType", "GarageFinish", "GarageQual"]] = df_all_data[["GarageCond", "GarageType", "GarageFinish", "GarageQual"]].fillna("None")
 df_all_data["GarageYrBlt"].fillna(0, inplace=True)
 df_all_data[["BsmtExposure","BsmtFinType1", "BsmtFinType2", "BsmtCond", "BsmtQual"]] = df_all_data[["BsmtExposure", "BsmtFinType1", "BsmtFinType2", "BsmtCond", "BsmtQual"]].fillna("None")
 df_all_data[["BsmtFullBath", "BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF", "TotalBsmtSF", "BsmtHalfBath"]] = df_all_data[["BsmtFullBath", "BsmtFinSF1", "BsmtFinSF2", "BsmtUnfSF", "TotalBsmtSF", "BsmtHalfBath"]].fillna(0)
 df_all_data["MasVnrArea"].fillna(0, inplace=True)
 df_all_data["MasVnrType"].fillna("None", inplace=True)
 most_common = df_all_data["Electrical"].value_counts().index[0]
df_all_data["Electrical"].fillna(most_common, inplace=True)
 df_all_data["Functional"].fillna("Typ", inplace=True)
 df_all_data["KitchenQual"].fillna("TA", inplace=True)
 most_common = df_all_data["SaleType"].value_counts().index[0]
df_all_data["SaleType"].fillna(most_common, inplace=True)
 df_all_data["Utilities"].fillna("None", inplace=True)
 df_all_data["Exterior1st"].fillna("None", inplace=True)
 df_all_data["Exterior2nd"].fillna("None", inplace=True)
 most_common = df_all_data["MSZoning"].value_counts().index[0]
df_all_data["MSZoning"].fillna(most_common, inplace=True )<groupby>
cum = train_df.groupby('user_id')['lag'].agg(['cumsum', 'cumcount'])
 train_df['explanation_mean'] = cum['cumsum'] / cum['cumcount']
train_df['explanation_cumsum'] = cum['cumsum']
train_df.drop(columns=['lag'], inplace=True)
 train_df['explanation_mean'].fillna(0, inplace=True)
 train_df['explanation_cumsum'].fillna(0, inplace=True)
 train_df.explanation_mean=train_df.explanation_mean.astype('float16')
 train_df.explanation_cumsum=train_df.explanation_cumsum.astype('int16' )
grp=df_all_data.groupby("Neighborhood")["GarageCars"].mean()
nan_idx = df_all_data[df_all_data["GarageCars"].isnull() ==True].index.tolist()
for idx in nan_idx:
df_all_data.loc[idx, "GarageCars"] = int(round(grp.loc[df_all_data.iloc[idx]["Neighborhood"]]))
grp=df_all_data.groupby("Neighborhood")["GarageArea"].mean()
nan_idx = df_all_data[df_all_data["GarageArea"].isnull() ==True].index.tolist()
for idx in nan_idx:
df_all_data.loc[idx, "GarageArea"] = int(round(grp.loc[df_all_data.iloc[idx]["Neighborhood"]]))<feature_engineering>
del cum
gc.collect()
df_all_data["LotFrontage"] = df_all_data.groupby("BldgType")["LotFrontage"].transform(lambda x: x.fillna(x.mean()))<data_type_conversions>
train_df["attempt_no"] = 1
train_df.attempt_no=train_df.attempt_no.astype('int8')
 train_df["attempt_no"] = train_df[["user_id","content_id",'attempt_no']].groupby(["user_id","content_id"])["attempt_no"].cumsum()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column] >= 0.75)]
print(to_drop )<drop_column>
explanation_agg = train_df.groupby('user_id')['prior_question_had_explanation'].agg(['sum', 'count'])
 explanation_agg=explanation_agg.astype('int16')
df_all_data.drop(to_drop, axis=1, inplace=True )<drop_column>
user_agg = train_df.groupby('user_id')[target].agg(['sum', 'count'])
 content_agg = train_df.groupby('content_id')[target].agg(['sum', 'count','var'])
 task_container_agg = train_df.groupby('task_container_id')[target].agg(['sum', 'count','var'])
drop_points = df_train.sort_values(by = 'GrLivArea', ascending = False)[:2]["GrLivArea"]
drop_points_list = drop_points.index.tolist()
df_all_data.drop(drop_points_list, inplace=True)
 y_train = np.delete(y_train, drop_points_list)
 df_train_last_index=df_train.shape[0]-len(drop_points_list )<categorify>
user_agg=user_agg.astype('int16')
 content_agg=content_agg.astype('float32')
 task_container_agg=task_container_agg.astype('float32' )
df_all_data['MSSubClass'] = df_all_data['MSSubClass'].apply(str)
 df_all_data["MSSubClass"] = LabelEncoder().fit_transform(df_all_data["MSSubClass"] )<sort_values>
attempt_no_agg=train_df.groupby(["user_id","content_id"])["attempt_no"].agg(['sum'])
 attempt_no_agg=attempt_no_agg.astype('int8')
skew_thresh = 0.5
skewed = df_all_data.skew().sort_values(ascending=False)
 a=skewed[abs(skewed)>skew_thresh]<feature_engineering>
train_df['content_count'] = train_df['content_id'].map(content_agg['count'] ).astype('int32')
 train_df['content_sum'] = train_df['content_id'].map(content_agg['sum'] ).astype('int32')
 train_df['content_correctness'] = train_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 train_df.content_correctness=train_df.content_correctness.astype('float16')
 train_df['task_container_sum'] = train_df['task_container_id'].map(task_container_agg['sum'] ).astype('int32')
 train_df['task_container_std'] = train_df['task_container_id'].map(task_container_agg['var'] ).astype('float16')
 train_df['task_container_correctness'] = train_df['task_container_id'].map(task_container_agg['sum'] / task_container_agg['count'])
 train_df.task_container_correctness=train_df.task_container_correctness.astype('float16' )
skewed_cols = skewed[abs(skewed)>skew_thresh].index.tolist()
print(len(skewed_cols))
print(skewed_cols)
 df_all_data[skewed_cols] = df_all_data[skewed_cols].apply(np.log1p)
 df_all_data[skewed_cols].head()<categorify>
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 1,3,4],
dtype={'question_id': 'int16','bundle_id': 'int16', 'part': 'int8','tags': 'str'} 
)
 questions_df['part_bundle_id']=questions_df['part']*100000+questions_df['bundle_id']
questions_df.part_bundle_id=questions_df.part_bundle_id.astype('int32')
 tag = questions_df["tags"].str.split(" ", n = 10, expand = True)
 tag.columns = ['tags1','tags2','tags3','tags4','tags5','tags6']
tag.fillna(0, inplace=True)
 tag = tag.astype('int16')
 questions_df = pd.concat([questions_df,tag],axis=1 ).drop(['tags'],axis=1 )
categorical_data_cols = df_all_data.select_dtypes(include=['object'])
 print(categorical_data_cols.columns.tolist())
 df_all_data = pd.get_dummies(df_all_data )<compute_test_metric>
questions_df.rename(columns={'question_id':'content_id'}, inplace=True )
y_train = np.log1p(y_train )<feature_engineering>
questions_df['content_correctness'] = questions_df['content_id'].map(content_agg['sum'] / content_agg['count'])
 questions_df.content_correctness=questions_df.content_correctness.astype('float16')
 questions_df['content_correctness_std'] = questions_df['content_id'].map(content_agg['var'])
 questions_df.content_correctness_std=questions_df.content_correctness_std.astype('float16' )
df_all_data =(df_all_data - df_all_data.mean())/(df_all_data.max() - df_all_data.min() )<train_model>
part_agg = questions_df.groupby('part')['content_correctness'].agg(['mean', 'var'])
 questions_df['part_correctness_mean'] = questions_df['part'].map(part_agg['mean'])
 questions_df['part_correctness_std'] = questions_df['part'].map(part_agg['var'])
 questions_df.part_correctness_mean=questions_df.part_correctness_mean.astype('float16')
 questions_df.part_correctness_std=questions_df.part_correctness_std.astype('float16' )
def _ApplyLinearAlgo(model_obj, df_tr, df_te, y_train):
model_obj.fit(df_tr, y_train)
 y_predict = model_obj.predict(df_tr)
 print("r2 score train " + str(r2_score(y_train, y_predict)))
 print("rmse score train " + str(mean_squared_error(y_train, y_predict)))
 print(df_tr.shape)
 print(df_te.shape)
 y_te_pred = np.expm1(model_obj.predict(df_te))
return y_te_pred<prepare_x_and_y>
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness=questions_df.bundle_correctness.astype('float16' )
print("
")
 print("ElasticNetCV")
 lr = ElasticNetCV(alphas=[0.0001, 0.0005, 0.001, 0.01, 0.1, 1, 10], l1_ratio=[.01,.1,.5,.9,.99], max_iter=5000)
 y_pred_Elastic = _ApplyLinearAlgo(lr, df_tr, df_te, y_train)
 print("
")
 print("
RidgeCV")
 lr=RidgeCV(alphas=[0.0001, 0.0005, 0.001, 0.01, 0.1, 1, 10])
 y_te_Ridge = _ApplyLinearAlgo(lr, df_tr, df_te, y_train)
 print("
")
 print("RandomForestRegressor")
 lr = RandomForestRegressor()
y_te_RF = _ApplyLinearAlgo(lr, df_tr, df_te, y_train )<save_to_csv>
tags1_agg = questions_df.groupby('tags1')['content_correctness'].agg(['mean', 'var'])
 questions_df['tags1_correctness_mean'] = questions_df['tags1'].map(tags1_agg['mean'])
 questions_df['tags1_correctness_std'] = questions_df['tags1'].map(tags1_agg['var'])
 questions_df.tags1_correctness_mean=questions_df.tags1_correctness_mean.astype('float16')
 questions_df.tags1_correctness_std=questions_df.tags1_correctness_std.astype('float16' )
idx = pd.read_csv(".. /input/test.csv" ).Id
my_submission = pd.DataFrame({'Id': idx, 'SalePrice': y_pred_Elastic})
 my_submission.to_csv('submission.csv', index=False)
 my_submission.head()<set_options>
questions_df.drop(columns=['content_correctness'], inplace=True )
%matplotlib inline 
<load_from_csv>
del bundle_agg
del part_agg
del tags1_agg
gc.collect()
train_df = pd.read_csv('.. /input/train.csv')
 test_df = pd.read_csv('.. /input/test.csv')
 combine = [train_df, test_df]<feature_engineering>
train_df['user_correctness'].fillna(1, inplace=True)
 train_df['attempt_no'].fillna(1, inplace=True)
 train_df.fillna(0, inplace=True )
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )<feature_engineering>
MAX_SEQ = 160
skills = train_df["content_id"].unique()
n_skill = len(skills)
 print("number skills", len(skills))
group = train_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for user_id in group.index:
q, qa = group[user_id]
if len(q)>MAX_SEQ:
group[user_id] =(q[-MAX_SEQ:],qa[-MAX_SEQ:])
 pickle.dump(group, open("group.pkl", "wb"))
del group
gc.collect()
for dataset in combine:
dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Capt', 'Col',\
	'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 train_df[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<categorify>
features = [
'lagtime',
'lagtime_mean',
'user_lecture_cumsum',
'user_lecture_lv',
'prior_question_elapsed_time',
'delta_prior_question_elapsed_time',
'user_correctness',
'user_correct_cumcount',
'user_correct_cumsum',
'content_correctness',
'content_count',
'content_sum',
'task_container_correctness',
'bundle_correctness',
'attempt_no',
'part',
'part_correctness_mean',
'tags1',
'tags1_correctness_mean',
'bundle_id',
'explanation_mean',
'explanation_cumsum',
'prior_question_had_explanation',
]
categorical_columns= [
'part',
'tags1',
'bundle_id',
'prior_question_had_explanation',
]
title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
for dataset in combine:
dataset['Title'] = dataset['Title'].map(title_mapping)
 dataset['Title'] = dataset['Title'].fillna(0)
 train_df.head()<drop_column>
flag_lgbm=True
clfs = list()
train_df = train_df.drop(['Name', 'PassengerId'], axis=1)
 test_df = test_df.drop(['Name'], axis=1)
 combine = [train_df, test_df]
train_df.shape, test_df.shape<data_type_conversions>
params = pd.read_pickle('.. /input/the-best-of-all-parameters-on-lightgbm/The_best_of_all_parameters_on_LightGBM.pickle')
 print('Optimized parameters with Optuna:')
 print(params )
for dataset in combine:
dataset['Sex'] = dataset['Sex'].map({'female': 1, 'male': 0} ).astype(int)
 train_df.head()<define_variables>
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df.sample(n=20000*1000, random_state = SEED)
 print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.025, random_state = SEED)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 valid_df=train_df_clf.sample(frac=0.09, random_state = SEED)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('valid_df length：',len(valid_df))
guess_ages = np.zeros(( 2,3))
guess_ages<find_best_params>
del train_df_clf
del valid_df
gc.collect()
for dataset in combine:
for i in range(0, 2):
for j in range(0, 3):
guess_df = dataset[(dataset['Sex'] == i)& \ 
(dataset['Pclass'] == j+1)]['Age'].dropna()
age_guess = guess_df.median()
guess_ages[i,j] = int(age_guess/0.5 + 0.5)* 0.5
for i in range(0, 2):
for j in range(0, 3):
dataset.loc[(dataset.Age.isnull())&(dataset.Sex == i)&(dataset.Pclass == j+1),\
'Age'] = guess_ages[i,j]
dataset['Age'] = dataset['Age'].astype(int)
 train_df.head()<sort_values>
for i in range(0, num):
tr_data = lgb.Dataset(trains[i][features], label=trains[i][target])
 va_data = lgb.Dataset(valids[i][features], label=valids[i][target], reference = tr_data)
 del trains
del valids
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=10000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 del tr_data
del va_data
gc.collect()
train_df['AgeBand'] = pd.cut(train_df['Age'], 5)
 train_df[['AgeBand', 'Survived']].groupby(['AgeBand'], as_index=False ).mean().sort_values(by='AgeBand', ascending=True )<feature_engineering>
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int))
content_sum_dict = content_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
content_count_dict = content_agg['count'].astype('int32' ).to_dict(defaultdict(int))
del user_agg
del content_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()
for dataset in combine:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 32), 'Age'] = 1
dataset.loc[(dataset['Age'] > 32)&(dataset['Age'] <= 48), 'Age'] = 2
dataset.loc[(dataset['Age'] > 48)&(dataset['Age'] <= 64), 'Age'] = 3
dataset.loc[ dataset['Age'] > 64, 'Age']
train_df.head()<drop_column>
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
lagtime_mean_dict = lagtime_agg['mean'].astype('int32' ).to_dict(defaultdict(int))
del user_lecture_agg
del lagtime_agg
gc.collect()
train_df = train_df.drop(['AgeBand'], axis=1)
 combine = [train_df, test_df]
train_df.head()<sort_values>
attempt_no_agg=attempt_no_agg[attempt_no_agg['sum'] >1]
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()
for dataset in combine:
dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1
train_df[['FamilySize', 'Survived']].groupby(['FamilySize'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<feature_engineering>
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del user_prior_question_elapsed_time
gc.collect()
for dataset in combine:
dataset['IsAlone'] = 0
dataset.loc[dataset['FamilySize'] == 1, 'IsAlone'] = 1
train_df[['IsAlone', 'Survived']].groupby(['IsAlone'], as_index=False ).mean()<drop_column>
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
train_df = train_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 test_df = test_df.drop(['Parch', 'SibSp', 'FamilySize'], axis=1)
 combine = [train_df, test_df]
train_df.head()<feature_engineering>
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
class TestDataset(Dataset):
def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples = samples
self.user_ids = [x for x in test_df["user_id"].unique() ]
self.test_df = test_df
self.skills = skills
self.n_skill = len(skills)
 self.max_seq = max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info["user_id"]
target_id = test_info["content_id"]
q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 if seq_len >= self.max_seq:
q = q_[-self.max_seq:]
qa = qa_[-self.max_seq:]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[1:].copy()
x +=(qa[1:] == 1)* self.n_skill
questions = np.append(q[2:], [target_id])
 return x, questions
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 SAKT_model = SAKTModel(n_skill, embed_dim=128)
 try:
SAKT_model.load_state_dict(torch.load(".. /input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt"))
except:
SAKT_model.load_state_dict(torch.load(".. /input/sakt-with-randomization-state-updates/SAKT-HDKIM.pt", map_location='cpu'))
SAKT_model.to(device)
 SAKT_model.eval()
for dataset in combine:
dataset['Age*Class'] = dataset.Age * dataset.Pclass
train_df.loc[:, ['Age*Class', 'Age', 'Pclass']].head(10 )<correct_missing_values>
group = pickle.load(open("group.pkl", "rb"))
freq_port = train_df.Embarked.dropna().mode() [0]
freq_port<sort_values>
env = riiideducation.make_env()
iter_test = env.iter_test()
prior_test_df = None
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].fillna(freq_port)
 train_df[['Embarked', 'Survived']].groupby(['Embarked'], as_index=False ).mean().sort_values(by='Survived', ascending=False )<data_type_conversions>
%%time
for(test_df, sample_prediction_df)in iter_test:
if(prior_test_df is not None)&(psutil.virtual_memory().percent<90):
print(psutil.virtual_memory().percent)
 prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 prev_group = prior_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
prev_group_content = prev_group[prev_user_id][0]
prev_group_ac = prev_group[prev_user_id][1]
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0],prev_group_content),
np.append(group[prev_user_id][1],prev_group_ac))
else:
group[prev_user_id] =(prev_group_content,prev_group_ac)
 if len(group[prev_user_id][0])>MAX_SEQ:
new_group_content = group[prev_user_id][0][-MAX_SEQ:]
new_group_ac = group[prev_user_id][1][-MAX_SEQ:]
group[prev_user_id] =(new_group_content,new_group_ac)
 user_ids = prior_test_df['user_id'].values
content_ids = prior_test_df['content_id'].values
task_container_ids = prior_test_df['task_container_id'].values
prior_question_had_explanations = prior_test_df['prior_question_had_explanation'].values
targets = prior_test_df[target].values
for user_id, content_id,prior_question_had_explanation,task_container_id,answered_correctly in zip(user_ids, content_ids, prior_question_had_explanations,task_container_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
prior_test_df = test_df.copy()
lecture_test_df = test_df[test_df['content_type_id'] == 1].reset_index(drop=True)
 for i,(user_id,content_type_id, content_id)in enumerate(zip(lecture_test_df['user_id'].values,lecture_test_df['content_type_id'].values,lecture_test_df['content_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(len(user_lecture_stats_part[user_lecture_stats_part.user_id==user_id])==0):
user_lecture_stats_part = user_lecture_stats_part.append([{'user_id':user_id}], ignore_index=True)
 user_lecture_stats_part.fillna(0, inplace=True)
 user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
else:
user_lecture_stats_part.loc[user_lecture_stats_part.user_id==user_id,part_lectures_columns + types_of_lectures_columns]+=lectures_df[lectures_df.lecture_id==content_id][part_lectures_columns + types_of_lectures_columns].values
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 SAKT_outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = SAKT_model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
SAKT_outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(len(test_df), dtype=np.int16)
 user_lecture_count = np.zeros(len(test_df), dtype=np.int16)
 user_sum = np.zeros(len(test_df), dtype=np.int16)
 user_count = np.zeros(len(test_df), dtype=np.int16)
 content_sum = np.zeros(len(test_df), dtype=np.int32)
 content_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_sum = np.zeros(len(test_df), dtype=np.int32)
 task_container_count = np.zeros(len(test_df), dtype=np.int32)
 task_container_std = np.zeros(len(test_df), dtype=np.float16)
 content_task_mean = np.zeros(len(test_df), dtype=np.float16)
 explanation_sum = np.zeros(len(test_df), dtype=np.int32)
 explanation_count = np.zeros(len(test_df), dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(len(test_df), dtype=np.int32)
 attempt_no_count = np.zeros(len(test_df), dtype=np.int16)
 lagtime = np.zeros(len(test_df), dtype=np.int32)
 lagtime_mean = np.zeros(len(test_df), dtype=np.int32)
 for i,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
content_sum[i] = content_sum_dict[content_id]
content_count[i] = content_count_dict[content_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
else:
lagtime[i]=0
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime_mean_dict.update({user_id:timestamp})
 lagtime_mean[i]=(lagtime_mean_dict[user_id]+lagtime[i])/2
if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=0
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_cumsum'] = user_lecture_sum
test_df['user_correctness'] = user_sum / user_count
test_df['user_correct_cumcount'] =user_count
test_df['user_correct_cumsum'] =user_sum
test_df['content_correctness'] = content_sum / content_count
test_df['content_count'] = content_count
test_df['content_sum'] = content_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_sum'] = task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_cumsum'] = explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime_mean"]=lagtime_mean
test_df['user_correctness'].fillna(1, inplace=True)
 test_df['attempt_no'].fillna(1, inplace=True)
 test_df.fillna(0, inplace=True)
 test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('int16')
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features], num_iteration = model.best_iteration)
 sub_preds += test_preds
lgbm_final = sub_preds / len(clfs)
 test_df[target] = np.array(SAKT_outs)* 0.5 + lgbm_final * 0.5
env.predict(test_df[['row_id', target]] )
for dataset in combine:
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} ).astype(int)
 train_df.head()<data_type_conversions>
pickle.dump(test_df, open('test_df.pickle', 'wb'))
test_df['Fare'].fillna(test_df['Fare'].dropna().median() , inplace=True)
 test_df.head()<sort_values>
model.save_model('model.txt')
 print('Save the best of model.' )
train_df['FareBand'] = pd.qcut(train_df['Fare'], 4)
 train_df[['FareBand', 'Survived']].groupby(['FareBand'], as_index=False ).mean().sort_values(by='FareBand', ascending=True )<data_type_conversions>
pickle.dump(params, open('The_best_of_all_parameters_on_LightGBM.pickle', 'wb'))
print('Save the best of parameters.' )
for dataset in combine:
dataset.loc[ dataset['Fare'] <= 7.91, 'Fare'] = 0
dataset.loc[(dataset['Fare'] > 7.91)&(dataset['Fare'] <= 14.454), 'Fare'] = 1
dataset.loc[(dataset['Fare'] > 14.454)&(dataset['Fare'] <= 31), 'Fare'] = 2
dataset.loc[ dataset['Fare'] > 31, 'Fare'] = 3
dataset['Fare'] = dataset['Fare'].astype(int)
 train_df = train_df.drop(['FareBand'], axis=1)
 combine = [train_df, test_df]
train_df.head(10 )<import_modules>
target = 'answered_correctly'
y_true = pd.read_csv('.. /input/riiid-test-answer-prediction/train.csv', usecols=[target])
 print('Length of y_true =', len(y_true))
y_pred = pd.read_csv('./submission.csv', usecols=[target])
 print('Length of y_pred =', len(y_pred))
test_df = pd.read_pickle('./test_df.pickle')
 print('Length of test_df =', len(test_df))
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import BaggingClassifier<import_modules>
params = pd.read_pickle('./The_best_of_all_parameters_on_LightGBM.pickle')
 print('Optimized parameters with Optuna:')
 print(params )
from sklearn.model_selection import train_test_split<import_modules>
model = lgb.Booster(model_file='./model.txt', params=params)
 print('Load the model of LightGBM:')
 print(model )
from sklearn.model_selection import train_test_split<split>
LightGBM = model.predict(test_df[features])
 LightGBM
X_train, X_test, y_train, y_test = train_test_split(train_df.drop("Survived", axis=1), train_df["Survived"], test_size=0.3 )<import_modules>
y_true_boot = resample(y_true, n_samples = len(y_pred))
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix<train_model>
def Statistical_Significance_Test(y_true_boot, y_pred):
A = np.array(y_true_boot)
 B = np.array(y_pred)
 print('Paired t-test:', stats.ttest_rel(A, B))
A_var = np.var(A, ddof=1)
 B_var = np.var(B, ddof=1)
 A_df = len(A)- 1
B_df = len(B)- 1
f = A_var / B_var
one_sided_pval1 = stats.f.cdf(f, A_df, B_df)
 one_sided_pval2 = stats.f.sf(f, A_df, B_df)
 two_sided_pval = min(one_sided_pval1, one_sided_pval2)* 2
print("Student's t-test:")
 print('F: ', round(f, 3))
print('p-value: ', round(two_sided_pval, 3))
two_sided_pval = min(one_sided_pval1, one_sided_pval2)* 2
print("Welch's t-test:")
 print('F: ', round(f, 3))
print('p-value: ', round(two_sided_pval, 4))
print("Mann-Whitney's U-test:", stats.mannwhitneyu(A, B, alternative='two-sided'))
print("Wilcoxon signed rank sum test:", stats.wilcoxon(A, B))
clf = DecisionTreeClassifier(random_state=42)
 clf.fit(X_train, y_train)
 print_score(clf, X_train, y_train, X_test, y_test, train=True)
 print_score(clf, X_train, y_train, X_test, y_test, train=False)
<train_model>
Statistical_Significance_Test(y_true_boot[target], y_pred[target] )
bag_clf = BaggingClassifier(base_estimator=clf, n_estimators=1000,
bootstrap=True, n_jobs=-1,
random_state=42)
 bag_clf.fit(X_train, y_train)
 print_score(bag_clf, X_train, y_train, X_test, y_test, train=True)
 print_score(bag_clf, X_train, y_train, X_test, y_test, train=False )<choose_model_class>
distribution(y_pred[target] )
<train_model><EOS> 
print('Operation completed.' )