Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<set_options>
import riiideducation
import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn import preprocessing
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_validate, cross_val_predict, cross_val_score, train_test_split, validation_curve, ShuffleSplit, RandomizedSearchCV
from sklearn.metrics import recall_score, precision_score, accuracy_score, confusion_matrix, classification_report
import matplotlib.pyplot as plt
%matplotlib inline
print(os.listdir(".. /input/house-prices-advanced-regression-techniques")) 
<load_from_csv>
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
nrows=5000000,
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)
train_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 test_data = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 sample_submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 data_desc = open(".. /input/house-prices-advanced-regression-techniques/data_description.txt")
 print(data_desc.read() )<create_dataframe>
n = len(train)
 n
data_null = train_data.isnull().sum() /len(train_data)* 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio': data_null})
 missing_data.head(10 )<sort_values>
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv' )
corr_data["SalePrice"].sort_values(ascending=False )<prepare_x_and_y>
questions_df['num_tags'] = questions_df['tags'].apply(lambda x:len(x.split())if pd.notna(x)else 0)
y_train = train_data["SalePrice"]
data = train_data.drop(["SalePrice"], axis=1 )<sort_values>
questions_df = questions_df[['question_id','part','num_tags']]
train_data.sort_values(by = 'GrLivArea', ascending = False)[:2]<drop_column>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
train_data = train_data.drop(train_data[train_data['Id'] == 1299].index)
 train_data = train_data.drop(train_data[train_data['Id'] == 524].index )<predict_on_test>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'] )
def find_outliers(model, X, y, sigma=3):
try:
y_pred = pd.Series(model.predict(X), index=y.index)
 except:
model.fit(X,y)
 y_pred = pd.Series(model.predict(X), index=y.index)
 resid = y - y_pred
mean_resid = resid.mean()
std_resid = resid.std()
z =(resid - mean_resid)/std_resid
outliers = z[abs(z)>sigma].index
return outliers<categorify>
part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
def changeSeqCat(data):
cols_ExGd = ['ExterQual','ExterCond','BsmtQual','BsmtCond',
'HeatingQC','KitchenQual','FireplaceQu','GarageQual',
'GarageCond','PoolQC']
dict_ExGd = {'Ex':5,'Gd':4,'TA':3,'Fa':2,'Po':1,'None':0}
for col in cols_ExGd:
data[col].replace(dict_ExGd, inplace=True)
 data['BsmtExposure'].replace({'Gd':4,'Av':3,'Mn':2,'No':1,'None':0}, inplace=True)
 data['CentralAir'].replace({'Y':1,'N':0}, inplace=True)
 data['Functional'].replace({'Typ':7,'Min1':6,'Min2':5,'Mod':4,'Maj1':3,'Maj2':2,'Sev':1,'Sal':0}, inplace=True)
 data['GarageFinish'].replace({'Fin':3,'RFn':2,'Unf':1,'None':0}, inplace=True)
 data['LotShape'].replace({'Reg':3,'IR1':2,'IR2':1,'IR3':0}, inplace=True)
 data['Utilities'].replace({'AllPub':3,'NoSewr':2,'NoSeWa':1,'ELO':0}, inplace=True)
 data['LandSlope'].replace({'Gtl':2,'Mod':1,'Sev':0}, inplace=True)
 return data<data_type_conversions>
train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
def setMissingData(data):
data['MSZoning'] = data['MSZoning'].fillna(data['MSZoning'].mode() [0])
 data['LotFrontage'] = data['LotFrontage'].fillna(data['LotFrontage'].mean())
 data.OverallCond = data.OverallCond.astype(str)
 data['Fence'] = data['Fence'].fillna('NA')
 data['Alley'] = data['Alley'].fillna('NA')
 data['Functional'] = data['Functional'].fillna('Typ')
 data['MasVnrType'] = data['MasVnrType'].fillna(data['MasVnrType'].mode() [0])
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
data[col] = data[col].fillna('None')
 data['TotalBsmtSF'] = data['TotalBsmtSF'].fillna(0)
 data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0])
 data['FireplaceQu'] = data['FireplaceQu'].fillna('None')
 for col in('GarageType', 'GarageFinish', 'GarageQual'):
data[col] = data[col].fillna('None')
 data['GarageCars'] = data['GarageCars'].fillna(0.0)
 data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0])
 data['YrSold'] = data['YrSold'].astype(str)
 data['MoSold'] = data['MoSold'].astype(str)
 data['TotalSF'] = data['TotalBsmtSF'] + data['1stFlrSF'] + data['2ndFlrSF']
data['YrBltAndRemod']=data['YearBuilt']+data['YearRemodAdd']
data['Total_sqr_footage'] =(data['BsmtFinSF1'] + data['BsmtFinSF2'] +
data['1stFlrSF'] + data['2ndFlrSF'])
 data['Total_Bathrooms'] =(data['FullBath'] +(0.5 * data['HalfBath'])+
data['BsmtFullBath'] +(0.5 * data['BsmtHalfBath']))
data['Total_porch_sf'] =(data['OpenPorchSF'] + data['3SsnPorch'] +
data['EnclosedPorch'] + data['ScreenPorch'] +
data['WoodDeckSF'])
 data['haspool'] = data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 data['has2ndfloor'] = data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasgarage'] = data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 data['hasbsmt'] = data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 data['hasfireplace'] = data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0)
 return data<sort_values>
user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()
def handleSkew(all_data):
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 print("
Skew in numerical features:
")
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness.head(10)
 skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam)
 return all_data<train_model>
for column in user_lecture_stats_part.columns:
bool_column = column + '_boolean'
user_lecture_stats_part[bool_column] =(user_lecture_stats_part[column] > 0 ).astype(int )
class LabelBinarizerPipelineFriendly(MultiLabelBinarizer):
def fit(self, X, y=None):
super(LabelBinarizerPipelineFriendly,self ).fit(X)
 def transform(self, X, y=None):
return super(LabelBinarizerPipelineFriendly, self ).transform(X)
 def fit_transform(self, X, y=None):
return super(LabelBinarizerPipelineFriendly, self ).fit(X ).transform(X )<train_model>
del(train_lectures )
class DataFrameSelector(BaseEstimator, TransformerMixin):
def __init__(self, attr):
self.attributes = attr
def fit(self, X, y=None):
return self
def transform(self, X):
return X[self.attributes].values<categorify>
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )
class LabelEncoderCat(BaseEstimator, TransformerMixin):
def fit(self, X, y=None):
return self
def transform(self, X, y=None):
X = pd.DataFrame(X)
 X = X.apply(LabelEncoder().fit_transform)
 return X<drop_column>
train[(train.content_type_id == False)].task_container_id.nunique()
id_test = test_data['Id']
data = setMissingData(data)
 data = changeSeqCat(data)
 data = handleSkew(data )<define_variables>
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2
cat_attr = ['MSZoning', 'LandContour', 'Neighborhood', 'Condition1',
'BldgType', 'RoofMatl', 'Exterior1st', 'LotConfig', 'Alley',
'MasVnrType', 'BsmtFinType1', 'BsmtFinType2', 'Electrical',
'GarageType', 'SaleCondition', 'BsmtFinType1', 'RoofStyle']
num_attr = ['LotFrontage', 'LotArea', 'OverallQual', 'TotalSF', 'CentralAir',
'YearBuilt', 'BsmtFinSF1', 'BsmtUnfSF', 'BsmtExposure', 'GarageArea',
'GrLivArea', 'BsmtFullBath', 'FullBath', 'TotRmsAbvGrd', 'GarageCond',
'Fireplaces', 'WoodDeckSF','OpenPorchSF', 'GarageArea', 'GarageCars',
'haspool', 'has2ndfloor', 'hasgarage', 'hasbsmt', 'hasfireplace',
'YrBltAndRemod', 'Total_sqr_footage', 'Total_Bathrooms', 'Total_porch_sf',
'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageQual', 'CentralAir',
'BsmtCond', 'BsmtQual', 'OverallCond', 'Functional', 'ExterQual']
num_pipeline = Pipeline([ 
('selector', DataFrameSelector(num_attr)) , 
('imputer', SimpleImputer(strategy="median")) , 
('scaler', StandardScaler()),
])
 cat_pipeline = Pipeline([ 
('selector', DataFrameSelector(cat_attr)) , 
('imputer', SimpleImputer(strategy="most_frequent")) , 
('label_binarizer', LabelBinarizerPipelineFriendly()),
])
 full_pipeline = FeatureUnion(transformer_list=[ 
('num_pipeline', num_pipeline), 
('cat_pipeline', cat_pipeline),
] )<compute_train_metric>
group3['avg_questions_seen'] = group3.avg_questions.cumsum()
n_folds = 5
def rmse_cv(model, X_data, y_data):
rmse= np.sqrt(-cross_val_score(model, X_data, y_data, scoring="neg_mean_squared_error", cv = 5))
return(rmse )<create_dataframe>
group3.iloc[10].avg_questions_seen
X_prepared = pd.DataFrame(full_pipeline.fit_transform(data))
X_prepared = pd.DataFrame(X_prepared )<create_dataframe>
results_u_final = train.loc[train.content_type_id == False, ['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_final.columns = ['answered_correctly_user']
results_u2_final = train.loc[train.content_type_id == False, ['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_final.columns = ['explanation_mean_user']
data_null = data[cat_attr].isnull().sum() /len(data[cat_attr])* 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio': data_null})
 missing_data<categorify>
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )
final_data = pd.get_dummies(data[cat_attr] ).reset_index(drop=True)
 F, p_value = f_regression(final_data, y_train)
 np.array(final_data.columns)+ " = " +(p_value < 0.05 ).astype(str )<drop_column>
results_q_final = train.loc[train.content_type_id == False, ['question_id','answered_correctly']].groupby(['question_id'] ).agg(['mean'])
 results_q_final.columns = ['quest_pct']
outliers = find_outliers(Ridge() , X_prepared, y_train)
 X_prepared = X_prepared.drop(outliers)
 y_train = y_train.drop(outliers)
 y_train = np.log1p(y_train )<compute_train_metric>
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']
lin_reg = LinearRegression()
score = rmse_cv(lin_reg, X_prepared, y_train)
 score.mean()<compute_train_metric>
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left')
ridge_reg = Ridge(alpha=0.001, solver="cholesky")
 score = rmse_cv(ridge_reg, X_prepared, y_train)
 score.mean()<compute_train_metric>
question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )
lasso_reg = Lasso(alpha=0.0003)
 score = rmse_cv(lasso_reg, X_prepared, y_train)
 score.mean()<compute_train_metric>
question2.quest_pct = round(question2.quest_pct,5 )
elastic_net = ElasticNet(alpha=0.0005, l1_ratio=0.65)
 score = rmse_cv(elastic_net, X_prepared, y_train)
 score.mean()<compute_train_metric>
prior_mean_user = results_u2_final.explanation_mean_user.mean()
prior_mean_user
ransac_reg = RANSACRegressor(lasso_reg)
 score = rmse_cv(ridge_reg, X_prepared, y_train)
 score.mean()<compute_train_metric>
train.drop(['timestamp', 'content_type_id', 'question_id', 'part', 'num_tags'], axis=1, inplace=True )
random_reg = RandomForestRegressor(max_depth=10, random_state=42, n_estimators=200)
 score = rmse_cv(random_reg, X_prepared, y_train)
 score.mean()<compute_train_metric>
validation = train.groupby('user_id' ).tail(5)
 train = train[~train.index.isin(validation.index)]
len(train)+ len(validation )
huber_reg = HuberRegressor(epsilon=5)
 score = rmse_cv(huber_reg, X_prepared, y_train)
 score.mean()<compute_train_metric>
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']
svm_reg = LinearSVR(epsilon=0.001)
 score = rmse_cv(svm_reg, X_prepared, y_train)
 score.mean()<compute_train_metric>
X = train.groupby('user_id' ).tail(18)
 train = train[~train.index.isin(X.index)]
len(X)+ len(train)+ len(validation )
adf_clf = AdaBoostRegressor(DecisionTreeRegressor(max_depth=3), n_estimators=3000, learning_rate=0.05,
loss='square')
 score = rmse_cv(adf_clf, X_prepared, y_train)
 score.mean()<train_model>
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']
gbrt = GradientBoostingRegressor(max_depth=3, n_estimators=3000, learning_rate=0.05,
max_features='sqrt', loss='huber')
 gbrt.fit(X_prepared, y_train)
 errors = [mean_squared_error(y_train, y_pred)for y_pred in gbrt.staged_predict(X_prepared)]
bst_n_estimators = np.argmin(errors)
 bst_n_estimators<compute_test_metric>
del(train )
gbrt = GradientBoostingRegressor(max_depth=3, n_estimators=bst_n_estimators, learning_rate=0.05, max_features='sqrt',
loss='huber')
 score = rmse_cv(gbrt, X_prepared, y_train)
 score.mean()<train_model>
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left")
 X = pd.merge(X, user_lecture_stats_part, on=['user_id'], how="left" )
adf_clf.fit(X_prepared, y_train)
 gbrt.fit(X_prepared, y_train)
 lasso_reg.fit(X_prepared, y_train)
 elastic_net.fit(X_prepared, y_train)
 random_reg.fit(X_prepared, y_train)
 ridge_reg.fit(X_prepared, y_train)
 svm_reg.fit(X_prepared, y_train)
 ransac_reg.fit(X_prepared, y_train )<compute_train_metric>
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left")
 validation = pd.merge(validation, user_lecture_stats_part, on=['user_id'], how="left" )
voting_reg = VotingRegressor([ 
("gradient_boost", gbrt), 
("lasso_reg", lasso_reg), 
("random_reg", random_reg), 
("elastic_net", elastic_net),
])
 score = rmse_cv(voting_reg, X_prepared, y_train)
 score.mean()<train_model>
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )
voting_reg.fit(X_prepared, y_train )<train_model>
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y=None):
X = X.values
y = y.values
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=42)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def get_metafeatures(self, X):
return np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features )<compute_train_metric>
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )
stacked_averaged_models = StackingAveragedModels(base_models =(elastic_net, gbrt, ridge_reg, random_reg),
meta_model = lasso_reg)
 score = rmse_cv(stacked_averaged_models, X_prepared, y_train)
 score.mean()<concatenate>
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1
test_data = setMissingData(test_data)
 test_data = changeSeqCat(test_data)
 test_data = handleSkew(test_data )<create_dataframe>
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_valdf = validation.drop(['answered_correctly'], axis=1 )
test_prepared = full_pipeline.transform(test_data)
 test_prepared = pd.DataFrame(test_prepared )<predict_on_test>
X_train = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean', 'num_tags',]]
X_val = X_valdf[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean', 'num_tags',]]
stacked_averaged_models.fit(X_prepared, y_train)
 stack_y_pred = stacked_averaged_models.predict(test_prepared)
 voting_reg_y_pred = voting_reg.predict(test_prepared )<prepare_output>
X_train['answered_correctly_user'].fillna(0.65, inplace=True)
 X_train['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_train['quest_pct'].fillna(content_mean, inplace=True)
 X_train['num_tags'].fillna(0, inplace=True)
 X_train['part'].fillna(4, inplace = True)
 X_train['avg_questions_seen'].fillna(1, inplace = True)
 X_train['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_train['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X_train['part_1'].fillna(0, inplace = True)
 X_train['part_2'].fillna(0, inplace = True)
 X_train['part_3'].fillna(0, inplace = True)
 X_train['part_4'].fillna(0, inplace = True)
 X_train['part_5'].fillna(0, inplace = True)
 X_train['part_6'].fillna(0, inplace = True)
 X_train['part_7'].fillna(0, inplace = True)
 X_train['type_of_concept'].fillna(0, inplace = True)
 X_train['type_of_intention'].fillna(0, inplace = True)
 X_train['type_of_solving_question'].fillna(0, inplace = True)
 X_train['type_of_starter'].fillna(0, inplace = True)
 X_train['type_of_concept_boolean'].fillna(0, inplace = True)
 X_train['type_of_intention_boolean'].fillna(0, inplace = True)
 X_train['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X_train['type_of_starter_boolean'].fillna(0, inplace = True )
y_pred = np.expm1(stack_y_pred)* 0.75 + np.expm1(voting_reg_y_pred)* 0.25<create_dataframe>
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X_val['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X_val['num_tags'].fillna(0, inplace=True)
 X_val['part_1'].fillna(0, inplace = True)
 X_val['part_2'].fillna(0, inplace = True)
 X_val['part_3'].fillna(0, inplace = True)
 X_val['part_4'].fillna(0, inplace = True)
 X_val['part_5'].fillna(0, inplace = True)
 X_val['part_6'].fillna(0, inplace = True)
 X_val['part_7'].fillna(0, inplace = True)
 X_val['type_of_concept'].fillna(0, inplace = True)
 X_val['type_of_intention'].fillna(0, inplace = True)
 X_val['type_of_solving_question'].fillna(0, inplace = True)
 X_val['type_of_starter'].fillna(0, inplace = True)
 X_val['type_of_concept_boolean'].fillna(0, inplace = True)
 X_val['type_of_intention_boolean'].fillna(0, inplace = True)
 X_val['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X_val['type_of_starter_boolean'].fillna(0, inplace = True )
submission = pd.DataFrame({
"Id": id_test,
"SalePrice": y_pred
})
 submission<save_to_csv>
X_train_values = X_train.values
min_max_scaler = preprocessing.MinMaxScaler()
x_train_scaled = min_max_scaler.fit_transform(X_train_values)
 x_train_scaled
submission.to_csv('prediction.csv', index=False )<import_modules>
X_val_values = X_val.values
min_max_scaler = preprocessing.MinMaxScaler()
x_val_scaled = min_max_scaler.fit_transform(X_val_values)
 x_val_scaled
import keras
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten
from keras.optimizers import RMSprop
from keras.callbacks import EarlyStopping, ReduceLROnPlateau
from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split<load_from_csv>
y_train = y.values
y_val = y_val.values
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 y_train = train['label']
x_train = train.drop(labels=['label'] ,axis=1)
 del train
x_train.isnull().any().describe()
test.isnull().any().describe()<feature_engineering>
from tensorflow.keras import models, Sequential
from tensorflow.keras.layers import Embedding
x_train = x_train / 255.0
test = test / 255.0<concatenate>
dim = x_train_scaled.shape[1]
NUM_CLASSES = 10
y_train = to_categorical(y_train, num_classes=NUM_CLASSES )<split>
dim = x_train_scaled.shape[1]
inputs = tf.keras.Input(shape=(dim,))
x = tf.keras.layers.Dense(16, activation='relu' )(inputs)
 x = tf.keras.layers.Dropout(0.3 )(x)
 x = tf.keras.layers.Dense(16, activation='relu' )(x)
 x = tf.keras.layers.Dropout(0.3 )(x)
 out = tf.keras.layers.Dense(1, activation="sigmoid" )(x)
 model_DNN = tf.keras.Model(inputs=inputs, outputs = out)
 model_DNN.compile(optimizer=tf.keras.optimizers.Adam(lr=0.0002), loss="binary_crossentropy", metrics=[tf.keras.metrics.Precision() ])
 model_DNN.summary()
random_seed = 2
x_train, x_test, y_train, y_test = train_test_split(x_train, y_train, test_size=0.1, random_state=random_seed )<train_model>
epoch_num = 20
history = model.fit(x_train_scaled, np.array(y_train), validation_data=(x_val_scaled, np.array(y_val)) , batch_size=64, epochs=epoch_num)
%matplotlib inline
print(x_train.shape)
 test_show = plt.imshow(x_train[0][:,:,0] )<choose_model_class>
y_pred = model_DNN.predict(x_val_scaled)
 y_true = np.array(y_val)
 roc_auc_score(y_true, y_pred )
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu', input_shape=(28, 28, 1)))
 model.add(Conv2D(filters=32, kernel_size=(5,5), padding='Same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding='Same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(NUM_CLASSES, activation='softmax'))<choose_model_class>
iter_test = env.iter_test()
<train_model><EOS> 
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how="left")
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df['num_tags'].fillna(0, inplace = True)
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 test_df_train = test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean','num_tags',]]
test_df_values = test_df_train.values
min_max_scaler = preprocessing.MinMaxScaler()
test_df_values = min_max_scaler.fit_transform(test_df_values)
 test_df['answered_correctly'] = model_DNN.predict(test_df_values)
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )