Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<predict_on_test>
import pandas as pd
import numpy as np
from sklearn.metrics import roc_auc_score
Jack=[3,1,7.89,0]
Rose=[1,0,60,0]
predictions=xgb.predict([Jack,Rose])
 print("Did Jack survived? {}".format(predictions[0]))
print("Did Rose survived? {}".format(predictions[1]))<save_to_csv>
train = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/train.csv',
usecols=[1, 2, 3, 4, 5, 7, 8, 9],
dtype={'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly':'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'boolean'}
)
ids = test['PassengerId']
X_test=test[features1]
predictions = xgb.predict(X_test)
 df = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions })
 df.to_csv('submission.csv', index=False )<import_modules>
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols=[0, 3],
dtype={'question_id': 'int16',
'part': 'int8'}
)
%matplotlib inline<load_from_csv>
lectures_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv' )
df_train=pd.read_csv('.. /input/titanic/train.csv')
 df_test=pd.read_csv('.. /input/titanic/test.csv' )<prepare_x_and_y>
lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
ntrain = df_train.shape[0]
ntest = df_test.shape[0]
y_train = df_train['Survived'].values
passId = df_test['PassengerId']
data = pd.concat(( df_train, df_test))
print("data size is: {}".format(data.shape))<count_missing_values>
train_lectures = train[train.content_type_id == True].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left' )
print("TRAIN DATA:")
 df_train.isnull().sum()<count_missing_values>
user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()
print("TEST DATA:")
 df_test.isnull().sum()<feature_engineering>
for column in user_lecture_stats_part.columns:
bool_column = column + '_boolean'
user_lecture_stats_part[bool_column] =(user_lecture_stats_part[column] > 0 ).astype(int )
data['Last_Name'] = data['Name'].apply(lambda x: str.split(x, ",")[0])
 data['Fare'].fillna(data['Fare'].mean() , inplace=True)
 default_survival_chance = 0.5
data['Family_Survival'] = default_survival_chance
for grp, grp_df in data[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin == 0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
data.loc[data['Family_Survival']!=0.5].shape[0] )<groupby>
del(train_lectures )
for _, grp_df in data.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data.loc[data['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(data[data['Family_Survival']!=0.5].shape[0]))<drop_column>
train = train[train.content_type_id == False].sort_values('timestamp' ).reset_index(drop = True )
data = data.reset_index(drop=True)
 data = data.drop('Survived', axis=1)
 data.tail()<categorify>
train[(train.content_type_id == False)].task_container_id.nunique()
data['Fare'] = pd.qcut(data['Fare'], 4)
 lbl = LabelEncoder()
data['Fare'] = lbl.fit_transform(data['Fare'] )<define_variables>
group1 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['count'])
 group1.columns = ['avg_questions']
group2 = train.loc[(train.content_type_id == False), ['task_container_id', 'user_id']].groupby(['task_container_id'] ).agg(['nunique'])
 group2.columns = ['avg_questions']
group3 = group1 / group2
titles_data = sorted(set([x for x in data['Name'].map(lambda x: get_title(x)) ]))
print(len(titles_data), ':', titles_data )<feature_engineering>
group3['avg_questions_seen'] = group3.avg_questions.cumsum()
data['Title'] = data['Name'].map(lambda x: get_title(x))
data['Title'] = data.apply(set_title, axis=1 )<count_values>
group3.iloc[0].avg_questions_seen
print(data['Title'].value_counts() )<compute_test_metric>
results_u_final = train.loc[train.content_type_id == False, ['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_final.columns = ['answered_correctly_user']
results_u2_final = train.loc[train.content_type_id == False, ['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_final.columns = ['explanation_mean_user']
print('Total missing age data: ', pd.isnull(data['Age'] ).sum())
<feature_engineering>
train = pd.merge(train, questions_df, left_on = 'content_id', right_on = 'question_id', how = 'left' )
data['Age'] = data.groupby('Title')['Age'].apply(lambda x: x.fillna(x.median()))<categorify>
results_q_final = train.loc[train.content_type_id == False, ['question_id','answered_correctly']].groupby(['question_id'] ).agg(['mean'])
 results_q_final.columns = ['quest_pct']
data['Age'] = pd.qcut(data['Age'], 4)
 lbl = LabelEncoder()
data['Age'] = lbl.fit_transform(data['Age'] )<categorify>
results_q2_final = train.loc[train.content_type_id == False, ['question_id','part']].groupby(['question_id'] ).agg(['count'])
 results_q2_final.columns = ['count']
data['Title'] = data['Title'].replace(['Mr', 'Miss', 'Mrs', 'Master'], [0, 1, 2, 3] )<categorify>
question2 = pd.merge(questions_df, results_q_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )
data['Sex'] = data['Sex'].replace(['male', 'female'], [0, 1] )<categorify>
question2 = pd.merge(question2, results_q2_final, left_on = 'question_id', right_on = 'question_id', how = 'left' )
data['Embarked'] = data['Embarked'].fillna(data['Embarked'].mode() [0])
 data['Embarked'] = data['Embarked'].replace(['S', 'C', 'Q'], [0, 1, 2] )<feature_engineering>
question2.quest_pct = round(question2.quest_pct,5 )
data['Cabin'] = data['Cabin'].map(lambda x: x[0] )<count_values>
train.drop(['timestamp', 'content_type_id', 'question_id', 'part'], axis=1, inplace=True )
data['Cabin'].value_counts()<feature_engineering>
validation = train.groupby('user_id' ).tail(5)
 train = train[~train.index.isin(validation.index)]
len(train)+ len(validation )
def unknown_cabin(cabin):
if cabin != 'U':
return 1
else:
return 0
data['Cabin'] = data['Cabin'].apply(lambda x:unknown_cabin(x))<feature_engineering>
results_u_val = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_val.columns = ['answered_correctly_user']
results_u2_val = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_val.columns = ['explanation_mean_user']
data['FamilySize'] = data['SibSp'] + data['Parch'] + 1
data['IsAlone'] = 1
data['IsAlone'].loc[data['FamilySize'] > 1] = 0 
<drop_column>
X = train.groupby('user_id' ).tail(18)
 train = train[~train.index.isin(X.index)]
len(X)+ len(train)+ len(validation )
data = data.drop(['Name', 'Parch', 'SibSp', 'Ticket', 'Last_Name', 'PassengerId'], axis = 1 )<split>
results_u_X = train[['user_id','answered_correctly']].groupby(['user_id'] ).agg(['mean'])
 results_u_X.columns = ['answered_correctly_user']
results_u2_X = train[['user_id','prior_question_had_explanation']].groupby(['user_id'] ).agg(['mean'])
 results_u2_X.columns = ['explanation_mean_user']
train = data[:ntrain]
test = data[ntrain:]<normalization>
del(train )
X_test = test
X_train = train
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )<train_on_grid>
X = pd.merge(X, group3, left_on=['task_container_id'], right_index= True, how="left")
 X = pd.merge(X, results_u_X, on=['user_id'], how="left")
 X = pd.merge(X, results_u2_X, on=['user_id'], how="left")
 X = pd.merge(X, user_lecture_stats_part, on=['user_id'], how="left" )
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier()
gbc = GradientBoostingClassifier()
svc = SVC(probability=True)
 ext = ExtraTreesClassifier()
ada = AdaBoostClassifier()
gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier()
models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores.append(acc.mean() )<prepare_output>
validation = pd.merge(validation, group3, left_on=['task_container_id'], right_index= True, how="left")
 validation = pd.merge(validation, results_u_val, on=['user_id'], how="left")
 validation = pd.merge(validation, results_u2_val, on=['user_id'], how="left")
 validation = pd.merge(validation, user_lecture_stats_part, on=['user_id'], how="left" )
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Score': scores})
 result_df = results.sort_values(by='Score', ascending=False ).reset_index(drop=True)
 result_df.head(11 )<create_dataframe>
lb_make = LabelEncoder()
X.prior_question_had_explanation.fillna(False, inplace = True)
 validation.prior_question_had_explanation.fillna(False, inplace = True)
 validation["prior_question_had_explanation_enc"] = lb_make.fit_transform(validation["prior_question_had_explanation"])
 X["prior_question_had_explanation_enc"] = lb_make.fit_transform(X["prior_question_had_explanation"] )
fi = {'Features':train.columns.tolist() , 'Importance':xgb.feature_importances_}
importance = pd.DataFrame(fi, index=None ).sort_values('Importance', ascending=False )<create_dataframe>
fi = {'Features':train.columns.tolist() , 'Importance':np.transpose(log.coef_[0])}
importance = pd.DataFrame(fi, index=None ).sort_values('Importance', ascending=False )<create_dataframe>
question2.quest_pct = question2.quest_pct.mask(( question2['count'] < 3),.65)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct <.2)&(question2['count'] < 21),.2)
 question2.quest_pct = question2.quest_pct.mask(( question2.quest_pct >.95)&(question2['count'] < 21),.95 )
gbc_imp = pd.DataFrame({'Feature':train.columns, 'gbc importance':gbc.feature_importances_})
 xgb_imp = pd.DataFrame({'Feature':train.columns, 'xgb importance':xgb.feature_importances_})
 ran_imp = pd.DataFrame({'Feature':train.columns, 'ran importance':ran.feature_importances_})
 ext_imp = pd.DataFrame({'Feature':train.columns, 'ext importance':ext.feature_importances_})
 ada_imp = pd.DataFrame({'Feature':train.columns, 'ada importance':ada.feature_importances_})
 importances = gbc_imp.merge(xgb_imp, on='Feature' ).merge(ran_imp, on='Feature' ).merge(ext_imp, on='Feature' ).merge(ada_imp, on='Feature')
 importances['Average'] = importances.mean(axis=1)
 importances = importances.sort_values(by='Average', ascending=False ).reset_index(drop=True)
 importances<create_dataframe>
X = pd.merge(X, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 validation = pd.merge(validation, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 X.part = X.part - 1
validation.part = validation.part - 1
fi = {'Features':importances['Feature'], 'Importance':importances['Average']}
importance = pd.DataFrame(fi, index=None ).sort_values('Importance', ascending=False )<normalization>
y = X['answered_correctly']
X = X.drop(['answered_correctly'], axis=1)
 X.head()
y_val = validation['answered_correctly']
X_val = validation.drop(['answered_correctly'], axis=1 )
train = train.drop(['Embarked','IsAlone'], axis=1)
 test = test.drop(['Embarked', 'IsAlone'], axis=1)
 X_train = train
X_test = test
X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )<train_on_grid>
X = X[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]
X_val = X_val[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]
ran = RandomForestClassifier(random_state=1)
 knn = KNeighborsClassifier()
log = LogisticRegression()
xgb = XGBClassifier(random_state=1)
 gbc = GradientBoostingClassifier(random_state=1)
 svc = SVC(probability=True)
 ext = ExtraTreesClassifier(random_state=1)
 ada = AdaBoostClassifier(random_state=1)
 gnb = GaussianNB()
gpc = GaussianProcessClassifier()
bag = BaggingClassifier(random_state=1)
 models = [ran, knn, log, xgb, gbc, svc, ext, ada, gnb, gpc, bag]
scores_v2 = []
for mod in models:
mod.fit(X_train, y_train)
 acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 10)
 scores_v2.append(acc.mean() )<prepare_output>
X['answered_correctly_user'].fillna(0.65, inplace=True)
 X['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X['quest_pct'].fillna(content_mean, inplace=True)
 X['part'].fillna(4, inplace = True)
 X['avg_questions_seen'].fillna(1, inplace = True)
 X['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X['part_1'].fillna(0, inplace = True)
 X['part_2'].fillna(0, inplace = True)
 X['part_3'].fillna(0, inplace = True)
 X['part_4'].fillna(0, inplace = True)
 X['part_5'].fillna(0, inplace = True)
 X['part_6'].fillna(0, inplace = True)
 X['part_7'].fillna(0, inplace = True)
 X['type_of_concept'].fillna(0, inplace = True)
 X['type_of_intention'].fillna(0, inplace = True)
 X['type_of_solving_question'].fillna(0, inplace = True)
 X['type_of_starter'].fillna(0, inplace = True)
 X['part_1_boolean'].fillna(0, inplace = True)
 X['part_2_boolean'].fillna(0, inplace = True)
 X['part_3_boolean'].fillna(0, inplace = True)
 X['part_4_boolean'].fillna(0, inplace = True)
 X['part_5_boolean'].fillna(0, inplace = True)
 X['part_6_boolean'].fillna(0, inplace = True)
 X['part_7_boolean'].fillna(0, inplace = True)
 X['type_of_concept_boolean'].fillna(0, inplace = True)
 X['type_of_intention_boolean'].fillna(0, inplace = True)
 X['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X['type_of_starter_boolean'].fillna(0, inplace = True )
results = pd.DataFrame({
'Model': ['Random Forest', 'K Nearest Neighbour', 'Logistic Regression', 'XGBoost', 'Gradient Boosting', 'SVC', 'Extra Trees', 'AdaBoost', 'Gaussian Naive Bayes', 'Gaussian Process', 'Bagging Classifier'],
'Original Score': scores,
'Score with feature selection': scores_v2})
 result_df = results.sort_values(by='Score with feature selection', ascending=False ).reset_index(drop=True)
 result_df.head(11 )<train_on_grid>
X_val['answered_correctly_user'].fillna(0.65, inplace=True)
 X_val['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 X_val['quest_pct'].fillna(content_mean, inplace=True)
 X_val['part'].fillna(4, inplace = True)
 X_val['avg_questions_seen'].fillna(1, inplace = True)
 X_val['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 X_val['prior_question_had_explanation_enc'].fillna(0, inplace = True)
 X_val['part_1'].fillna(0, inplace = True)
 X_val['part_2'].fillna(0, inplace = True)
 X_val['part_3'].fillna(0, inplace = True)
 X_val['part_4'].fillna(0, inplace = True)
 X_val['part_5'].fillna(0, inplace = True)
 X_val['part_6'].fillna(0, inplace = True)
 X_val['part_7'].fillna(0, inplace = True)
 X_val['type_of_concept'].fillna(0, inplace = True)
 X_val['type_of_intention'].fillna(0, inplace = True)
 X_val['type_of_solving_question'].fillna(0, inplace = True)
 X_val['type_of_starter'].fillna(0, inplace = True)
 X_val['part_1_boolean'].fillna(0, inplace = True)
 X_val['part_2_boolean'].fillna(0, inplace = True)
 X_val['part_3_boolean'].fillna(0, inplace = True)
 X_val['part_4_boolean'].fillna(0, inplace = True)
 X_val['part_5_boolean'].fillna(0, inplace = True)
 X_val['part_6_boolean'].fillna(0, inplace = True)
 X_val['part_7_boolean'].fillna(0, inplace = True)
 X_val['type_of_concept_boolean'].fillna(0, inplace = True)
 X_val['type_of_intention_boolean'].fillna(0, inplace = True)
 X_val['type_of_solving_question_boolean'].fillna(0, inplace = True)
 X_val['type_of_starter_boolean'].fillna(0, inplace = True )
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd=GridSearchCV(estimator = RandomForestClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
scaler = MinMaxScaler(feature_range=(0, 1))
X = scaler.fit_transform(X)
 X_val = scaler.transform(X_val)
 X_train = X.reshape(X.shape[0], X.shape[1], 1)
 X_test = X_val.reshape(X_val.shape[0], X_val.shape[1], 1 )
n_neighbors = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16, 18, 20]
algorithm = ['auto']
weights = ['uniform', 'distance']
leaf_size = [1, 2, 3, 4, 5, 10, 15, 20, 25, 30]
hyperparams = {'algorithm': algorithm, 'weights': weights, 'leaf_size': leaf_size,
'n_neighbors': n_neighbors}
gd=GridSearchCV(estimator = KNeighborsClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.layers import BatchNormalization,Dropout,Dense,Flatten,Conv1D
from tensorflow.keras.optimizers import Adam
from keras.metrics import BinaryAccuracy
from keras import backend as K
from tensorflow import keras
from tensorflow.keras import layers
penalty = ['l1', 'l2']
C = np.logspace(0, 4, 10)
 hyperparams = {'penalty': penalty, 'C': C}
gd=GridSearchCV(estimator = LogisticRegression() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
class MultiHeadSelfAttention(layers.Layer):
def __init__(self, embed_dim, num_heads=8):
super(MultiHeadSelfAttention, self ).__init__()
self.embed_dim = embed_dim
self.num_heads = num_heads
if embed_dim % num_heads != 0:
raise ValueError(
 f"embedding dimension = {embed_dim} should be divisible by number of heads = {num_heads}" 
)
 self.projection_dim = embed_dim // num_heads
self.query_dense = layers.Dense(embed_dim)
 self.key_dense = layers.Dense(embed_dim)
 self.value_dense = layers.Dense(embed_dim)
 self.combine_heads = layers.Dense(embed_dim)
 def attention(self, query, key, value):
score = tf.matmul(query, key, transpose_b=True)
 dim_key = tf.cast(tf.shape(key)[-1], tf.float32)
 scaled_score = score / tf.math.sqrt(dim_key)
 weights = tf.nn.softmax(scaled_score, axis=-1)
 output = tf.matmul(weights, value)
 return output, weights
def separate_heads(self, x, batch_size):
x = tf.reshape(x,(batch_size, -1, self.num_heads, self.projection_dim))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, inputs):
batch_size = tf.shape(inputs)[0]
query = self.query_dense(inputs)
 key = self.key_dense(inputs)
 value = self.value_dense(inputs)
 query = self.separate_heads(
 query, batch_size 
)
 key = self.separate_heads(
 key, batch_size 
)
 value = self.separate_heads(
 value, batch_size 
)
 attention, weights = self.attention(query, key, value)
 attention = tf.transpose(
 attention, perm=[0, 2, 1, 3] 
)
 concat_attention = tf.reshape(
 attention,(batch_size, -1, self.embed_dim)
)
 output = self.combine_heads(
 concat_attention 
)
 return output
learning_rate = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
n_estimators = [10, 25, 50, 75, 100, 250, 500, 750, 1000]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gd=GridSearchCV(estimator = XGBClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
class TransformerBlock(layers.Layer):
def __init__(self, embed_dim, num_heads, ff_dim, rate=0.2):
super(TransformerBlock, self ).__init__()
self.att = MultiHeadSelfAttention(embed_dim, num_heads)
 self.ffn = keras.Sequential(
 [layers.Dense(ff_dim, activation="relu"), layers.Dense(embed_dim),] 
)
 self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = layers.Dropout(rate)
 self.dropout2 = layers.Dropout(rate)
 def call(self, inputs, training):
attn_output = self.att(inputs)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(inputs + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 return self.layernorm2(out1 + ffn_output )
max_depth = [3, 4, 5, 6, 7, 8, 9, 10]
min_child_weight = [1, 2, 3, 4, 5, 6]
hyperparams = {'max_depth': max_depth, 'min_child_weight': min_child_weight}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
class TokenAndPositionEmbedding(layers.Layer):
def __init__(self, maxlen, vocab_size, embed_dim):
super(TokenAndPositionEmbedding, self ).__init__()
self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)
 self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)
 def call(self, x):
maxlen = tf.shape(x)[-1]
positions = tf.range(start=0, limit=maxlen, delta=1)
 positions = self.pos_emb(positions)
 x = self.token_emb(x)
 return x + positions
gamma = [i*0.1 for i in range(0,5)]
hyperparams = {'gamma': gamma}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10, max_depth=3,
min_child_weight=1), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
maxlen = 32
X_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=maxlen)
 X_test = keras.preprocessing.sequence.pad_sequences(X_test, maxlen=maxlen )
subsample = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
colsample_bytree = [0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1]
hyperparams = {'subsample': subsample, 'colsample_bytree': colsample_bytree}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10, max_depth=3,
min_child_weight=1, gamma=0), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
embed_dim = 32
num_heads = 8
ff_dim = 64
vocab_size = 50000
inputs = layers.Input(shape=(maxlen,))
embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)
 x = embedding_layer(inputs)
 transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)
 x = transformer_block(x)
 x = layers.GlobalAveragePooling1D()(x)
 x = layers.Dropout(0.2 )(x)
 x = layers.Dense(20, activation="relu" )(x)
 x = layers.Dropout(0.2 )(x)
 outputs = layers.Dense(1, activation="sigmoid" )(x)
 model = keras.Model(inputs=inputs, outputs=outputs )
reg_alpha = [1e-5, 1e-2, 0.1, 1, 100]
hyperparams = {'reg_alpha': reg_alpha}
gd=GridSearchCV(estimator = XGBClassifier(learning_rate=0.0001, n_estimators=10, max_depth=3,
min_child_weight=1, gamma=0, subsample=0.6, colsample_bytree=0.9),
param_grid = hyperparams, verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
es = EarlyStopping(monitor='val_auc', mode='max',patience=10 )
learning_rate = [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]
n_estimators = [100, 250, 500, 750, 1000, 1250, 1500]
hyperparams = {'learning_rate': learning_rate, 'n_estimators': n_estimators}
gd=GridSearchCV(estimator = GradientBoostingClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
model.compile("adam", "binary_crossentropy", metrics=[tf.keras.metrics.AUC() ])
 history = model.fit(
 X_train, y, epochs=25,verbose=1,validation_split=0.1,batch_size=65536,callbacks=[es]
)
Cs = [0.001, 0.01, 0.1, 1, 5, 10, 15, 20, 50, 100]
gammas = [0.001, 0.01, 0.1, 1]
hyperparams = {'C': Cs, 'gamma' : gammas}
gd=GridSearchCV(estimator = SVC(probability=True), param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
y_pred = model.predict(X_test)
 y_true = np.array(y_val)
 roc_auc_score(y_true, y_pred )
n_estimators = [10, 25, 50, 75, 100]
max_depth = [3, None]
max_features = [1, 3, 5, 7]
min_samples_split = [2, 4, 6, 8, 10]
min_samples_leaf = [2, 4, 6, 8, 10]
hyperparams = {'n_estimators': n_estimators, 'max_depth': max_depth, 'max_features': max_features,
'min_samples_split': min_samples_split, 'min_samples_leaf': min_samples_leaf}
gd=GridSearchCV(estimator = ExtraTreesClassifier() , param_grid = hyperparams,
verbose=True, cv=5, scoring = "accuracy")
 gd.fit(X_train, y_train)
 print(gd.best_score_)
 print(gd.best_estimator_ )<train_on_grid>
env = riiideducation.make_env()
iter_test = env.iter_test()
<train_on_grid><EOS> 
for(test_df, sample_prediction_df)in iter_test:
test_df['task_container_id'] = test_df.task_container_id.mask(test_df.task_container_id > 9999, 9999)
 test_df = pd.merge(test_df, group3, left_on=['task_container_id'], right_index= True, how="left")
 test_df = pd.merge(test_df, question2, left_on = 'content_id', right_on = 'question_id', how = 'left')
 test_df = pd.merge(test_df, results_u_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, results_u2_final, on=['user_id'], how="left")
 test_df = pd.merge(test_df, user_lecture_stats_part, on=['user_id'], how="left")
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['part_1_boolean'].fillna(0, inplace = True)
 test_df['part_2_boolean'].fillna(0, inplace = True)
 test_df['part_3_boolean'].fillna(0, inplace = True)
 test_df['part_4_boolean'].fillna(0, inplace = True)
 test_df['part_5_boolean'].fillna(0, inplace = True)
 test_df['part_6_boolean'].fillna(0, inplace = True)
 test_df['part_7_boolean'].fillna(0, inplace = True)
 test_df['type_of_concept_boolean'].fillna(0, inplace = True)
 test_df['type_of_intention_boolean'].fillna(0, inplace = True)
 test_df['type_of_solving_question_boolean'].fillna(0, inplace = True)
 test_df['type_of_starter_boolean'].fillna(0, inplace = True)
 test_df['answered_correctly_user'].fillna(0.65, inplace=True)
 test_df['explanation_mean_user'].fillna(prior_mean_user, inplace=True)
 test_df['quest_pct'].fillna(content_mean, inplace=True)
 test_df['part'] = test_df.part - 1
test_df['part'].fillna(4, inplace = True)
 test_df['avg_questions_seen'].fillna(1, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(elapsed_mean, inplace = True)
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df["prior_question_had_explanation_enc"] = lb_make.fit_transform(test_df["prior_question_had_explanation"])
 X = test_df[['answered_correctly_user', 'explanation_mean_user', 'quest_pct', 'avg_questions_seen',
'prior_question_elapsed_time','prior_question_had_explanation_enc', 'part',
'part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7',
'type_of_concept', 'type_of_intention', 'type_of_solving_question', 'type_of_starter',
'part_1_boolean', 'part_2_boolean', 'part_3_boolean', 'part_4_boolean', 'part_5_boolean', 'part_6_boolean', 'part_7_boolean',
'type_of_concept_boolean', 'type_of_intention_boolean', 'type_of_solving_question_boolean', 'type_of_starter_boolean']]
X=scaler.transform(X)
 X = X.reshape(X.shape[0], X.shape[1], 1)
 X = keras.preprocessing.sequence.pad_sequences(X, maxlen=maxlen)
 test_df['answered_correctly'] = model.predict(X)
 env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )