Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<load_from_csv>
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm
import gc
from sklearn.model_selection import train_test_split
from lightgbm import LGBMClassifier
import optuna
from optuna.samplers import TPESampler
from sklearn.metrics import roc_auc_score
import riiideducation
import os
sample_submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 print(sample_submission.shape )<categorify>
%%time
used_data_types_dict = {
'question_id': 'int16',
'bundle_id': 'int16',
'correct_answer': 'int8',
'part': 'int8',
'tags': 'str',
}
questions = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv',
usecols = used_data_types_dict.keys() , dtype=used_data_types_dict)
 lectures_df = pd.read_csv('.. /input/riiid-test-answer-prediction/lectures.csv')
encoder = OneHotEncoder(sparse=False,categories='auto')
 yy = [[0],[1],[2],[3],[4],[5],[6],[7],[8],[9]]
encoder.fit(yy)
 train_label = train_label.reshape(-1,1)
 val_label = val_label.reshape(-1,1)
 train_label = encoder.transform(train_label)
 val_label = encoder.transform(val_label)
 print('train_label shape: %s'%str(train_label.shape))
print('val_label shape: %s'%str(val_label.shape))<import_modules>
%%time
features_df = pd.read_pickle('.. /input/riiid-splitting-train-and-test-data/features_q_only.pkl.zip')
 train_df = pd.read_pickle('.. /input/riiid-splitting-train-and-test-data/train_q_only.pkl.zip' )
import numpy as np
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.optimizers import SGD
from keras.layers.normalization import BatchNormalization
from keras.layers import LeakyReLU 
<choose_model_class>
def add_seen_before_to_train_df(features_df, train_df):
train_questions_only_df = features_df[features_df['answered_correctly']!=-1]
state = dict()
for user_id in train_questions_only_df['user_id'].unique() :
state[user_id] = {}
total = len(state.keys())
 user_content = train_questions_only_df.groupby('user_id')['content_id'].apply(np.array ).apply(np.sort ).apply(np.unique)
 user_attempts = train_questions_only_df.groupby(['user_id', 'content_id'])['content_id'].count().astype(np.uint8 ).groupby('user_id' ).apply(np.array ).values
for user_id, content, attempt in tqdm(zip(state.keys() , user_content, user_attempts),total=total):
state[user_id] = dict(zip(content, attempt))
del user_content, user_attempts, train_questions_only_df
big_list=[]
for pair in tqdm(train_df[['user_id','content_id']].values):
if pair[0] in state:
if pair[1] in state[pair[0]]:
big_list.append(state[pair[0]][pair[1]])
 state[pair[0]][pair[1]]+=1
else:
big_list.append(0)
 state[pair[0]][pair[1]]=1
else:
big_list.append(0)
 state[pair[0]]={pair[1]:1}
del state
train_df['seen_before']=big_list
train_df.seen_before = train_df.seen_before.clip(upper=5)
 del big_list
gc.collect()
return(train_df)
 train_df = add_seen_before_to_train_df(features_df, train_df )
model = Sequential()
model.add(Conv2D(32,(3, 3), activation='relu', input_shape=(28, 28, 1),padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(32,(3, 3), activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64,(3, 3), activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(64,(3, 3), activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(128, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(256, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(256, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation='relu', name='my_dense'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.summary()<set_options>
lects_df = features_df[features_df['answered_correctly']==-1]
lect_seen_df = pd.DataFrame(data=lects_df.user_id.value_counts())
 lect_seen_df.columns=['lectures_seen']
lect_seen_df.lectures_seen = lect_seen_df.lectures_seen.astype(float)
 lectures_df['type_of'] = lectures_df['type_of'].replace('solving question', 'solving_question')
 lectures_df = pd.get_dummies(lectures_df, columns=['part', 'type_of'])
 part_lectures_columns = [column for column in lectures_df.columns if column.startswith('part')]
types_of_lectures_columns = [column for column in lectures_df.columns if column.startswith('type_of_')]
lectures_df = lectures_df.set_index('lecture_id')
 train_lectures = features_df[features_df.answered_correctly == -1].merge(lectures_df, left_on='content_id', right_on='lecture_id', how='left')
 user_lecture_stats_part = train_lectures.groupby('user_id')[part_lectures_columns + types_of_lectures_columns].sum()
user_lecture_stats_part = user_lecture_stats_part.merge(lect_seen_df, left_index=True, right_index=True)
 user_lecture_stats_part.index.name = 'user_id'
user_lecture_stats_part['lectures_seen'] = user_lecture_stats_part['lectures_seen'].astype(int)
 del lects_df, lect_seen_df, train_lectures
user_lecture_stats_part
datagen = ImageDataGenerator(
 rotation_range=15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range = 15,
horizontal_flip = False,
zoom_range = 0.20 )<train_model>
train_questions_only_df = features_df[features_df['answered_correctly']!=-1]
train_questions_only_df = pd.merge(train_questions_only_df, questions[['part','tags']],
left_on='content_id', right_index=True, how = 'left')
 del features_df
model.compile(loss='categorical_crossentropy',optimizer=Adam() ,metrics=['accuracy'])
 datagen.fit(train_image)
 history = model.fit_generator(datagen.flow(train_image,train_label, batch_size=32),
epochs = 75,
shuffle=True,
validation_data =(val_image,val_label),
verbose = 1,
steps_per_epoch=train_image.shape[0] // 32 )<predict_on_test>
grouped_by_user_df = train_questions_only_df.groupby('user_id')
 user_answers_df = grouped_by_user_df.agg({'answered_correctly': ['mean', 'count', 'sum']} ).copy()
user_answers_df.columns = [
'user_mean_accuracy',
'user_questions_answered',
'user_questions_correct',
]
user_answers_df.user_questions_correct = user_answers_df.user_questions_correct.astype('int32')
 user_answers_df
intermediate_output = intermediate_layer_model.predict(train_image)
 intermediate_output = pd.DataFrame(data=intermediate_output )<split>
user_lagtime_max_dict = grouped_by_user_df.agg({'lag_time': ['max']} ).copy()
user_lagtime_max_dict.columns = [
'user_lag_time_max',
]
user_lagtime_max_dict = user_lagtime_max_dict.to_dict('index')
 for pair in grouped_by_user_df.tail(1)[['user_id','lag_time']].values:
user_lagtime_max_dict[pair[0]]['last_lagtime'] = pair[1]
gc.collect()
val_data = intermediate_output[40000:]<predict_on_test>
grouped_by_tags_df = train_questions_only_df.groupby('tags')
 tags_answers_df = grouped_by_tags_df.agg({'answered_correctly': ['mean', 'count', 'std', 'skew']} ).copy()
tags_answers_df.columns = [
'tags_mean_accuracy',
'tags_question_asked',
'tags_std_accuracy',
'tags_skew_accuracy'
]
tags_answers_df
submission_cnn = model.predict(test_image )<predict_on_test>
grouped_by_part_df = train_questions_only_df.groupby('part')
 part_answers_df = grouped_by_part_df.agg({'answered_correctly': ['mean', 'count']} ).copy()
part_answers_df.columns = [
'part_mean_accuracy',
'part_questions_answered',
]
part_answers_df
intermediate_test_output = intermediate_layer_model.predict(test_image)
 intermediate_test_output = pd.DataFrame(data=intermediate_test_output )<train_model>
del grouped_by_user_df
del grouped_by_tags_df
del grouped_by_part_df
xgbmodel = XGBClassifier(objective='multi:softprob',
num_class= 10)
 xgbmodel.fit(intermediate_output, train_label1)
 xgbmodel.score(val_data, val_label1 )<predict_on_test>
content_answers_df = pd.read_pickle('.. /input/riiid-content-answers-df-preprocessing/content_answers_df.pkl.zip')
 content_answers_df
submission_xgb = xgbmodel.predict(intermediate_test_output )<data_type_conversions>
features = [
'user_mean_accuracy',
'user_questions_answered',
'user_questions_correct',
'q_mean_accuracy',
'q_question_asked',
'q_question_correct',
'community',
'num_in_bundle',
'tags_mean_accuracy',
'tags_question_asked',
'tags_std_accuracy',
'tags_skew_accuracy',
'part_mean_accuracy',
'part_questions_answered',
'prior_question_elapsed_time',
'part_1',
'part_2',
'part_3',
'part_4',
'part_5',
'part_6',
'part_7',
'type_of_concept',
'type_of_intention',
'type_of_solving_question',
'lectures_seen',
'seen_before',
'avg_q_time',
'lag_time',
]
target = 'answered_correctly'
submission_cnn = submission_cnn.astype(int)
 submission_xgb = submission_xgb.astype(int)
<define_variables>
del train_questions_only_df
submission_cnn
label = np.argmax(submission_cnn,1)
 id_ = np.arange(0,label.shape[0])
 label<compute_test_metric>
def add_and_update_user_lects(user_lecture_stats_part, lectures_df, train_df):
lect_dict = lectures_df.to_dict('index')
 lect_stats_part_dict = user_lecture_stats_part.to_dict('index')
 part_1_list = []
part_2_list = []
part_3_list = []
part_4_list = []
part_5_list = []
part_6_list = []
part_7_list = []
type_of_concept_list = []
type_of_intention_list = []
type_of_solving_question_list = []
type_of_starter_list = []
lectures_seen_list = []
for pair in tqdm(train_df[['content_id','user_id','answered_correctly']].values):
if pair[1] in lect_stats_part_dict:
if pair[2]!=-1:
part_1_list.append(lect_stats_part_dict[pair[1]]['part_1'])
 part_2_list.append(lect_stats_part_dict[pair[1]]['part_2'])
 part_3_list.append(lect_stats_part_dict[pair[1]]['part_3'])
 part_4_list.append(lect_stats_part_dict[pair[1]]['part_4'])
 part_5_list.append(lect_stats_part_dict[pair[1]]['part_5'])
 part_6_list.append(lect_stats_part_dict[pair[1]]['part_6'])
 part_7_list.append(lect_stats_part_dict[pair[1]]['part_7'])
 type_of_concept_list.append(lect_stats_part_dict[pair[1]]['type_of_concept'])
 type_of_intention_list.append(lect_stats_part_dict[pair[1]]['type_of_intention'])
 type_of_solving_question_list.append(lect_stats_part_dict[pair[1]]['type_of_solving_question'])
 type_of_starter_list.append(lect_stats_part_dict[pair[1]]['type_of_starter'])
 lectures_seen_list.append(lect_stats_part_dict[pair[1]]['lectures_seen'])
 else:
part_1_list.append(lect_stats_part_dict[pair[1]]['part_1'])
 part_2_list.append(lect_stats_part_dict[pair[1]]['part_2'])
 part_3_list.append(lect_stats_part_dict[pair[1]]['part_3'])
 part_4_list.append(lect_stats_part_dict[pair[1]]['part_4'])
 part_5_list.append(lect_stats_part_dict[pair[1]]['part_5'])
 part_6_list.append(lect_stats_part_dict[pair[1]]['part_6'])
 part_7_list.append(lect_stats_part_dict[pair[1]]['part_7'])
 type_of_concept_list.append(lect_stats_part_dict[pair[1]]['type_of_concept'])
 type_of_intention_list.append(lect_stats_part_dict[pair[1]]['type_of_intention'])
 type_of_solving_question_list.append(lect_stats_part_dict[pair[1]]['type_of_solving_question'])
 type_of_starter_list.append(lect_stats_part_dict[pair[1]]['type_of_starter'])
 lectures_seen_list.append(lect_stats_part_dict[pair[1]]['lectures_seen'])
 lect_stats_part_dict[pair[1]]['part_1'] += lect_dict[pair[0]]['part_1']
lect_stats_part_dict[pair[1]]['part_2'] += lect_dict[pair[0]]['part_2']
lect_stats_part_dict[pair[1]]['part_3'] += lect_dict[pair[0]]['part_3']
lect_stats_part_dict[pair[1]]['part_4'] += lect_dict[pair[0]]['part_4']
lect_stats_part_dict[pair[1]]['part_5'] += lect_dict[pair[0]]['part_5']
lect_stats_part_dict[pair[1]]['part_6'] += lect_dict[pair[0]]['part_6']
lect_stats_part_dict[pair[1]]['part_7'] += lect_dict[pair[0]]['part_7']
lect_stats_part_dict[pair[1]]['type_of_concept'] += lect_dict[pair[0]]['type_of_concept']
lect_stats_part_dict[pair[1]]['type_of_intention'] += lect_dict[pair[0]]['type_of_intention']
lect_stats_part_dict[pair[1]]['type_of_solving_question'] += lect_dict[pair[0]]['type_of_solving_question']
lect_stats_part_dict[pair[1]]['type_of_starter'] += lect_dict[pair[0]]['type_of_starter']
lect_stats_part_dict[pair[1]]['lectures_seen'] += 1
else:
part_1_list.append(0)
 part_2_list.append(0)
 part_3_list.append(0)
 part_4_list.append(0)
 part_5_list.append(0)
 part_6_list.append(0)
 part_7_list.append(0)
 type_of_concept_list.append(0)
 type_of_intention_list.append(0)
 type_of_solving_question_list.append(0)
 type_of_starter_list.append(0)
 lectures_seen_list.append(0)
 if pair[2]==-1:
lect_stats_part_dict[pair[1]]={}
lect_stats_part_dict[pair[1]]['part_1'] = lect_dict[pair[0]]['part_1']
lect_stats_part_dict[pair[1]]['part_2'] = lect_dict[pair[0]]['part_2']
lect_stats_part_dict[pair[1]]['part_3'] = lect_dict[pair[0]]['part_3']
lect_stats_part_dict[pair[1]]['part_4'] = lect_dict[pair[0]]['part_4']
lect_stats_part_dict[pair[1]]['part_5'] = lect_dict[pair[0]]['part_5']
lect_stats_part_dict[pair[1]]['part_6'] = lect_dict[pair[0]]['part_6']
lect_stats_part_dict[pair[1]]['part_7'] = lect_dict[pair[0]]['part_7']
lect_stats_part_dict[pair[1]]['type_of_concept'] = lect_dict[pair[0]]['type_of_concept']
lect_stats_part_dict[pair[1]]['type_of_intention'] = lect_dict[pair[0]]['type_of_intention']
lect_stats_part_dict[pair[1]]['type_of_solving_question'] = lect_dict[pair[0]]['type_of_solving_question']
lect_stats_part_dict[pair[1]]['type_of_starter'] = lect_dict[pair[0]]['type_of_starter']
lect_stats_part_dict[pair[1]]['lectures_seen'] = 1
train_df['part_1'] = part_1_list
train_df['part_2'] = part_2_list
train_df['part_3'] = part_3_list
train_df['part_4'] = part_4_list
train_df['part_5'] = part_5_list
train_df['part_6'] = part_6_list
train_df['part_7'] = part_7_list
train_df['type_of_concept'] = type_of_concept_list
train_df['type_of_intention'] = type_of_intention_list
train_df['type_of_solving_question'] = type_of_solving_question_list
train_df['type_of_starter'] = type_of_starter_list
train_df['lectures_seen'] = lectures_seen_list
lect_stats_part = pd.DataFrame.from_dict(lect_stats_part_dict, orient='index')
 return(lect_stats_part, train_df )
final_sub = submission_xgb<save_to_csv>
def add_and_update_user_stats(user_answers_df, train_df):
my_dict=user_answers_df.to_dict('index')
 user_acc_list=[]
user_answered_list=[]
user_correct_list=[]
for pair in tqdm(train_df[['user_id','answered_correctly']].values):
if pair[0] in my_dict:
user_acc_list.append(my_dict[pair[0]]['user_mean_accuracy'])
 user_answered_list.append(my_dict[pair[0]]['user_questions_answered'])
 user_correct_list.append(my_dict[pair[0]]['user_questions_correct'])
 my_dict[pair[0]]['user_questions_answered']+=1
my_dict[pair[0]]['user_questions_correct']+=pair[1]
my_dict[pair[0]]['user_mean_accuracy'] = my_dict[pair[0]]['user_questions_correct']/my_dict[pair[0]]['user_questions_answered']
else:
my_dict[pair[0]]={'user_mean_accuracy':0.645,'user_questions_answered': 1, 'user_questions_correct': pair[1]}
user_acc_list.append(0.645)
 user_answered_list.append(0)
 user_correct_list.append(0)
 train_df['user_mean_accuracy']=user_acc_list
train_df['user_questions_answered']=user_answered_list
train_df['user_questions_correct']=user_correct_list
user_answers_df = pd.DataFrame.from_dict(my_dict, orient='index')
 return(user_answers_df, train_df )
save = pd.DataFrame({'ImageId':sample_submission.ImageId,'label':final_sub})
 print(save.head(10))
save.to_csv('submission.csv',index=False )<set_options>
def add_and_update_content_stats(content_answers_df, train_df):
my_dict=content_answers_df.to_dict('index')
 q_mean_accuracy_list=[]
q_question_asked=[]
q_question_correct=[]
community_list=[]
num_in_bundle_list=[]
avg_q_time_list=[]
for pair in tqdm(train_df[['content_id','answered_correctly']].values):
q_mean_accuracy_list.append(my_dict[pair[0]]['q_mean_accuracy'])
 q_question_asked.append(my_dict[pair[0]]['q_question_asked'])
 q_question_correct.append(my_dict[pair[0]]['q_question_correct'])
 community_list.append(my_dict[pair[0]]['community'])
 num_in_bundle_list.append(my_dict[pair[0]]['num_in_bundle'])
 avg_q_time_list.append(my_dict[pair[0]]['avg_q_time'])
 my_dict[pair[0]]['q_question_asked']+=1
my_dict[pair[0]]['q_question_correct']+=pair[1]
my_dict[pair[0]]['q_mean_accuracy'] = my_dict[pair[0]]['q_question_correct']/my_dict[pair[0]]['q_question_asked']
train_df['q_mean_accuracy'] = q_mean_accuracy_list
train_df['q_question_asked'] = q_question_asked
train_df['q_question_correct'] = q_question_correct
train_df['community'] = community_list
train_df['num_in_bundle'] = num_in_bundle_list
train_df['avg_q_time'] = avg_q_time_list
content_answers_df = pd.DataFrame.from_dict(my_dict, orient='index')
 return(content_answers_df, train_df )
%matplotlib inline 
<load_from_csv>
def update_user_max_timestamp(user_lagtime_max_dict, train_df):
for pair in train_df[['user_id','timestamp','lag_time']].values:
if pair[0] in user_lagtime_max_dict:
user_lagtime_max_dict[pair[0]]['user_lag_time_max'] = pair[1]
user_lagtime_max_dict[pair[0]]['last_lagtime'] = pair[2]
else:
user_lagtime_max_dict[pair[0]] = {}
user_lagtime_max_dict[pair[0]]['user_lag_time_max'] = pair[1]
user_lagtime_max_dict[pair[0]]['last_lagtime'] = pair[2]
return(user_lagtime_max_dict )
data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 print(data.shape )<load_from_csv>
%%time
lect_stats_part, train_df = add_and_update_user_lects(user_lecture_stats_part, lectures_df, train_df)
 train_df = train_df[train_df[target] != -1]
user_lagtime_max_dict = update_user_max_timestamp(user_lagtime_max_dict, train_df)
 user_answers_df, train_df = add_and_update_user_stats(user_answers_df, train_df)
 content_answers_df, train_df = add_and_update_content_stats(content_answers_df, train_df)
 train_df = pd.merge(train_df, questions[['part','tags']], left_on='content_id', right_index=True, how = 'left')
 train_df = train_df.merge(part_answers_df, how='left', left_on='part', right_index=True)
 train_df = train_df.merge(tags_answers_df, how='left', left_on='tags', right_index=True)
 train_df['part_1'].fillna(0, inplace = True)
 train_df['part_2'].fillna(0, inplace = True)
 train_df['part_3'].fillna(0, inplace = True)
 train_df['part_4'].fillna(0, inplace = True)
 train_df['part_5'].fillna(0, inplace = True)
 train_df['part_6'].fillna(0, inplace = True)
 train_df['part_7'].fillna(0, inplace = True)
 train_df['type_of_concept'].fillna(0, inplace = True)
 train_df['type_of_intention'].fillna(0, inplace = True)
 train_df['type_of_solving_question'].fillna(0, inplace = True)
 train_df['type_of_starter'].fillna(0, inplace = True)
 train_df['lectures_seen'].fillna(0, inplace = True)
 train_df['prior_question_elapsed_time'].fillna(25423, inplace = True)
 train_df['avg_q_time'].fillna(25423, inplace = True)
 train_df['lag_time'].fillna(26161758, inplace = True)
 train_df[types_of_lectures_columns + part_lectures_columns] = train_df[types_of_lectures_columns + part_lectures_columns].astype(int)
 train_df['lectures_seen'] = train_df['lectures_seen'].astype(int)
 train_df['prior_question_had_explanation'] = train_df['prior_question_had_explanation'].fillna(value=False ).astype(bool)
 train_df = train_df.fillna(value=0.5)
 train_df = train_df[features + [target]]
train_df = train_df.replace([np.inf, -np.inf], np.nan)
 train_df['prior_question_elapsed_time'].fillna(25423, inplace = True)
 train_df['avg_q_time'].fillna(25423, inplace = True)
 train_df['lag_time'].fillna(26161758, inplace = True)
 train_df = train_df.fillna(0 )
test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 print(test_data.shape )<load_from_csv>
sampler = TPESampler(seed=314)
 def create_model(trial):
num_leaves = trial.suggest_int("num_leaves", 20, 40)
 n_estimators = trial.suggest_int("n_estimators", 50, 400)
 max_depth = trial.suggest_int('max_depth', 3, 8)
 min_child_samples = trial.suggest_int('min_child_samples', 100, 1200)
 learning_rate = trial.suggest_uniform('learning_rate', 0.0001, 0.30)
 min_data_in_leaf = trial.suggest_int('min_data_in_leaf', 5, 90)
 bagging_fraction = trial.suggest_uniform('bagging_fraction', 0.50, 1.0)
 feature_fraction = trial.suggest_uniform('feature_fraction', 0.50, 1.0)
 model = LGBMClassifier(
 num_leaves=num_leaves,
n_estimators=n_estimators,
max_depth=max_depth,
min_child_samples=min_child_samples,
min_data_in_leaf=min_data_in_leaf,
learning_rate=learning_rate,
feature_fraction=feature_fraction,
random_state=314 
)
 return model
def objective(trial):
model = create_model(trial)
 model.fit(train_df[features], train_df[target])
 score = roc_auc_score(test_df[target].values, model.predict_proba(test_df[features])[:,1])
 return score
sample_submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 print(sample_submission.shape )<categorify>
params = {'num_leaves': 30,
'n_estimators': 300,
'max_depth': 5,
'min_child_samples': 371,
'learning_rate': 0.28285171125399805,
'min_data_in_leaf': 23,
'bagging_fraction': 0.8057106694835638,
'feature_fraction': 0.5688885590495344,
}
model = LGBMClassifier(**params)
 model.fit(train_df[features], train_df[target])
encoder = OneHotEncoder(sparse=False,categories='auto')
 yy = [[0],[1],[2],[3],[4],[5],[6],[7],[8],[9]]
encoder.fit(yy)
 train_label = train_label.reshape(-1,1)
 val_label = val_label.reshape(-1,1)
 train_label = encoder.transform(train_label)
 val_label = encoder.transform(val_label)
 print('train_label shape: %s'%str(train_label.shape))
print('val_label shape: %s'%str(val_label.shape))<import_modules>
print(model.feature_importances_)
 print(train_df.columns[:-1])
 pd.DataFrame({'col_name': model.feature_importances_},
index=train_df.columns[:-1] ).sort_values(by='col_name', ascending=False )
import numpy as np
import keras
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.optimizers import SGD
from keras.layers.normalization import BatchNormalization
from keras.layers import LeakyReLU 
<choose_model_class>
del train_df
all_data = pd.read_pickle('.. /input/riiid-train-df/train_df.pkl.gzip')
 all_data = all_data[all_data[target] != -1]
model = Sequential()
model.add(Conv2D(32,(3, 3), activation='relu', input_shape=(28, 28, 1),padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(32,(3, 3), activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(64,(3, 3), activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(64,(3, 3), activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(128, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(128, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Conv2D(256, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(Conv2D(256, kernel_size=5, activation='relu',padding='same'))
model.add(BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer="uniform"))
model.add(LeakyReLU(alpha=0.1))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation='relu', name='my_dense'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.summary()<set_options>
def get_me_all_seen_befores(all_data):
state = dict()
for user_id in all_data['user_id'].unique() :
state[user_id] = {}
total = len(state.keys())
 user_content = all_data.groupby('user_id')['content_id'].apply(np.array ).apply(np.sort ).apply(np.unique)
 user_attempts = all_data.groupby(['user_id', 'content_id'])['content_id'].count().astype(np.uint8 ).groupby('user_id' ).apply(np.array ).values
for user_id, content, attempt in tqdm(zip(state.keys() , user_content, user_attempts),total=total):
state[user_id] = dict(zip(content, attempt))
del user_content, user_attempts, all_data
return(state)
 state = get_me_all_seen_befores(all_data )
datagen = ImageDataGenerator(
 rotation_range=15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range = 15,
horizontal_flip = False,
zoom_range = 0.20 )<train_model>
def update_content_stats(content_answers_df, previous_test_df):
for row in previous_test_df[['content_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
content_answers_df.at[row[0],'q_question_correct'] += row[1]
content_answers_df.at[row[0],'q_question_asked'] += 1
content_answers_df['q_mean_accuracy']= content_answers_df['q_question_correct']/content_answers_df['q_question_asked']
model.compile(loss='categorical_crossentropy',optimizer=Adam() ,metrics=['accuracy'])
 datagen.fit(train_image)
 history = model.fit_generator(datagen.flow(train_image,train_label, batch_size=32),
epochs = 75,
shuffle=True,
validation_data =(val_image,val_label),
verbose = 1,
steps_per_epoch=train_image.shape[0] // 32 )<predict_on_test>
def update_user_stats(user_answers_df, previous_test_df):
for row in previous_test_df[['user_id','answered_correctly','content_type_id']].values:
if row[2] == 0:
try:
user_answers_df.at[row[0],'user_questions_correct'] += row[1]
user_answers_df.at[row[0],'user_questions_answered'] += 1
except:
user_answers_df.at[row[0]]=(0,1,row[1])
 user_answers_df['user_mean_accuracy']= user_answers_df['user_questions_correct']/user_answers_df['user_questions_answered']
intermediate_output = intermediate_layer_model.predict(train_image)
 intermediate_output = pd.DataFrame(data=intermediate_output )<split>
def update_lect_stats(user_lecture_stats_part, lectures_df, previous_test_df):
for row in previous_test_df[['user_id','content_type_id', 'content_id']].values:
if row[1] == 1:
y = lectures_df.query('lecture_id == {}'.format(row[2])).drop('tag', 1 ).reset_index(drop=True)
 y = y.loc[:,(y != 0 ).any(axis=0)]
if row[0] in user_lecture_stats_part.index:
for i in y.columns:
user_lecture_stats_part.at[row[0], i] +=1
user_lecture_stats_part.at[row[0], 'lectures_seen'] +=1
else:
user_lecture_stats_part.loc[row[0]] = 0
for i in y.columns:
user_lecture_stats_part.at[row[0], i] +=1
user_lecture_stats_part.at[row[0], 'lectures_seen'] +=1
val_data = intermediate_output[40000:]<predict_on_test>
def add_and_update_seen_before(train_df, state):
big_list=[]
for pair in train_df[['user_id','content_id','content_type_id']].values:
if pair[2] == 0:
if pair[0] in state:
if pair[1] in state[pair[0]]:
big_list.append(state[pair[0]][pair[1]])
 state[pair[0]][pair[1]]+=1
else:
big_list.append(0)
 state[pair[0]][pair[1]]=1
else:
big_list.append(0)
 state[pair[0]]={pair[1]:1}
else:
big_list.append(0)
 train_df['seen_before']= big_list
train_df.seen_before = train_df.seen_before.clip(upper=5)
 return(train_df, state )
submission_cnn = model.predict(test_image )<predict_on_test>
def add_and_update_lag_time(test_df, user_lagtime_max_dict):
lag_time_list = []
for pair in test_df[['user_id','timestamp']].values:
if pair[0] in user_lagtime_max_dict:
if pair[1] != user_lagtime_max_dict[pair[0]]['user_lag_time_max']:
lag_time_list.append(pair[1] -(user_lagtime_max_dict[pair[0]]['user_lag_time_max']))
user_lagtime_max_dict[pair[0]]['last_lagtime'] = pair[1] -(user_lagtime_max_dict[pair[0]]['user_lag_time_max'])
 user_lagtime_max_dict[pair[0]]['user_lag_time_max'] = pair[1]
else:
lag_time_list.append(user_lagtime_max_dict[pair[0]]['last_lagtime'])
 else:
lag_time_list.append(0)
 user_lagtime_max_dict[pair[0]] = {}
user_lagtime_max_dict[pair[0]]['user_lag_time_max'] = pair[1]
user_lagtime_max_dict[pair[0]]['last_lagtime'] = 0
test_df['lag_time'] = lag_time_list
return(test_df, user_lagtime_max_dict )
intermediate_test_output = intermediate_layer_model.predict(test_image)
 intermediate_test_output = pd.DataFrame(data=intermediate_test_output )<train_model>
env = riiideducation.make_env()
iter_test = env.iter_test()
previous_test_df = None
<predict_on_test><EOS> 
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[target] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_content_stats(content_answers_df, previous_test_df)
 update_user_stats(user_answers_df, previous_test_df)
 update_lect_stats(user_lecture_stats_part, lectures_df, previous_test_df)
 train_df, state = add_and_update_seen_before(test_df, state)
 test_df, user_lagtime_max_dict = add_and_update_lag_time(test_df, user_lagtime_max_dict)
 test_df = pd.merge(test_df, questions[['part','tags']], left_on='content_id', right_index=True, how = 'left')
 test_df = test_df.merge(user_answers_df, how='left', left_on='user_id',right_index=True)
 test_df = test_df.merge(content_answers_df, how='left', left_on='content_id', right_index=True)
 test_df = test_df.merge(part_answers_df, how='left', left_on='part', right_index=True)
 test_df = test_df.merge(tags_answers_df, how='left', left_on='tags', right_index=True)
 test_df = test_df.merge(user_lecture_stats_part, how='left', left_on='user_id', right_index=True)
 test_df['part_1'].fillna(0, inplace = True)
 test_df['part_2'].fillna(0, inplace = True)
 test_df['part_3'].fillna(0, inplace = True)
 test_df['part_4'].fillna(0, inplace = True)
 test_df['part_5'].fillna(0, inplace = True)
 test_df['part_6'].fillna(0, inplace = True)
 test_df['part_7'].fillna(0, inplace = True)
 test_df['type_of_concept'].fillna(0, inplace = True)
 test_df['type_of_intention'].fillna(0, inplace = True)
 test_df['type_of_solving_question'].fillna(0, inplace = True)
 test_df['type_of_starter'].fillna(0, inplace = True)
 test_df['lectures_seen'].fillna(0, inplace = True)
 test_df['prior_question_elapsed_time'].fillna(25423, inplace = True)
 test_df['avg_q_time'].fillna(0, inplace = True)
 test_df['lag_time'].fillna(26161758, inplace = True)
 test_df['lectures_seen'] = test_df['lectures_seen'].astype(int)
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value=False ).astype(bool)
 test_df.fillna(value = 0.6, inplace = True)
 test_df['answered_correctly'] = model.predict_proba(test_df[features])[:,1]
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
 previous_test_df = test_df.copy()