Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<merge>
np.set_printoptions(suppress=True)
%%time
train = cut_and_merge_with_class(train, df_ans_class, "ans_rate", "ans_rate_class")
 train = cut_and_merge_with_class(train, df_ans_class, "study_rate", "study_rate_class")
 for p in range(1,8):
train = cut_and_merge_with_class(train, df_ans_class, f"part{p}_rate", f"part{p}_rate_class")
 train.drop("base_class", axis=1, inplace=True)
 train.shape<merge>
df1 = pd.read_csv('.. /input/train.csv')
 df2 = pd.read_csv('.. /input/test.csv' )
train = train.merge(df_qs.drop(["part","tag1"], axis=1), on="question_id")
 train = train.merge(df_question_rate, on=["question_id","bundle_id","prior_question_had_explanation"])
 train = train.merge(df_etime_rate, on="question_id")
 train = train.merge(df_part_rate, on="part")
 gc.collect()<normalization>
train_data = np.array(df1)
 test_data = np.array(df2)
 X_train_orig = np.asarray(train_data[:, 1::], dtype=np.float32)
 Y_train_orig = np.asarray(train_data[:, 0], dtype=np.float32)
 X_test_orig = np.asarray(test_data[:, 0::], dtype=np.float32 )
t_cols = ["ans_rate_class","study_rate_class"] + ["part"+str(p)+"_rate_class" for p in range(1,8)]
n_cols = ["num_que", "num_que"] + num_part_cols
for t, n in zip(t_cols, n_cols):
train = change_rate_with_num(train, t, n, method="respectively")
 train = change_rate_with_num(train, t, n, method="bundle" )<feature_engineering>
X_train = X_train_orig / 255.
 X_test = X_test_orig / 255 .
train["prob_mistake_que"] = train.num_que.apply(lambda x: 0.6*x**(-0.2)if x!=0 else 0.6)
 part_mistake_cols = []
for p in range(1,8):
part_mistake_cols.append("prob_mistake_part"+str(p))
train["prob_mistake_part"+str(p)] = train["num_part"+str(p)].apply(lambda x: 0.6*x**(-0.2)if x!=0 else 0.6)
 all_prob_mistake = np.array(train[part_mistake_cols])
 train["prob_mistake_part_curt"] = np.nanmax(np.identity(8)[train.part][:,1:]*all_prob_mistake, axis=1)
 all_part_rate = np.array(train[["part1_rate","part2_rate","part3_rate","part4_rate","part5_rate","part6_rate","part7_rate"]])
 all_part_ratec = np.array(train[["part1_rate_class","part2_rate_class","part3_rate_class","part4_rate_class","part5_rate_class","part6_rate_class","part7_rate_class"]])
 all_abi_rate = np.array(train[["abi_rate0","abi_rate3","abi_rate4","abi_rate5","abi_rate6","abi_rate7","abi_rate8","abi_rate9"]])
 train["part_rate_curt"] = np.nanmax(np.identity(8)[train.part][:,1:]*all_part_rate, axis=1)
 train["part_ratec_curt"] = np.nanmax(np.identity(8)[train.part][:,1:]*all_part_ratec, axis=1)
 train["abi_rate_curt"] = np.nanmax(np.identity(10)[train.question_rate_class_exp][:,[0,3,4,5,6,7,8,9]]*all_abi_rate, axis=1)
 train["part_ratec_mean"] = np.nan_to_num(all_part_ratec ).sum(1)/ 7
del all_part_rate, all_part_ratec, all_abi_rate
gc.collect()
train["harmonic_ar_prc"] = 2 *(train.ans_rate * train.part_rate_curt)/(train.ans_rate + train.part_rate_curt)
 train["harmonic_arc_qr"] = 2 *(train.ans_rate_class * train.question_rate_exp)/(train.ans_rate_class + train.question_rate_exp)
 train["harmonic_arc_prcc"] = 2 *(train.ans_rate_class * train.part_ratec_curt)/(train.ans_rate_class + train.part_ratec_curt)
 train["harmonic_prc_pr"] = 2 *(train.part_rate_curt * train.part_rate)/(train.part_rate_curt + train.part_rate)
 train["harmonic_prcc_qr"] = 2 *(train.part_ratec_curt * train.question_rate_exp)/(train.part_ratec_curt + train.question_rate_exp)
 train["harmonic_qr_pr"] = 2 *(train.part_rate * train.question_rate_exp)/(train.part_rate + train.question_rate_exp)
 train["harmonic_abrc_qr"] = 2 *(train.abi_rate_curt * train.question_rate_exp)/(train.abi_rate_curt + train.question_rate_exp)
 train["harmonic_arc_src"] = 2 *(train.ans_rate_class * train.study_rate_class)/(train.ans_rate_class + train.study_rate_class)
 train["harmonic_prcc_src"] = 2 *(train.part_ratec_curt * train.study_rate_class)/(train.part_ratec_curt + train.study_rate_class)
 train["harmonic_src_qr"] = 2 *(train.study_rate_class * train.question_rate_exp)/(train.study_rate_class + train.question_rate_exp)
 train["rate_arc_prcc"] = train.ans_rate_class / train.part_ratec_curt
train["rate_arc_src"] = train.ans_rate_class / train.study_rate_class
train["rate_prcc_src"] = train.part_ratec_curt / train.study_rate_class
train["rate_arc_qr"] = train.ans_rate_class / train.question_rate_exp
train["rate_qr_pr"] = train.part_rate / train.question_rate_exp
train["rate_prcc_prccm"] = train.part_ratec_curt / train.part_ratec_mean<sort_values>
X_train = X_train.reshape(-1,28,28,1)
 X_test = X_test.reshape(-1,28,28,1)
 Y_train = Y_train_orig.reshape(-1, 1)
 Y_train = to_categorical(Y_train, num_classes=10 )
train.fillna(0, inplace=True)
 train.sort_values(by="row_id", inplace=True)
 train.reset_index(drop=True, inplace=True )<create_dataframe>
X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.2 )
class BaseModel(object):

 def __init__(self, train_df, test_df, target, features,
valid_df=None, tune_df=None, categoricals=[], alpha=0,
n_splits=3, cv_method="KFold", group=None,
task="regression", params=None, parameter_tuning=False,
tuning_type="optuna_tuner", seed=42, verbose=True):
self.train_df = train_df
self.test_df = test_df
self.valid_df = valid_df
self.tune_df = tune_df
self.target = target
self.features = features
self.n_splits = n_splits
self.categoricals = categoricals
self.alpha = alpha
self.cv_method = cv_method
self.group = group
self.task = task
self.parameter_tuning = parameter_tuning
self.seed = seed
self.cv = self.get_cv()
self.verbose = verbose
self.tuning_type = tuning_type
if params is None:
self.params = self.get_params()
else:
self.params = params
self.y_pred, self.y_valid, self.score, self.models, self.oof, self.y_val, self.fi_df = self.fit()
def train_model(self, train_set, val_set):
raise NotImplementedError
def get_params(self):
raise NotImplementedError
def convert_dataset(self, x_train, y_train, x_val, y_val):
raise NotImplementedError
def calc_metric(self, y_true, y_pred):
if self.task in("multiclass","nn_multiclass"):
preds = np.argmax(y_pred, axis=1)if y_true.shape != y_pred.shape else y_pred
return f1_score(y_true, preds, average='macro')
 elif self.task == "binary":
return roc_auc_score(y_true, y_pred, average='macro')
 elif self.task in("regression","quantile"):
return np.sqrt(mean_squared_error(y_true, y_pred))
def get_cv(self):
if self.cv_method == "KFold":
cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)
 return cv.split(self.train_df)
 if self.cv_method == "StratifiedKFold":
cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)
 return cv.split(self.train_df, self.train_df[self.target])
 if self.cv_method == "TimeSeriesSplit":
cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)
 return cv.split(self.train_df)
 if self.cv_method == "GroupKFold":
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 cv = GroupKFold(n_splits=self.n_splits)
 return cv.split(self.train_df[self.features+self.categoricals], self.train_df[self.target], self.train_df[self.group])
 if self.cv_method == "GroupShuffleSplit":
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 cv = GroupShuffleSplit(n_splits=self.n_splits, random_state=self.seed)
 return cv.split(self.train_df[self.features+self.categoricals], self.train_df[self.target], self.train_df[self.group])
 def fit(self):
y_vals = np.zeros(( self.train_df.shape[0],))
if self.task in("multiclass","nn_multiclass"):
col_len = self.train_df[self.target].nunique()
else:
col_len = 1
oof_pred = np.zeros(( self.train_df.shape[0], col_len))
y_pred = np.zeros(( self.test_df.shape[0], col_len))
y_valid = np.zeros(( self.valid_df.shape[0], col_len)) if self.valid_df is not None else None
models = []
if self.group is not None:
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 fi = np.zeros(( self.n_splits, len(self.features+self.categoricals)))
 if y_valid is not None:
x_valid = self.valid_df[self.features+self.categoricals].copy()
del self.valid_df
gc.collect()
x_test = self.test_df[self.features+self.categoricals]
for fold,(train_idx, val_idx)in enumerate(self.cv):
x_train = self.train_df.iloc[train_idx, :][self.features+self.categoricals]
y_train = self.train_df.iloc[train_idx, :][self.target]
x_val = self.train_df.iloc[val_idx, :][self.features+self.categoricals]
y_val = self.train_df.iloc[val_idx, :][self.target]
train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)
 del x_train, y_train
gc.collect()
model, importance = self.train_model(train_set, val_set)
 fi[fold, :] = importance
y_vals[val_idx] = y_val
oof_pred[val_idx] = model.predict(x_val, num_iteration=model.best_iteration ).reshape(oof_pred[val_idx].shape)
 if y_valid is not None:
y_valid += model.predict(x_valid, num_iteration=model.best_iteration ).reshape(y_valid.shape)/ self.n_splits
y_pred += model.predict(x_test, num_iteration=model.best_iteration ).reshape(y_pred.shape)/ self.n_splits
print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))
 models.append(model)
 del train_set, val_set, x_val, y_val
gc.collect()
fi_df = pd.DataFrame()
for n in np.arange(self.n_splits):
tmp = pd.DataFrame()
tmp["features"] = self.features+self.categoricals
tmp["importance"] = fi[n, :]
tmp["fold"] = n
fi_df = pd.concat([fi_df, tmp], ignore_index=True)
 gfi = fi_df[["features", "importance"]].groupby(["features"] ).mean().reset_index()
fi_df = fi_df.merge(gfi, on="features", how="left", suffixes=('', '_mean'))
loss_score = self.calc_metric(y_vals, oof_pred)
 print('Our oof loss score is: ', loss_score)
 return y_pred, y_valid, loss_score, models, oof_pred, y_vals, fi_df
def plot_feature_importance(self, rank_range=[1, 100]):
fig, ax = plt.subplots(1, 1, figsize=(16, 12))
sorted_df = self.fi_df.sort_values(by="importance_mean", ascending=False ).reset_index()
sns.barplot(data=sorted_df.iloc[self.n_splits*(rank_range[0]-1): self.n_splits*rank_range[1]],
x="importance", y="features", orient='h')
 ax.set_xlabel("feature importance")
 ax.spines['top'].set_visible(False)
 ax.spines['right'].set_visible(False)
 return sorted_df
class LgbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
model = lgb.train(self.params, train_set, num_boost_round=3000,
valid_sets=[train_set, val_set], verbose_eval=verbosity)
 fi = model.feature_importance(importance_type="gain")
 return model, fi
def convert_dataset(self, x_train, y_train, x_val=None, y_val=None):
train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)
 if x_val is not None:
val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)
 return train_set, val_set
return train_set
def get_params(self):
params = {
'boosting_type' : "gbdt",
'objective' : self.task,
"subsample" : 0.4,
"subsample_freq" : 1,
'max_depth' : 4,
'min_data_in_leaf' : 50,
'learning_rate' : 0.05,
'early_stopping_rounds' : 100,
'bagging_seed' : 11,
'random_state' : 42,
'verbosity' : -1
}
if self.task == "regression":
params["metric"] = "regression_l2"
elif self.task == "quantile":
params["metric"] = "quantile"
params["alpha"] = self.alpha
elif self.task == "binary":
params["metric"] = "auc"
elif self.task == "multiclass":
params["metric"] = "multi_logloss"
params["num_class"] = len(self.train_df[self.target].unique())
 if self.parameter_tuning:
def get_dataset() :
if self.tune_df is not None:
tune = self.tune_df.copy()
else:
tune = self.train_df.copy()
if self.group is not None:
train_num = int(tune.shape[0] * 0.7)
 test_num = tune.shape[0] - train_num
train_x = tune.head(train_num)[self.features+self.categoricals].reset_index(drop=True)
 train_y = tune.head(train_num)[[self.target]].reset_index(drop=True)
 test_x = tune.tail(test_num)[self.features+self.categoricals].reset_index(drop=True)
 test_y = tune.tail(test_num)[[self.target]].reset_index(drop=True)
 else:
train_x, test_x, train_y, test_y = train_test_split(tune[self.features+self.categoricals],
tune[self.target], test_size=0.3, random_state=self.seed)
 if self.categoricals != []:
dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)
 dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)
 else:
dtrain = lgb.Dataset(train_x, train_y)
 dtest = lgb.Dataset(test_x, test_y)
 return dtrain, dtest, test_x, test_y
def objective(trial):
dtrain, dtest, test_x, test_y = get_dataset()
hyperparams = {'max_depth' : trial.suggest_int('max_depth', 4, 16),
'max_bin' : trial.suggest_int('max_bin', 200, 1000),
'num_leaves' : trial.suggest_int('num_leaves', 200, 1000),
'min_data_in_leaf' : trial.suggest_int('min_data_in_leaf', 1, 300),
'min_child_samples' : trial.suggest_int('min_child_samples', 5, 100),
'feature_fraction' : trial.suggest_uniform('feature_fraction', 0.2, 1.0),
'bagging_fraction' : trial.suggest_uniform('bagging_fraction', 0.2, 1.0),
'bagging_freq' : trial.suggest_int('bagging_freq', 0, 7),
'lambda_l1' : trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2' : trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'early_stopping_rounds' : 150}
params.update(hyperparams)
 verbosity = 100 if self.verbose else 0
model = lgb.train(params, dtrain, valid_sets=dtest,
num_boost_round=1000, verbose_eval=verbosity)
 pred = model.predict(test_x)
 return self.calc_metric(test_y, pred)
 if self.tuning_type == "optuna_tuner":
dtrain, dtest, _, _ = get_dataset()
verbosity = 100 if self.verbose else 0
tuner_params = {"objective": params["objective"],
"metric" : params["metric"]}
model = optuna_lgb.train(tuner_params, dtrain, valid_sets=dtest,
num_boost_round=1000, early_stopping_rounds=50,
verbose_eval=verbosity)
 print('Best params:')
 for key, value in model.params.items() :
print(' {}: {}'.format(key, value))
params = model.params
params["feature_pre_filter"] = True
else:
study = optuna.create_study(direction='maximize')
 study.optimize(objective, n_trials=40)
 print('Number of finished trials: {}'.format(len(study.trials)))
 trial = study.best_trial
print('Best trial:')
 print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
params.update(trial.params)
 plot_optimization_history(study)
 return params<define_variables>
print("number of training examples = " + str(X_train.shape[0]))
print("number of test examples = " + str(X_test.shape[0]))
print("X_train shape: " + str(X_train.shape))
print("Y_train shape: " + str(Y_train.shape))
print("X_test shape: " + str(X_test.shape))
not_use_cols = ["row_id","user_id","user_answer","prior_question_had_explanation",
"question_rate_class","question_rate_class_exp",'part_rate_curt','part_ratec_curt',"abi_rate_curt",
"cv_group","max_time_stamp","rand_time_stamp","viretual_time_stamp"]
not_use_cols = not_use_cols + ans_part_cols + num_part_cols + part_mistake_cols + tags_cols<sort_values>
datagen = ImageDataGenerator(
 rotation_range=15,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1)
 datagen.fit(X_train )
df_corr[["answered_correctly"]].abs().sort_values("answered_correctly")[["answered_correctly"]]<define_variables>
def Keras_Model(input_shape):
X_input = Input(input_shape)
 X = Conv2D(64,(3, 3), strides =(1, 1), padding = 'same', name = 'conv0' )(X_input)
 X = BatchNormalization(axis = 3, name = 'bn0' )(X)
 X = Activation('relu' )(X)
 X = Conv2D(64,(3, 3), strides =(1, 1), padding = 'same', name = 'conv1' )(X)
 X = BatchNormalization(axis = 3, name = 'bn1' )(X)
 X = Activation('relu' )(X)
 X = MaxPooling2D(( 2, 2), name='max_pool_1' )(X)
 X = Dropout(0.35 )(X)
 X = Conv2D(128,(3, 3), strides =(1, 1), padding = 'same', name = 'conv2' )(X)
 X = BatchNormalization(axis = 3, name = 'bn2' )(X)
 X = Activation('relu' )(X)
 X = Conv2D(128,(3, 3), strides =(1, 1), padding = 'same', name = 'conv3' )(X)
 X = BatchNormalization(axis = 3, name = 'bn3' )(X)
 X = Activation('relu' )(X)
 X = MaxPooling2D(( 2, 2), name='max_pool_2' )(X)
 X = Dropout(0.35 )(X)
 X = Conv2D(256,(3, 3), strides =(1, 1), padding = 'same', name = 'conv4' )(X)
 X = BatchNormalization(axis = 3, name = 'bn4' )(X)
 X = Activation('relu' )(X)
 X = Dropout(0.35 )(X)
 X = Flatten()(X)
 X = Dense(1000, activation='relu', name='fc0' )(X)
 X = Dropout(0.5 )(X)
 X = Dense(256, activation='relu', name='fc2' )(X)
 X = Dense(10, activation='softmax', name='fc3' )(X)
 model = Model(inputs = X_input, outputs = X, name='model')
 return model
not_use_cols += ["prior_question_elapsed_time",'bundle_id',"bundle_skew","num_solving_question",
"num_intention","num_lec","num_concept","num_concept","num_tag",'deviation_m',"LorR",'part']
categoricals = []
keep_cols = ["user_id"]
target = "answered_correctly"
group = "cv_group"
features = [c for c in train.columns if c not in not_use_cols+categoricals+[target]]
print(len(features+categoricals))
print("Numeric features")
 display(sorted(features))
print("Categorical features")
 print(sorted(categoricals))<count_unique_values>
Keras_Model = Keras_Model(X_train.shape[1:4] )
user_ids = train.user_id.unique()
random.shuffle(user_ids)
 use_features = [target, group] + keep_cols + features + categoricals
trains = []
for i in range(1, MODEL_NUM+1):
print("Model %s will be from 0 to %s in cv_group" %(i, CV-1))
train_data = train.loc[train.cv_group.isin(np.arange(CV)) , use_features].copy()
train_data.reset_index(drop=True, inplace=True)
 trains.append(train_data)
 test_data = train.loc[train.cv_group==CV, use_features].copy()
test_data.reset_index(drop=True, inplace=True)
 if TUNE:
tune_data = train.loc[train.cv_group==CV+1, use_features].copy()
tune_data.reset_index(drop=True, inplace=True)
 else:
tune_data = pd.DataFrame()
del train, train_data
gc.collect()<count_unique_values>
epochs = 100
batch_size = 64
lrate = 0.0001
decay = lrate/epochs
optimizer = Adam(lr=lrate, epsilon=1e-08, decay = 0.00 )
print("Training data")
 for i, train_data in enumerate(trains):
print("No.", i+1)
 print(train_data.shape, train_data.cv_group.unique() , train_data.user_id.nunique())
 display(train_data.head())
 print("Test data")
 print(test_data.shape, test_data.cv_group.unique() , test_data.user_id.nunique())
 display(test_data.head())
 if TUNE:
print("Tune data")
 print(tune_data.shape, tune_data.cv_group.unique() , tune_data.user_id.nunique())
 display(tune_data.head())
 del train_data
gc.collect()<init_hyperparams>
Keras_Model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'] )
params1 = {
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
'learning_rate': 0.1,
"max_depth": 4,
"max_bin": 853,
"num_leaves": 492,
"min_data_in_leaf": 229,
"min_child_samples": 55,
"feature_fraction": 0.7894275993264507,
"bagging_fraction": 0.8305608180096481,
"bagging_freq": 6,
"lambda_l1": 0.00010208001912287773,
"lambda_l2": 9.315928833771094,
'early_stopping_rounds' : 100,
'random_state': 42,
"bagging_seed": 11,
"verbosity": -1}
params2 = {
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
'learning_rate': 0.05,
'max_depth': 4,
'max_bin': 700,
'num_leaves': 350,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'early_stopping_rounds' : 100,
'random_state': 42,
"bagging_seed": 11,
"verbosity": -1}
params3 = {
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
'learning_rate': 0.05,
"max_depth": 9,
"max_bin": 872,
"num_leaves": 743,
"min_data_in_leaf": 274,
"min_child_samples": 27,
"feature_fraction": 0.5922083852690104,
"bagging_fraction": 0.7638997668812837,
"bagging_freq": 5,
"lambda_l1": 4.603407547894731e-07,
"lambda_l2": 4.833680631174746e-07,
'early_stopping_rounds' : 200,
'random_state': 42,
"bagging_seed": 11,
"verbosity": -1}
params4 = {
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
'learning_rate': 0.05,
"max_depth": 8,
"max_bin": 990,
"num_leaves": 885,
"min_data_in_leaf": 214,
"min_child_samples": 89,
"feature_fraction": 0.4990107527526446,
"bagging_fraction": 0.9513631160877628,
"bagging_freq": 7,
"lambda_l1": 4.856465786762882e-05,
"lambda_l2": 1.6804951000850403e-08,
'early_stopping_rounds' : 200,
'random_state': 42,
"bagging_seed": 11,
"verbosity": -1}
if TUNE:
params = [None] * MODEL_NUM
elif MODEL_NUM == 1:
params = [params3]
else:
params = [params3, params4]
tuning_type = "optuna"<choose_model_class>
learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=2,
verbose=1,
factor=0.5,
min_lr=0.0000001 )
%%time
lgbms = []
for idx, train_data in enumerate(trains):
if DEBUG:
lgbm = LgbModel(train_data.sample(n=30000 ).reset_index(drop=True), test_data,
target, features, categoricals=categoricals,
task="binary", params=None, parameter_tuning=TUNE, tuning_type=tuning_type,
tune_df=tune_data, cv_method="GroupKFold", n_splits=4, group=group, verbose=False)
 else:
lgbm = LgbModel(train_data, test_data,
target, features, categoricals=categoricals,
task="binary", params=params[idx], parameter_tuning=TUNE, tuning_type=tuning_type,
tune_df=tune_data, cv_method="GroupKFold", n_splits=4, group=group, verbose=False)
 lgbms.append(lgbm)
 del lgbm
gc.collect()<compute_train_metric>
early_stopping = EarlyStopping(monitor='val_loss', patience=3, verbose=1, mode='auto' )
for lgbm in lgbms:
score = roc_auc_score(test_data.answered_correctly, lgbm.y_pred)
 fpr, tpr, thresholds = roc_curve(test_data.answered_correctly, lgbm.y_pred)
 print(score)
 plt.plot(fpr, tpr )<predict_on_test>
history = Keras_Model.fit(x = X_train, y = Y_train, batch_size = batch_size,
epochs=epochs, verbose=1,
validation_data =(X_val, Y_val),
steps_per_epoch= None, validation_steps=None,
callbacks=[learning_rate_reduction, early_stopping] )
show_cols = ["correct","answered_correctly","predict","predict_prob","user_id"]+features
test_data["predict_prob"] = lgbm.y_pred
test_data["predict"] = test_data.predict_prob.apply(lambda x: 1 if x>0.5 else 0)
 test_data["correct"] = test_data[["answered_correctly","predict"]].apply(lambda x: 1 if x[0]==x[1] else 0, axis=1 )<sort_values>
preds = Keras_Model.evaluate(X_train, Y_train)
 print("Loss = " + str(preds[0]))
print("Train set Accuracy = " + str(preds[1]))
test_data.sort_values("predict_prob", ascending=False, inplace=True)
 test_data[show_cols].head()<sort_values>
preds = Keras_Model.evaluate(X_val, Y_val)
 print("Loss on Val set= " + str(preds[0]))
print("Val set Accuracy = " + str(preds[1]))
df_res_sorted = test_data.loc[(test_data.correct==0), show_cols].sort_values("predict_prob", ascending=False)
 df_res_sorted.head(20 )<save_to_csv>
val_loss = history_dict['val_loss']
val_acc = history_dict['val_acc']
loss = history_dict['loss']
acc = history_dict['acc']
epochs = range(1,len(history_dict['val_loss'])+1 )
for u in df_res_sorted.user_id.unique() [:3]:
test_data.loc[test_data.user_id==u, show_cols].to_csv("debug_"+str(u)+".csv", index=False )<find_best_params>
predicted_val_probability = Keras_Model.predict(X_val, batch_size=32)
 Y_val_pred_label = np.argmax(predicted_val_probability, axis = 1 )
models = []
for lgbm in lgbms:
models += lgbm.models
print(len(models))
del lgbms, train_data, tune_data
gc.collect()<categorify>
Y_val_True_label = np.argmax(Y_val, axis = 1 )
def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask )<choose_model_class>
j = []
max_probability = []
for i in range(len(Y_val_pred_label)) :
if Y_val_pred_label[i] != Y_val_True_label[i]:
j.append(i )
class FFN(nn.Module):
def __init__(self, state_size=200, forward_expansion=1, bn_size=(180-1), dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads=8, dropout=0.1, forward_expansion=1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion=forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.dropout(self.layer_normal_2(x + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=180, embed_dim=128, dropout=0.1, forward_expansion=1, num_layers=1, heads=8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill + 1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=180, embed_dim=128, dropout=0.1, forward_expansion=1, enc_layers=1, heads=8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim+1, 1)
 def forward(self, x1, x2, question_ids):
device = x1.device
x, att_weight = self.encoder(x1, question_ids)
 x = torch.cat([x, x2.unsqueeze(2)], dim=2 ).to(device)
 x = self.pred(x)
 return x.squeeze(-1), att_weight<load_pretrained>
l = []
for ele in j:
if np.max(predicted_val_probability[ele])> 0.95:
l.append(ele )
%%time
device = torch.device("cpu")
 skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-own/group.pkl.zip")
 lags = joblib.load("/kaggle/input/riiid-sakt-model-dataset-own/lags.pkl.zip")
 nn_model = SAKTModel(n_skill, max_seq=180, embed_dim=128, forward_expansion=1, enc_layers=1, heads=8, dropout=0.1)
 nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-own/sakt_model.pt", map_location='cpu'))
nn_model.to(device)
 nn_model.eval()<categorify>
classes = Keras_Model.predict(X_test, batch_size=32 )
def insert_new_data_to_hist(ary, ins):
ary = ary.copy()
ary = np.vstack([ary, ins])
 ary = ary[np.argsort(ary[:,0])]
return ary
def groupby_with_numpy(ary, t_idxs, qucnt):
ary = ary.copy()
g = sum([[i+1 for _ in range(c)] for i, c in enumerate(qucnt)], [])
 r = np.zeros(( qucnt.shape[0], t_idxs.shape[0]))
for i, t in enumerate(t_idxs):
r[:,i] = np.bincount(g, ary[:,t])[1:]
return r
def add_mistake_feat(df, t_col, n_col):
df = df.copy()
if df[n_col].min() == 0:
df[t_col] = df[n_col].apply(lambda x: 0.6*x**(-0.2)if x!=0 else 0.6)
 else:
df[t_col] = 0.6*df[n_col]**(-0.2)
 return df
def get_num_etimes(df):
num_etimes = []
for u, et in np.array(df[["user_id","prior_question_elapsed_time"]]):
if u in d.keys() :
etime_mean = d_etime[d[u][-1, 1]]
et_norm = et /(1000*3600)
 num_etimes.append(et_norm - etime_mean)
 else:
num_etimes.append(0)
 return num_etimes
def add_to_timedict(df):
for u, t in np.array(df[["user_id","timestamp"]]):
if u in d_timestamp.keys() :
d_timestamp[u] = np.unique(np.hstack([d_timestamp[u], np.array([t])]))
else:
d_timestamp[u] = np.array([t])
 def add_to_history(df, new_row_id):
for r in np.array(df[["user_id","question_id","part","tag1"]]):
if r[0] in d.keys() :
d[r[0]] = np.vstack([d[r[0]], np.hstack([np.array(new_row_id), r[1:], np.array(0)])])
 else:
d[r[0]] = np.array([np.hstack([np.array(new_row_id), r[1:], np.array(0)])])
 def upd_history_ans(df):
for r in np.array(df[["user_id","answered_correctly"]]):
if r[0] in d.keys() and r[1] == 1:
d[r[0]][-1,-1] = r[1]<merge>
class_test_set = np.argmax(classes, axis = 1 )
<prepare_x_and_y><EOS> 
prediction = pd.DataFrame()
prediction['ImageId'] = np.asarray(range(1,28001))
prediction['Label'] = class_test_set
prediction.to_csv('submission.csv', index = False )