Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<concatenate>
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl
!pip install.. /input/pandarallel151whl/pandarallel-1.5.1-py3-none-any.whl
data = pd.concat(objs=[train, test], axis=0 ).reset_index(drop=True )<drop_column>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 session_conf = tf.compat.v1.ConfigProto(
 intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1 
)
 sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph() , config=session_conf)
 tf.compat.v1.keras.backend.set_session(sess)
 warnings.simplefilter('ignore')
 pd.set_option("max_columns", 150)
 pd.set_option('display.max_rows', 150)
 seed_everything(42)
 pandarallel.initialize()
data.drop('PassengerId', axis=1, inplace=True )<data_type_conversions>
DEBUG = False
TUNE = False
ENSEMBLE = True
S = 0.3
L = 0.7
median = data["Age"].median()
std = data["Age"].std()
is_null = data["Age"].isnull().sum()
rand_age = np.random.randint(median - std, median + std, size = is_null)
 age_slice = data["Age"].copy()
age_slice[np.isnan(age_slice)] = rand_age
data["Age"] = age_slice
data["Age"] = data["Age"].astype(int )<correct_missing_values>
%%time
train = reduce_mem_usage(dt.fread(".. /input/riiid-test-answer-prediction/train.csv" ).to_pandas())
 df_qs = reduce_mem_usage(pd.read_csv('.. /input/riiid-test-answer-prediction/questions.csv'))
df_lecs = reduce_mem_usage(pd.read_csv('.. /input/riiid-test-answer-prediction/lectures.csv'))
data['Embarked'].fillna(data['Embarked'].mode() [0], inplace = True )<count_missing_values>
train.prior_question_elapsed_time =(train.prior_question_elapsed_time.fillna(0)).astype(int)
 train.prior_question_had_explanation =(train.prior_question_had_explanation.fillna(False)* 1 ).astype(int)
 train.content_type_id =(train.content_type_id * 1 ).astype(int)
 train = reduce_mem_usage(train )
data["Embarked"].isnull().sum()<data_type_conversions>
train_lec = train.query("content_type_id==1" ).copy()
train_lec.reset_index(drop=True, inplace=True)
 train.drop(train.loc[train.content_type_id==1].index, inplace=True)
 train.reset_index(drop=True, inplace=True)
 train.rename(columns={"content_id": "question_id"}, inplace=True)
 train_lec.rename(columns={"content_id": "question_id"}, inplace=True)
 gc.collect()
train.shape, train_lec.shape
data['Fare'].fillna(data['Fare'].mean() , inplace = True )<data_type_conversions>
def cut(df, t_col, n_col):
df = df.copy()
df[n_col] = pd.cut(df[t_col], [-0.1, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.1], labels=False)
 df[n_col] = df[n_col].fillna(99 ).astype("int8")
 return df
def add_que_rate_class(df):
df = df.copy()
df["question_rate_class"] = 0
for sta, end in zip(np.arange(0.3, 1.0, 0.1), np.arange(0.4, 1.1, 0.1)) :
sta = round(sta,1)
 end = round(end,1)
 end = 1.1 if end==1 else end
df.loc[(sta <= df.question_rate)&(df.question_rate < end), "question_rate_class"] = int(sta*10)
 return df
def get_user_items_dict(ary, sizes_cumsum):
end = sizes_cumsum
sta = np.hstack([np.array([0]), end[:-1]])
 users = np.unique(ary[:,0])
 d = {}
for u, s, e in zip(users, sta, end):
d[u] = ary[s:e, 1:]
return d
def change_rate_with_num(df, t_col, n_col, method="respectively", n1=5, n2=3):
df = df.copy()
if method == "respectively":
df.loc[(df[n_col] < n1)&(df[t_col] <= 0.3), t_col] = 0.3
df.loc[(df[n_col] < n1)&(0.3 < df[t_col])&(df[t_col] <= 0.7), t_col] = 0.5
df.loc[(df[n_col] < n1)&(0.7 < df[t_col]), t_col] = 0.7
else:
df.loc[(df[n_col] < n2), t_col] = 0.5
return df
def change_rate_high_low(df, t_col, h=0.9, l=0.1):
df = df.copy()
df.loc[df[t_col] < l, t_col] = l
df.loc[df[t_col] > h, t_col] = h
return df
data["CabinBool"] =(data["Cabin"].notnull().astype('int'))<feature_engineering>
CV = 4
MODEL_NUM = 1
USE_USER_NUM = 10000
if DEBUG:
USE_USER_NUM = 500
if TUNE:
TRAIN_RATE, TUNE_RATE, TEST_RATE = 0.4, 0.3, 0.3
else:
TRAIN_RATE, TEST_RATE = 0.7, 0.3
print("%s users reduce to %s" %(train.user_id.nunique() , USE_USER_NUM))
user_ids = train.user_id.unique()
random.shuffle(user_ids)
 ext_user = user_ids[:USE_USER_NUM]
ext_user_bol = np.isin(np.array(train.user_id), np.sort(ext_user))
sizes_quser = np.array(train.user_id.value_counts().sort_index().values, dtype="int16")
 sizes_qcumsum = sizes_quser.cumsum()
unique_user_ids = np.array(train.user_id)[sizes_qcumsum-1]
data['Deck'] = data.Cabin.str.extract('([a-zA-Z]+)', expand=False)
 data[['Cabin', 'Deck']].sample(10)
 data['Deck'] = data['Deck'].fillna('Z')
 data = data.drop(['Cabin'], axis=1 )<groupby>
df_etime = pd.DataFrame({"question_id" : np.array(train.question_id[:-1]),
"prior_question_elapsed_time": np.array(train.prior_question_elapsed_time[1:])})
 df_etime = df_etime.loc[df_etime.prior_question_elapsed_time!=0].copy()
df_etime.prior_question_elapsed_time = df_etime.prior_question_elapsed_time /(1000*3600)
 df_etime["etime_kurt"] = df_etime.prior_question_elapsed_time
df_etime["etime_std"] = df_etime.prior_question_elapsed_time
df_etime["etime_skew"] = df_etime.prior_question_elapsed_time
df_etime_kurt = df_etime.groupby("question_id", as_index=False ).etime_kurt.apply(pd.DataFrame.kurt)
 df_etime_rate = df_etime.groupby("question_id", as_index=False ).agg({"prior_question_elapsed_time": "mean",
"etime_std" : "std",
"etime_skew": "skew"})
 df_etime_rate.rename(columns={"prior_question_elapsed_time": "etime_mean"}, inplace=True)
 df_etime_rate = df_etime_rate.merge(df_etime_kurt, on="question_id")
 df_etime_rate.fillna(0, inplace=True)
 df_etime_rate.question_id = df_etime_rate.question_id.astype("int32")
 df_etime_rate.etime_mean = df_etime_rate.etime_mean.astype("float32")
 df_etime_rate.etime_std = df_etime_rate.etime_std.astype("float32")
 df_etime_rate.etime_kurt = df_etime_rate.etime_kurt.astype("float32")
 df_etime_rate.etime_skew = df_etime_rate.etime_skew.astype("float32")
 d_etime = dict(zip(df_etime_rate.question_id, df_etime_rate.etime_mean))
del df_etime, df_etime_kurt
gc.collect()
print(df_etime_rate.shape)
 df_etime_rate.head(2 )
data.groupby(['Embarked'])['Survived'].count()<feature_engineering>
df_etmie_rate = train[["row_id","user_id","question_id","prior_question_elapsed_time"]].copy()
df_etmie_rate.rename(columns={"prior_question_elapsed_time":"lag_etime"}, inplace=True)
 df_etmie_rate.lag_etime = df_etmie_rate.lag_etime /(1000*3600)
 df_etmie_rate.lag_etime = df_etmie_rate.groupby("user_id" ).lag_etime.shift(-1)
 df_etmie_rate = df_etmie_rate.merge(df_etime_rate[["question_id","etime_mean"]], on="question_id")
 df_etmie_rate.lag_etime = df_etmie_rate.lag_etime - df_etmie_rate.etime_mean
df_etmie_rate.sort_values("row_id", inplace=True)
 df_etmie_rate.reset_index(drop=True, inplace=True)
 df_etmie_rate["num"] = 1
df_etmie_rate["num"] = df_etmie_rate.groupby("user_id" ).num.shift(1)
 df_etmie_rate["num"] = df_etmie_rate.groupby("user_id" ).num.cumsum()
df_etmie_rate["num_etime"] = df_etmie_rate.groupby("user_id" ).lag_etime.shift(1)
 df_etmie_rate["num_etime"] = df_etmie_rate.groupby("user_id" ).num_etime.cumsum()
df_etmie_rate["etime_rate"] = df_etmie_rate.num_etime / df_etmie_rate.num
df_etmie_rate.fillna(0, inplace=True)
 df_etmie_rate.num_etime = df_etmie_rate.num_etime.astype("float32")
 df_etmie_rate.etime_rate = df_etmie_rate.etime_rate.astype("float32")
 df_etmie_rate.drop(["row_id","user_id","question_id","lag_etime","etime_mean","num"], axis=1, inplace=True)
 print(df_etmie_rate.shape)
 display(df_etmie_rate.head(3))
ar_etmie_rate = np.array(df_etmie_rate)
 hist_ar_etmie_rate, ar_etmie_rate = get_hist_and_training_data(ar_etmie_rate)
 hist_ar_etmie_rate, ar_etmie_rate = hist_ar_etmie_rate[:,0], ar_etmie_rate[:,1]
del df_etmie_rate
gc.collect()
data['FamilySize'] = data['SibSp'] + data['Parch']<feature_engineering>
df_lecs = df_lecs.join(pd.DataFrame(pd.get_dummies(df_lecs.type_of)))
 df_lecs.rename(columns={"lecture_id" :"question_id",
"solving question":"solving_question"}, inplace=True)
 df_lecs.drop(["tag","type_of"], axis=1, inplace=True)
 df_lecs = reduce_mem_usage(df_lecs)
 tags = sum(df_qs.tags.apply(lambda x: [] if x is np.nan else x.split(" ")) , [])
 df_tags = pd.DataFrame(pd.Series(tags ).value_counts() , columns=["tag"])
 tags_num = df_tags.to_dict() ["tag"]
df_tags.reset_index(inplace=True)
 df_tags.columns = ["tag","num"]
df_tags["total"] = df_qs.shape[0]
df_tags["tfidf"] = 1 * np.log(df_tags.total / df_tags.num)
 df_tags.index = df_tags.tag
tags_tfidf = df_tags[["tfidf"]].to_dict() ["tfidf"]
df_tags = pd.DataFrame(list(df_qs.tags.apply(lambda x: [] if x is np.nan else x.split(" "))),
columns=["tag"+str(t)for t in range(1,7)])
 df_tags = df_tags.fillna(0 ).astype("int16")
 df_tags = df_qs[["question_id"]].join(df_tags)
 df_qs["tag_rel"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else sum([tags_num[t] for t in x.split(" ")]))
df_qs["tag_tfidf_sum"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else sum([tags_tfidf[t] for t in x.split(" ")]))
df_qs["num_tag"] = df_qs.tags.apply(lambda x: 0 if x is np.nan else len(x.split(" ")))
 df_qs.drop(["correct_answer","tags"], axis=1, inplace=True)
 df_qs = df_qs.merge(df_tags, on="question_id")
 df_qs = reduce_mem_usage(df_qs)
 del df_tags, tags_num, tags_tfidf
gc.collect()
print(df_lecs.shape, df_qs.shape)
 display(df_lecs.head(2))
display(df_qs.head(2))
data['IsAlone'] = 1<feature_engineering>
train = train.merge(df_qs[["question_id","bundle_id","part","tag1"]], on='question_id', how='left')
 train = reduce_mem_usage(train )
data['IsAlone'].loc[data['FamilySize'] > 0] = 0<data_type_conversions>
df = train[["user_id","question_id","bundle_id","user_answer","answered_correctly"]].copy()
df_question_rate = df_qs[["question_id","bundle_id"]].copy()
for c in ["question_id", "bundle_id"]:
c_name = c[:-3]
df[c_name+"_kurt"] = df.user_answer
df[c_name+"_skew"] = df.user_answer
df_kurt = df.groupby(c, as_index=False)[c_name+"_kurt"].apply(pd.DataFrame.kurt)
 df_rate = df.groupby(c, as_index=False ).agg({"answered_correctly": "mean",
"user_answer" : "std",
c_name+"_skew" : "skew"})
 df_rate.rename(columns={"answered_correctly":c_name+"_rate",
"user_answer" :c_name+"_std"}, inplace=True)
 df_rate = df_rate.merge(df_kurt, on=c)
 df_question_rate = df_question_rate.merge(df_rate, on=c, how="left")
 df_question_rate.fillna(0, inplace=True)
 min2_val = sorted(df_question_rate.question_rate.unique())[1]
ins_row = df_question_rate[df_question_rate.question_rate == min2_val].iloc[:,2:]
df_question_rate.loc[df_question_rate.question_rate == 0, ins_row.columns] = ins_row.values
df_question_rate["deviation_p"] = df_question_rate[["question_rate","question_std"]].apply(lambda x: get_deviation_value(x[0],x[1], "p"), axis=1)
 df_question_rate["deviation_m"] = df_question_rate[["question_rate","question_std"]].apply(lambda x: get_deviation_value(x[0],x[1], "m"), axis=1)
 df_question_rate = add_que_rate_class(df_question_rate)
 df_question_rate.bundle_id = df_question_rate.bundle_id.astype("int16")
 df_question_rate.question_rate_class = df_question_rate.question_rate_class.astype("int8")
 df_question_rate = reduce_mem_usage(df_question_rate)
 df_part_rate = train.groupby("part", as_index=False ).answered_correctly.mean()
df_part_rate.rename(columns={"answered_correctly": "part_rate"}, inplace=True)
 df_part_rate = reduce_mem_usage(df_part_rate)
 del df, df_kurt, df_rate, min2_val, ins_row
gc.collect()
print(df_question_rate.shape, df_part_rate.shape)
 display(df_question_rate.head(2))
display(df_part_rate.head(2))
bins = [-1, 13, 31, 60, 80]
labels = ['Child', 'Young Adult', 'Adult', 'Senior']
data['AgeBin'] = pd.cut(data["Age"], bins, labels = labels ).astype('object' )<feature_engineering>
df_question_rate = df_question_rate.merge(df_que_exp_rate, on="question_id", suffixes=("", "_exp"))
del df_que_exp_rate
gc.collect()
data['IsBaby'] = 0<feature_engineering>
train = train.merge(df_question_rate[["question_id","prior_question_had_explanation","question_rate_exp","deviation_p","deviation_m"]],
on=["question_id","prior_question_had_explanation"])
 train.sort_values(by="row_id", inplace=True)
 train.reset_index(drop=True, inplace=True )
data['IsBaby'].loc[data['Age'] <= 5] = 1<feature_engineering>
ans = np.array(train.answered_correctly, dtype="int16")
 num = np.ones(train.shape[0], dtype="int16")
 exp = np.array(train.prior_question_had_explanation, dtype="int16")
 devp = ans * np.array(train.deviation_p, dtype="float64")
 devm = devp + np.where(ans==0, -1, 0)* np.array(train.deviation_m, dtype="float64")
 ans_cumsum = intervaled_cumsum(ans, sizes_qcumsum ).astype("int16")
 num_cumsum = intervaled_cumsum(num, sizes_qcumsum ).astype("int16")
 exp_cumsum = intervaled_cumsum(exp, sizes_qcumsum ).astype("int16")
 devp_cumsum = intervaled_cumsum(devp, sizes_qcumsum ).astype("float32")
 devm_cumsum = intervaled_cumsum(devm, sizes_qcumsum ).astype("float32")
 ans_rate =(ans_cumsum / num_cumsum ).astype("float32")
 study_rate =(exp_cumsum / shift_to_prior(num_cumsum, sizes_qcumsum)).astype("float32")
 study_rate[np.isnan(study_rate)] = 0
study_rate[np.isinf(study_rate)] = 0
hist_ans_cumsum, ans_cumsum = get_hist_and_training_data(ans_cumsum)
 hist_num_cumsum, num_cumsum = get_hist_and_training_data(num_cumsum)
 hist_exp_cumsum, exp_cumsum = get_hist_and_training_data(exp_cumsum)
 hist_devp_cumsum, devp_cumsum = get_hist_and_training_data(devp_cumsum)
 hist_devm_cumsum, devm_cumsum = get_hist_and_training_data(devm_cumsum)
 hist_ans_rate, ans_rate = get_hist_and_training_data(ans_rate)
 hist_study_rate, _ = get_hist_and_training_data(study_rate)
 train.drop(["deviation_p","deviation_m"], axis=1, inplace=True)
 del ans, num, exp, devp, devm
gc.collect()
data['Title'] = data.Name.str.extract('([A-Za-z]+)\.', expand=False )<feature_engineering>
%%time
num_part_cols = ["num_part"+str(p)for p in range(1,8)]
ans_part_cols = ["ans_part"+str(p)for p in range(1,8)]
part_rate_cols = ["part"+str(p)+"_rate" for p in range(1,8)]
part_cols = ans_part_cols + num_part_cols
ar_part_num = np.array(pd.get_dummies(train.part))
ar_part_ans = ar_part_num * np.array(train.answered_correctly ).reshape(-1,1)
 ar_part = np.hstack([ar_part_ans, ar_part_num] ).astype("int16")
 del ar_part_num, ar_part_ans
gc.collect()
for i in range(ar_part.shape[1]):
ar_part[:,i] = intervaled_cumsum(ar_part[:,i], sizes_qcumsum)
 ar_part_rate =(ar_part[:,:7] / ar_part[:,7:] ).astype("float32")
 np.nan_to_num(ar_part_rate, copy=False)
 hist_ar_part_rate, _ = get_hist_and_training_data(ar_part_rate)
 hist_ar_part, ar_part = get_hist_and_training_data(ar_part )
data['Title'] = data['Title'].replace(['Lady', 'Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Jonkheer', 'Dona'], 'Rare')
 data['Title'] = data['Title'].replace(['Countess', 'Lady', 'Sir'], 'Royal')
 data['Title'] = data['Title'].replace('Mlle', 'Miss')
 data['Title'] = data['Title'].replace('Ms', 'Miss')
 data['Title'] = data['Title'].replace('Mme', 'Mrs' )<data_type_conversions>
abi_cols = []
abi_rate_cols = []
for sta, end in zip([0.0] + list(np.arange(0.3, 1.0, 0.1)) ,
[0.3] + list(np.arange(0.4, 1.1, 0.1))):
sta = round(sta,1)
 end = round(end,1)
 abi_cols += ["ans_abi" +str(int(sta*10)) , "num_abi"+str(int(sta*10)) ]
abi_rate_cols += ["abi_rate"+str(int(sta*10)) ]
end = 1.1 if end == 1 else end
print("Target question rate is from %s to %s" %(sta, end))
df_ability = train[["answered_correctly","question_rate_exp"]].copy()
df_ability["num"] = 1
df_ability.loc[(df_ability["question_rate_exp"] <= sta)|(end < df_ability["question_rate_exp"]), ["answered_correctly","num"]] = 0
abians = np.array(df_ability.answered_correctly, dtype="int16")
 abinum = np.array(df_ability.num, dtype="int16")
 del df_ability
gc.collect()
abians_cumsum = intervaled_cumsum(abians, sizes_qcumsum ).astype("int16")
 abinum_cumsum = intervaled_cumsum(abinum, sizes_qcumsum ).astype("int16")
 del abians, abinum
gc.collect()
abians_num = np.array([abians_cumsum, abinum_cumsum] ).T
hist_abians_num, abians_num = get_hist_and_training_data(abians_num)
 if sta == 0:
hist_ar_ability = hist_abians_num
ar_ability = abians_num
else:
hist_ar_ability = np.hstack([hist_ar_ability, hist_abians_num])
 ar_ability = np.hstack([ar_ability, abians_num])
 del abians_cumsum, abinum_cumsum, abians_num, hist_abians_num
gc.collect()
abi_cols = sorted(abi_cols)
 abi_rate_cols = sorted(abi_rate_cols)
 hist_ar_ability = np.hstack([hist_ar_ability[:,::2], hist_ar_ability[:,1::2]])
 ar_ability = np.hstack([ar_ability[:,::2], ar_ability[:,1::2]] )
data['Title'] = data['Title'].astype('object' )<groupby>
ar_class = np.hstack([unique_user_ids.reshape(-1,1),
hist_num_cumsum.reshape(-1,1),
hist_ans_rate.reshape(-1,1),
hist_study_rate.reshape(-1,1),
hist_ar_part[:,7:],
hist_ar_part_rate])
 df_class = pd.DataFrame(ar_class, columns=["user_id","num_que","ans_rate","study_rate"]+num_part_cols+part_rate_cols)
 del ar_class, hist_ans_rate, hist_study_rate, hist_ar_part_rate
gc.collect()
df_class.loc[df_class.num_que < 5, ["ans_rate","study_rate"]] = np.nan
for p in range(1,8):
df_class.loc[df_class["num_part"+str(p)] < 5, "part"+str(p)+"_rate"] = np.nan
df_class = cut(df_class, "ans_rate", "ans_class")
 df_class = cut(df_class, "study_rate", "study_class")
 for p in range(1,8):
df_class = cut(df_class, "part"+str(p)+"_rate", "part"+str(p)+"_class")
 df_class.user_id = df_class.user_id.astype("int32")
 df_class.ans_rate = df_class.ans_rate.astype("float32")
 df_class.study_rate = df_class.study_rate.astype("float32")
 df_class = df_class[["user_id"] + [c for c in df_class.columns if -1 < c.find("class")]]
df_class = reduce_mem_usage(df_class)
 print(df_class.shape)
 df_class.head()
data[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean()<data_type_conversions>
%%time
def make_rate_class(df, m_col):
df_rc = df.groupby([m_col, "question_id"], as_index=False ).agg(aggs)
 df_rc = df_rc.loc[df_rc[m_col] != 99].copy()
df_rc = change_rate_with_num(df_rc, "ans_rate_class", "ans_cnt")
 df_rc.drop("ans_cnt", axis=1, inplace=True)
 return reduce_mem_usage(df_rc)
 df_ans_class = train[["user_id","question_id","answered_correctly"]].merge(df_class, on="user_id")
 df_ans_class["ans_cnt"] = df_ans_class.answered_correctly
df_ans_class.rename(columns={"answered_correctly": "ans_rate_class"}, inplace=True)
 del df_class
gc.collect()
df_parts = []
aggs = {"ans_rate_class":"mean", "ans_cnt":"count"}
df_ans_rate_class = make_rate_class(df_ans_class, "ans_class")
 df_study_rate_class = make_rate_class(df_ans_class, "study_class")
 df_study_rate_class.rename(columns={"ans_rate_class":"study_rate_class"}, inplace=True)
 for p in range(1,8):
df_part_rate_class = make_rate_class(df_ans_class, f"part{p}_class")
 df_part_rate_class.rename(columns={"ans_rate_class":f"part{p}_rate_class"}, inplace=True)
 df_parts.append(df_part_rate_class)
 lo = ["base_class","question_id"]
df_ans_class = pd.DataFrame({"base_class" : np.repeat(np.arange(0,10), df_qs.question_id.nunique()),
"question_id": np.tile(df_qs.question_id.unique() , 10)})
 df_ans_class = df_ans_class.merge(df_ans_rate_class, left_on=lo, right_on=["ans_class", "question_id"], how="left")
 df_ans_class.drop("ans_class", axis=1, inplace=True)
 del df_ans_rate_class
df_ans_class = df_ans_class.merge(df_study_rate_class, left_on=lo, right_on=["study_class", "question_id"], how="left")
 df_ans_class.drop("study_class", axis=1, inplace=True)
 del df_study_rate_class
for p, df in enumerate(df_parts):
df_ans_class = df_ans_class.merge(df, left_on=lo, right_on=[f"part{p+1}_class","question_id"], how="left")
 df_ans_class.drop(f"part{p+1}_class", axis=1, inplace=True)
 del df_parts
df_ans_class.fillna(0.5, inplace=True)
 df_ans_class = reduce_mem_usage(df_ans_class)
 gc.collect()
print(df_ans_class.shape)
 df_ans_class.head()
bins = [-np.inf, 20, 30, 110, np.inf]
labels = ['Low', 'Mid', 'High', 'Extreme']
data['FareBin'] = pd.cut(data["Fare"], bins, labels = labels ).astype('object' )<create_dataframe>
add_a_cols = [c for c in train_lec.columns if c not in train.columns]
add_b_cols = [c for c in train.columns if c not in train_lec.columns]
for c in add_a_cols: train[c] = 0
for c in add_b_cols: train_lec[c] = 0
train_lec = train_lec[train.columns]
train_lec = reduce_mem_usage(train_lec)
 train = reduce_mem_usage(train)
 train = reduce_mem_usage(pd.concat([train, train_lec]))
del train_lec
gc.collect()
train.sort_values(by="row_id", inplace=True)
 train.reset_index(drop=True, inplace=True)
 train.shape
df = data<drop_column>
sizes_all_user = np.array(train.user_id.value_counts().sort_index().values, dtype="int16")
 sizes_all_cumsum = sizes_all_user.cumsum()
t_ary = np.array(train[["user_id","timestamp"]])
 d_timestamp = get_user_items_dict(t_ary, sizes_all_cumsum)
 for key, item in d_timestamp.items() :
d_timestamp[key] = np.unique(item)
 del t_ary
gc.collect()
df = df.drop(['Ticket', 'Name', 'Fare'], axis=1 )<train_model>
train.drop(["bundle_id","question_rate_exp","user_answer"], axis=1, inplace=True)
 train.drop(train.loc[train.content_type_id==1].index, inplace=True)
 train.drop(["content_type_id"], axis=1, inplace=True)
 train.reset_index(drop=True, inplace=True)
 train["LorR"] = 0
train.loc[train.part.isin([5,6,7]), "LorR"] = 1
train = reduce_mem_usage(train )
train = df[:split]
test = df[split:]
x = train.drop(["Survived"], axis=1)
 y = train["Survived"]
X_train, X_test, y_train, y_test = train_test_split(x,y, test_size=0.2, random_state=42)
 test.drop(["Survived"], axis = 1, inplace=True)
 cat_features = np.where(x.dtypes != float)[0]
cat = CatBoostClassifier(one_hot_max_size=7, iterations=21, random_seed=42, use_best_model=True, eval_metric='Accuracy', loss_function='Logloss')
 cat.fit(X_train, y_train, cat_features = cat_features, eval_set=(X_test, y_test))
pred = cat.predict(X_test)
 pool = Pool(X_train, y_train, cat_features=cat_features)
 cv_scores = cv(pool, cat.get_params() , fold_count=10, plot=True)
 print('CV score: {:.5f}'.format(cv_scores['test-Accuracy-mean'].values[-1]))
print('The test accuracy is :{:.6f}'.format(accuracy_score(y_test, cat.predict(X_test))))<drop_column>
ar_history = np.hstack([unique_user_ids.reshape(-1,1),
hist_ans_cumsum.reshape(-1,1),
hist_num_cumsum.reshape(-1,1),
hist_exp_cumsum.reshape(-1,1),
hist_devp_cumsum.reshape(-1,1),
hist_devm_cumsum.reshape(-1,1),
hist_ar_etmie_rate.reshape(-1,1),
hist_ar_part,
hist_ar_ability])
 df_history = pd.DataFrame(ar_history, columns=["user_id","num_ans","num_que","num_exp","num_devp","num_devm","num_etime"]+part_cols+abi_cols)
 for c in df_history.columns:
if c not in ["num_devp","num_devm","num_etime"]:
df_history[c] = df_history[c].astype(int)
 df_history = reduce_mem_usage(df_history)
 df_history.sort_values(by="user_id", inplace=True)
 df_history.reset_index(drop=True, inplace=True)
 del hist_ans_cumsum, hist_num_cumsum, hist_exp_cumsum, hist_devp_cumsum, hist_devm_cumsum, hist_ar_etmie_rate, hist_ar_part, hist_ar_ability
gc.collect()
print(df_history.shape)
 df_history.head()
to_drop = ['Age', 'AgeBin', 'SibSp', 'Parch', 'FamilySize', 'Embarked', 'Title']
df = df.drop(to_drop, axis=1, inplace=False )<categorify>
def get_attempt_que(u, r, q):
if u in d.keys() :
ua = d[u]
return np.count_nonzero(( ua[:,0] < r)*(ua[:,1] == q))
return 0
def get_attempt_curt_part(u, r, p):
if u in d.keys() :
ua = d[u]
return np.count_nonzero(ua[ua[:,0]<r][-5:,2] == p)
 return 0
def get_attempt_tags(u, r, t):
if u in d.keys() :
ua = d[u]
return np.count_nonzero(( ua[:,0] < r)*(ua[:,3] == t))
return 0
def get_attempt_curt_ans_rate(u, r):
if u in d.keys() :
ua = d[u]
return np.mean(ua[ua[:,0]<r][-10:, 4])
 return 0
def get_lagtime(u, t, l):
if u in d_timestamp.keys() :
ul = d_timestamp[u]
ul = ul[ul < t]
if ul.shape[0] >= 3:
return ul[l]
elif ul.shape[0] == 2 and l != -3:
return ul[l]
elif ul.shape[0] == 1 and l == -1:
return ul[l]
return 0
def get_noplay_days(u, t):
if u in d_timestamp.keys() :
ul = d_timestamp[u] /(1000*3600)
 ul = ul[ul <= t]
if 1 < ul.shape[0]:
di = np.diff(ul)
 return np.where(di<24,0,di ).sum()
return 0
def get_attempt_thedaybefore(u, t, d):
if u in d_timestamp.keys() :
ul = d_timestamp[u] /(1000*3600)
 return np.count_nonzero(( t-d <= ul)*(ul < t))
return 0
def cut_and_merge_with_class(df, df_class, t_col, c_col):
df = df.copy()
df["base_class"] = pd.cut(df[t_col], [-0.1, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.1], labels=False)
 return df.merge(df_class[["question_id","base_class",c_col]], on=["question_id","base_class"], sort=False )
categorical_feature_mask = df.dtypes==object
categorical_cols = df.columns[categorical_feature_mask].tolist()
le = LabelEncoder()
df[categorical_cols] = df[categorical_cols].apply(lambda col: le.fit_transform(col))
df[categorical_cols].head()<split>
data = train[train.user_id.isin(ext_user)].copy()
data.reset_index(drop=True, inplace=True)
 q_ary = np.array(train[["user_id","row_id","question_id","part","tag1","answered_correctly"]])
 q_ary = q_ary[np.argsort(q_ary[:,1])]
del train
gc.collect()
train = data.copy()
sizes_train_user = np.array(train.user_id.value_counts().sort_index().values, dtype="int16")
 sizes_train_cumsum = sizes_train_user.cumsum()
del data
gc.collect()
train.shape
train = df[:split]
test = df[split:]
x = train.drop(["Survived"], axis=1)
 y = train["Survived"]
X_train, X_test, y_train, y_test = train_test_split(x,y, test_size=0.2, random_state=42)
 test.drop(["Survived"], axis = 1, inplace=True )<choose_model_class>
%%time
def rand_time(max_time_stamp):
interval = MAX_TIME_STAMP - max_time_stamp
rand_time_stamp = random.randint(0,interval)
 return rand_time_stamp
max_timestamp_u = train[['user_id','timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
max_timestamp_u.columns = ['user_id', 'max_time_stamp']
MAX_TIME_STAMP = max_timestamp_u.max_time_stamp.max()
max_timestamp_u['rand_time_stamp'] = max_timestamp_u.max_time_stamp.apply(lambda x: rand_time(x))
train = train.merge(max_timestamp_u, on='user_id')
 train['viretual_time_stamp'] = train.timestamp + train['rand_time_stamp']
train.sort_values(['viretual_time_stamp', 'row_id'], inplace=True)
 train.reset_index(drop=True, inplace=True)
 train_num = int(train.shape[0] * TRAIN_RATE)
 if TUNE:
tune_num = int(train.shape[0] * TUNE_RATE)
 test_num = train.shape[0] -(train_num + tune_num)
 else:
test_num = train.shape[0] - train_num
tune_num = 0
print(train_num, test_num, tune_num)
 train["cv_group"] = 9
sta = 0
for c in range(CV):
end = int(train_num/CV)*(c+1)if(c+1)!= CV else train_num
train.loc[sta:end, "cv_group"] = c
print("CV:%s -- Start with %07d and end with %07d.Num of user is %s and mean correct answer rate is %f" %(c, sta, end, train.loc[train.cv_group==c].user_id.nunique() , train.loc[train.cv_group==c].answered_correctly.mean()))
sta = end
if TUNE:
train.loc[train_num+1:train_num+tune_num, "cv_group"] = 5
train.loc[train_num+tune_num+1:, "cv_group"] = 4
else:
train.loc[train_num+1:train_num+test_num, "cv_group"] = 4
train.sort_values('row_id', inplace=True)
 train.reset_index(drop=True, inplace=True )
MLA = [
ensemble.AdaBoostClassifier() ,
ensemble.ExtraTreesClassifier() ,
ensemble.GradientBoostingClassifier() ,
ensemble.RandomForestClassifier() ,
gaussian_process.GaussianProcessClassifier() ,
linear_model.LogisticRegressionCV() ,
linear_model.RidgeClassifierCV() ,
linear_model.Perceptron() ,
naive_bayes.BernoulliNB() ,
naive_bayes.GaussianNB() ,
neighbors.KNeighborsClassifier() ,
svm.SVC(probability=True),
svm.NuSVC(probability=True),
svm.LinearSVC() ,
tree.DecisionTreeClassifier() ,
tree.ExtraTreeClassifier() ,
xgb.XGBClassifier()
]<compute_train_metric>
devp_rate =(devp_cumsum / num_cumsum ).astype("float32")
 devm_rate =(devm_cumsum / num_cumsum ).astype("float32")
 train["ans_rate"] = shift_to_prior(ans_rate, sizes_train_cumsum ).astype("float32")
 train["devp_rate"] = shift_to_prior(devp_rate, sizes_train_cumsum ).astype("float32")
 train["devm_rate"] = shift_to_prior(devm_rate, sizes_train_cumsum ).astype("float32")
 train["num_que"] = shift_to_prior(num_cumsum, sizes_train_cumsum ).astype("int16")
 train["num_exp"] = exp_cumsum.astype("int16")
 train["study_rate"] =(train.num_exp / train.num_que ).replace([np.inf,-np.inf], np.nan ).fillna(0 ).astype("float32")
 train = reduce_mem_usage(train)
 del ans_rate, devp_rate, devm_rate, num_cumsum, exp_cumsum
gc.collect()
col = []
algorithms = pd.DataFrame(columns = col)
 idx = 0
for a in MLA:
a.fit(X_train, y_train)
 pred = a.predict(X_test)
 acc = accuracy_score(y_test, pred)
 f1 = f1_score(y_test, pred)
 cv = cross_val_score(a, X_test, y_test ).mean()
Alg = a.__class__.__name__
algorithms.loc[idx, 'Algorithm'] = Alg
algorithms.loc[idx, 'Accuracy'] = round(acc * 100, 2)
 algorithms.loc[idx, 'F1 Score'] = round(f1 * 100, 2)
 algorithms.loc[idx, 'CV Score'] = round(cv * 100, 2)
 idx+=1<sort_values>
ar_part_rate = ar_part[:,:7] / ar_part[:,7:]
np.nan_to_num(ar_part_rate, copy=False)
 ar_part_rate = ar_part_rate.astype("float32")
 for i in range(ar_part_rate.shape[1]):
ar_part_rate[:,i] = shift_to_prior(ar_part_rate[:,i], sizes_train_cumsum)
 for i in range(ar_part.shape[1]):
ar_part[:,i] = shift_to_prior(ar_part[:,i], sizes_train_cumsum)
 ar_part_rate.shape, ar_part.shape
algorithms.sort_values(by = ['CV Score'], ascending = False, inplace = True)
 algorithms.head()<choose_model_class>
ar_ability_rate = ar_ability[:,:8] / ar_ability[:,8:]
np.nan_to_num(ar_ability_rate, copy=False)
 ar_ability_rate = ar_ability_rate.astype("float32")
 for i in range(ar_ability_rate.shape[1]):
ar_ability_rate[:,i] = shift_to_prior(ar_ability_rate[:,i], sizes_train_cumsum)
 ar_ability_rate.shape
kfold = StratifiedKFold(n_splits=10 )<train_on_grid>
train = train.join(pd.DataFrame(ar_part_rate, columns=part_rate_cols))
train = train.join(pd.DataFrame(ar_part[:,7:], columns=num_part_cols))
train = train.join(pd.DataFrame(ar_ability_rate, columns=abi_rate_cols))
train = train.join(pd.DataFrame(ar_etmie_rate, columns=["etime_rate"]))
train.sort_values("row_id", inplace=True)
 train.reset_index(drop=True, inplace=True)
 del ar_part_rate, ar_part, ar_ability_rate
gc.collect()
train.shape
XGB = XGBClassifier()
xgb_param = {
'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsXGB = GridSearchCV(XGB, param_grid = xgb_param, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsXGB.fit(X_train,y_train)
 XGB_best = gsXGB.best_estimator_
gsXGB.best_score_<train_on_grid>
%%time
print("Calculating the gap of continuous")
 train["lagtime1"] =(train[["user_id","timestamp"]].parallel_apply(lambda x: get_lagtime(x[0], x[1], -1), axis=1)/(1000*3600)).astype("float32")
 train["lagtime2"] =(train[["user_id","timestamp"]].parallel_apply(lambda x: get_lagtime(x[0], x[1], -2), axis=1)/(1000*3600)).astype("float32")
 train["lagtime3"] =(train[["user_id","timestamp"]].parallel_apply(lambda x: get_lagtime(x[0], x[1], -3), axis=1)/(1000*3600)).astype("float32")
 train["timestamp"] =(train.timestamp /(1000*3600)).astype("float32")
 train["lagtime1"] =(train.timestamp - train.lagtime1 ).astype("float32")
 train["lagtime2"] =(train.timestamp - train.lagtime2 ).astype("float32")
 train["lagtime3"] =(train.timestamp - train.lagtime3 ).astype("float32")
 print("Calculating total of no play days")
 train["noplay_days"] =(train[["user_id","timestamp"]].parallel_apply(lambda x: get_noplay_days(x[0], x[1]), axis=1)).astype("float32")
 train["elapsed_days"] =(train.timestamp / 24 ).astype("int32")
 train["noplay_rate"] =(train.noplay_days / train.num_que ).fillna(0 ).astype("float32")
 train["timestamp_rate"] =(train.timestamp / train.num_que ).fillna(0 ).astype("float32")
 train["elapsed_rate"] =(train.elapsed_days / train.num_que ).fillna(0 ).astype("float32" )
SVC = svm.SVC(probability=True)
 svc_param = {
'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1, 10, 50, 100,200,300, 1000]
}
gsSVC = GridSearchCV(SVC, param_grid = svc_param, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVC.fit(X_train,y_train)
 SVC_best = gsSVC.best_estimator_
gsSVC.best_score_<train_on_grid>
%%time
d = get_user_items_dict(q_ary, sizes_qcumsum)
 print("Calculating the number of attempt against the same question")
 train["attempt_que"] = train[["user_id","row_id","question_id"]].parallel_apply(lambda x: get_attempt_que(x[0], x[1], x[2]), axis=1)
 tags_cols = ["tag"+str(t)for t in range(1,7)]
print("Calculating the number of attempt against the same tag")
 train["attempt_tags"] = train[["user_id","row_id","tag1"]].parallel_apply(lambda x: get_attempt_tags(x[0], x[1], x[2]), axis=1)
 print("Calculating the current correct answer rate")
 train["attempt_curtar"] = train[["user_id","row_id"]].parallel_apply(lambda x: get_attempt_curt_ans_rate(x[0], x[1]), axis=1)
GB = ensemble.GradientBoostingClassifier()
gb_param = {
'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGB = GridSearchCV(GB, param_grid = gb_param, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsGB.fit(X_train,y_train)
 GB_best = gsGB.best_estimator_
gsGB.best_score_<choose_model_class>
%%time
train = cut_and_merge_with_class(train, df_ans_class, "ans_rate", "ans_rate_class")
 train = cut_and_merge_with_class(train, df_ans_class, "study_rate", "study_rate_class")
 for p in range(1,8):
train = cut_and_merge_with_class(train, df_ans_class, f"part{p}_rate", f"part{p}_rate_class")
 train.drop("base_class", axis=1, inplace=True)
 train.shape
vc = ensemble.VotingClassifier(
 estimators = [('xgb', XGB_best),('gbc',GB_best),('svc', SVC_best)],
voting='soft', n_jobs=4 )<compute_train_metric>
train = train.merge(df_qs.drop(["part","tag1"], axis=1), on="question_id")
 train = train.merge(df_question_rate, on=["question_id","bundle_id","prior_question_had_explanation"])
 train = train.merge(df_etime_rate, on="question_id")
 train = train.merge(df_part_rate, on="part")
 gc.collect()
vc = vc.fit(X_train, y_train)
 pred = vc.predict(X_test)
 acc = accuracy_score(y_test, pred)
 f1 = f1_score(y_test, pred)
 cv = cross_val_score(vc, X_test, y_test ).mean()
print("Accuracy: ", round(acc*100,2), "
F1-Score: ", round(f1*100,2), "
CV Score: ", round(cv*100,2))<train_on_grid>
t_cols = ["ans_rate_class","study_rate_class"] + ["part"+str(p)+"_rate_class" for p in range(1,8)]
n_cols = ["num_que", "num_que"] + num_part_cols
for t, n in zip(t_cols, n_cols):
train = change_rate_with_num(train, t, n, method="respectively")
 train = change_rate_with_num(train, t, n, method="bundle" )
ada = ensemble.AdaBoostClassifier()
ada.fit(X_train, y_train)
 lg = linear_model.LogisticRegressionCV()
lg.fit(X_train, y_train)
 vc2 = ensemble.VotingClassifier(
 estimators = [('ada', ada),('lg',lg),('VotingClassifier', vc)],
voting='soft', n_jobs=4)
 vc2.fit(X_train, y_train )<predict_on_test>
train["prob_mistake_que"] = train.num_que.apply(lambda x: 0.6*x**(-0.2)if x!=0 else 0.6)
 part_mistake_cols = []
for p in range(1,8):
part_mistake_cols.append("prob_mistake_part"+str(p))
train["prob_mistake_part"+str(p)] = train["num_part"+str(p)].apply(lambda x: 0.6*x**(-0.2)if x!=0 else 0.6)
 all_prob_mistake = np.array(train[part_mistake_cols])
 train["prob_mistake_part_curt"] = np.nanmax(np.identity(8)[train.part][:,1:]*all_prob_mistake, axis=1)
 all_part_rate = np.array(train[["part1_rate","part2_rate","part3_rate","part4_rate","part5_rate","part6_rate","part7_rate"]])
 all_part_ratec = np.array(train[["part1_rate_class","part2_rate_class","part3_rate_class","part4_rate_class","part5_rate_class","part6_rate_class","part7_rate_class"]])
 all_abi_rate = np.array(train[["abi_rate0","abi_rate3","abi_rate4","abi_rate5","abi_rate6","abi_rate7","abi_rate8","abi_rate9"]])
 train["part_rate_curt"] = np.nanmax(np.identity(8)[train.part][:,1:]*all_part_rate, axis=1)
 train["part_ratec_curt"] = np.nanmax(np.identity(8)[train.part][:,1:]*all_part_ratec, axis=1)
 train["abi_rate_curt"] = np.nanmax(np.identity(10)[train.question_rate_class_exp][:,[0,3,4,5,6,7,8,9]]*all_abi_rate, axis=1)
 train["part_ratec_mean"] = np.nan_to_num(all_part_ratec ).sum(1)/ 7
del all_part_rate, all_part_ratec, all_abi_rate
gc.collect()
train["harmonic_ar_prc"] = 2 *(train.ans_rate * train.part_rate_curt)/(train.ans_rate + train.part_rate_curt)
 train["harmonic_arc_qr"] = 2 *(train.ans_rate_class * train.question_rate_exp)/(train.ans_rate_class + train.question_rate_exp)
 train["harmonic_arc_prcc"] = 2 *(train.ans_rate_class * train.part_ratec_curt)/(train.ans_rate_class + train.part_ratec_curt)
 train["harmonic_prc_pr"] = 2 *(train.part_rate_curt * train.part_rate)/(train.part_rate_curt + train.part_rate)
 train["harmonic_prcc_qr"] = 2 *(train.part_ratec_curt * train.question_rate_exp)/(train.part_ratec_curt + train.question_rate_exp)
 train["harmonic_qr_pr"] = 2 *(train.part_rate * train.question_rate_exp)/(train.part_rate + train.question_rate_exp)
 train["harmonic_abrc_qr"] = 2 *(train.abi_rate_curt * train.question_rate_exp)/(train.abi_rate_curt + train.question_rate_exp)
 train["harmonic_arc_src"] = 2 *(train.ans_rate_class * train.study_rate_class)/(train.ans_rate_class + train.study_rate_class)
 train["harmonic_prcc_src"] = 2 *(train.part_ratec_curt * train.study_rate_class)/(train.part_ratec_curt + train.study_rate_class)
 train["harmonic_src_qr"] = 2 *(train.study_rate_class * train.question_rate_exp)/(train.study_rate_class + train.question_rate_exp)
 train["rate_arc_prcc"] = train.ans_rate_class / train.part_ratec_curt
train["rate_arc_src"] = train.ans_rate_class / train.study_rate_class
train["rate_prcc_src"] = train.part_ratec_curt / train.study_rate_class
train["rate_arc_qr"] = train.ans_rate_class / train.question_rate_exp
train["rate_qr_pr"] = train.part_rate / train.question_rate_exp
train["rate_prcc_prccm"] = train.part_ratec_curt / train.part_ratec_mean
y_scores = vc2.predict_proba(X_test)
 y_scores = y_scores[:,1]<compute_test_metric>
train.fillna(0, inplace=True)
 train.sort_values(by="row_id", inplace=True)
 train.reset_index(drop=True, inplace=True )
false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, y_scores )<compute_test_metric>
class BaseModel(object):

 def __init__(self, train_df, test_df, target, features,
valid_df=None, tune_df=None, categoricals=[], alpha=0,
n_splits=3, cv_method="KFold", group=None,
task="regression", params=None, parameter_tuning=False,
tuning_type="optuna_tuner", seed=42, verbose=True):
self.train_df = train_df
self.test_df = test_df
self.valid_df = valid_df
self.tune_df = tune_df
self.target = target
self.features = features
self.n_splits = n_splits
self.categoricals = categoricals
self.alpha = alpha
self.cv_method = cv_method
self.group = group
self.task = task
self.parameter_tuning = parameter_tuning
self.seed = seed
self.cv = self.get_cv()
self.verbose = verbose
self.tuning_type = tuning_type
if params is None:
self.params = self.get_params()
else:
self.params = params
self.y_pred, self.y_valid, self.score, self.models, self.oof, self.y_val, self.fi_df = self.fit()
def train_model(self, train_set, val_set):
raise NotImplementedError
def get_params(self):
raise NotImplementedError
def convert_dataset(self, x_train, y_train, x_val, y_val):
raise NotImplementedError
def calc_metric(self, y_true, y_pred):
if self.task in("multiclass","nn_multiclass"):
preds = np.argmax(y_pred, axis=1)if y_true.shape != y_pred.shape else y_pred
return f1_score(y_true, preds, average='macro')
 elif self.task == "binary":
return roc_auc_score(y_true, y_pred, average='macro')
 elif self.task in("regression","quantile"):
return np.sqrt(mean_squared_error(y_true, y_pred))
def get_cv(self):
if self.cv_method == "KFold":
cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)
 return cv.split(self.train_df)
 if self.cv_method == "StratifiedKFold":
cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)
 return cv.split(self.train_df, self.train_df[self.target])
 if self.cv_method == "TimeSeriesSplit":
cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)
 return cv.split(self.train_df)
 if self.cv_method == "GroupKFold":
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 cv = GroupKFold(n_splits=self.n_splits)
 return cv.split(self.train_df[self.features+self.categoricals], self.train_df[self.target], self.train_df[self.group])
 if self.cv_method == "GroupShuffleSplit":
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 cv = GroupShuffleSplit(n_splits=self.n_splits, random_state=self.seed)
 return cv.split(self.train_df[self.features+self.categoricals], self.train_df[self.target], self.train_df[self.group])
 def fit(self):
y_vals = np.zeros(( self.train_df.shape[0],))
if self.task in("multiclass","nn_multiclass"):
col_len = self.train_df[self.target].nunique()
else:
col_len = 1
oof_pred = np.zeros(( self.train_df.shape[0], col_len))
y_pred = np.zeros(( self.test_df.shape[0], col_len))
y_valid = np.zeros(( self.valid_df.shape[0], col_len)) if self.valid_df is not None else None
models = []
if self.group is not None:
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 fi = np.zeros(( self.n_splits, len(self.features+self.categoricals)))
 if y_valid is not None:
x_valid = self.valid_df[self.features+self.categoricals].copy()
del self.valid_df
gc.collect()
x_test = self.test_df[self.features+self.categoricals]
for fold,(train_idx, val_idx)in enumerate(self.cv):
x_train = self.train_df.iloc[train_idx, :][self.features+self.categoricals]
y_train = self.train_df.iloc[train_idx, :][self.target]
x_val = self.train_df.iloc[val_idx, :][self.features+self.categoricals]
y_val = self.train_df.iloc[val_idx, :][self.target]
train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)
 del x_train, y_train
gc.collect()
model, importance = self.train_model(train_set, val_set)
 fi[fold, :] = importance
y_vals[val_idx] = y_val
oof_pred[val_idx] = model.predict(x_val, num_iteration=model.best_iteration ).reshape(oof_pred[val_idx].shape)
 if y_valid is not None:
y_valid += model.predict(x_valid, num_iteration=model.best_iteration ).reshape(y_valid.shape)/ self.n_splits
y_pred += model.predict(x_test, num_iteration=model.best_iteration ).reshape(y_pred.shape)/ self.n_splits
print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))
 models.append(model)
 del train_set, val_set, x_val, y_val
gc.collect()
fi_df = pd.DataFrame()
for n in np.arange(self.n_splits):
tmp = pd.DataFrame()
tmp["features"] = self.features+self.categoricals
tmp["importance"] = fi[n, :]
tmp["fold"] = n
fi_df = pd.concat([fi_df, tmp], ignore_index=True)
 gfi = fi_df[["features", "importance"]].groupby(["features"] ).mean().reset_index()
fi_df = fi_df.merge(gfi, on="features", how="left", suffixes=('', '_mean'))
loss_score = self.calc_metric(y_vals, oof_pred)
 print('Our oof loss score is: ', loss_score)
 return y_pred, y_valid, loss_score, models, oof_pred, y_vals, fi_df
def plot_feature_importance(self, rank_range=[1, 100]):
fig, ax = plt.subplots(1, 1, figsize=(16, 12))
sorted_df = self.fi_df.sort_values(by="importance_mean", ascending=False ).reset_index()
sns.barplot(data=sorted_df.iloc[self.n_splits*(rank_range[0]-1): self.n_splits*rank_range[1]],
x="importance", y="features", orient='h')
 ax.set_xlabel("feature importance")
 ax.spines['top'].set_visible(False)
 ax.spines['right'].set_visible(False)
 return sorted_df
class LgbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
model = lgb.train(self.params, train_set, num_boost_round=3000,
valid_sets=[train_set, val_set], verbose_eval=verbosity)
 fi = model.feature_importance(importance_type="gain")
 return model, fi
def convert_dataset(self, x_train, y_train, x_val=None, y_val=None):
train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)
 if x_val is not None:
val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)
 return train_set, val_set
return train_set
def get_params(self):
params = {
'boosting_type' : "gbdt",
'objective' : self.task,
"subsample" : 0.4,
"subsample_freq" : 1,
'max_depth' : 4,
'min_data_in_leaf' : 50,
'learning_rate' : 0.05,
'early_stopping_rounds' : 100,
'bagging_seed' : 11,
'random_state' : 42,
'verbosity' : -1
}
if self.task == "regression":
params["metric"] = "regression_l2"
elif self.task == "quantile":
params["metric"] = "quantile"
params["alpha"] = self.alpha
elif self.task == "binary":
params["metric"] = "auc"
elif self.task == "multiclass":
params["metric"] = "multi_logloss"
params["num_class"] = len(self.train_df[self.target].unique())
 if self.parameter_tuning:
def get_dataset() :
if self.tune_df is not None:
tune = self.tune_df.copy()
else:
tune = self.train_df.copy()
if self.group is not None:
train_num = int(tune.shape[0] * 0.7)
 test_num = tune.shape[0] - train_num
train_x = tune.head(train_num)[self.features+self.categoricals].reset_index(drop=True)
 train_y = tune.head(train_num)[[self.target]].reset_index(drop=True)
 test_x = tune.tail(test_num)[self.features+self.categoricals].reset_index(drop=True)
 test_y = tune.tail(test_num)[[self.target]].reset_index(drop=True)
 else:
train_x, test_x, train_y, test_y = train_test_split(tune[self.features+self.categoricals],
tune[self.target], test_size=0.3, random_state=self.seed)
 if self.categoricals != []:
dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)
 dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)
 else:
dtrain = lgb.Dataset(train_x, train_y)
 dtest = lgb.Dataset(test_x, test_y)
 return dtrain, dtest, test_x, test_y
def objective(trial):
dtrain, dtest, test_x, test_y = get_dataset()
hyperparams = {'max_depth' : trial.suggest_int('max_depth', 4, 16),
'max_bin' : trial.suggest_int('max_bin', 200, 1000),
'num_leaves' : trial.suggest_int('num_leaves', 200, 1000),
'min_data_in_leaf' : trial.suggest_int('min_data_in_leaf', 1, 300),
'min_child_samples' : trial.suggest_int('min_child_samples', 5, 100),
'feature_fraction' : trial.suggest_uniform('feature_fraction', 0.2, 1.0),
'bagging_fraction' : trial.suggest_uniform('bagging_fraction', 0.2, 1.0),
'bagging_freq' : trial.suggest_int('bagging_freq', 0, 7),
'lambda_l1' : trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2' : trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'early_stopping_rounds' : 150}
params.update(hyperparams)
 verbosity = 100 if self.verbose else 0
model = lgb.train(params, dtrain, valid_sets=dtest,
num_boost_round=1000, verbose_eval=verbosity)
 pred = model.predict(test_x)
 return self.calc_metric(test_y, pred)
 if self.tuning_type == "optuna_tuner":
dtrain, dtest, _, _ = get_dataset()
verbosity = 100 if self.verbose else 0
tuner_params = {"objective": params["objective"],
"metric" : params["metric"]}
model = optuna_lgb.train(tuner_params, dtrain, valid_sets=dtest,
num_boost_round=1000, early_stopping_rounds=50,
verbose_eval=verbosity)
 print('Best params:')
 for key, value in model.params.items() :
print(' {}: {}'.format(key, value))
params = model.params
params["feature_pre_filter"] = True
else:
study = optuna.create_study(direction='maximize')
 study.optimize(objective, n_trials=40)
 print('Number of finished trials: {}'.format(len(study.trials)))
 trial = study.best_trial
print('Best trial:')
 print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
params.update(trial.params)
 plot_optimization_history(study)
 return params
auroc = roc_auc_score(y_test, y_scores)
 print("ROC-AUC Score:", auroc )<save_to_csv>
not_use_cols = ["row_id","user_id","user_answer","prior_question_had_explanation",
"question_rate_class","question_rate_class_exp",'part_rate_curt','part_ratec_curt',"abi_rate_curt",
"cv_group","max_time_stamp","rand_time_stamp","viretual_time_stamp"]
not_use_cols = not_use_cols + ans_part_cols + num_part_cols + part_mistake_cols + tags_cols
pred = vc2.predict(test ).astype(int)
 target = pd.Series(pred, name='Survived')
 output = pd.concat({'PassengerId':Id, 'Survived':target}
,axis='columns')
 output.to_csv('submission.csv', index=False, header=True )<import_modules>
df_corr[["answered_correctly"]].abs().sort_values("answered_correctly")[["answered_correctly"]]
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.impute import KNNImputer
from imblearn.over_sampling import SMOTE
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import BaggingClassifier
from lightgbm import LGBMClassifier 
<load_from_csv>
not_use_cols += ["prior_question_elapsed_time",'bundle_id',"bundle_skew","num_solving_question",
"num_intention","num_lec","num_concept","num_concept","num_tag",'deviation_m',"LorR",'part']
categoricals = []
keep_cols = ["user_id"]
target = "answered_correctly"
group = "cv_group"
features = [c for c in train.columns if c not in not_use_cols+categoricals+[target]]
print(len(features+categoricals))
print("Numeric features")
 display(sorted(features))
print("Categorical features")
 print(sorted(categoricals))
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )<drop_column>
user_ids = train.user_id.unique()
random.shuffle(user_ids)
 use_features = [target, group] + keep_cols + features + categoricals
trains = []
for i in range(1, MODEL_NUM+1):
print("Model %s will be from 0 to %s in cv_group" %(i, CV-1))
train_data = train.loc[train.cv_group.isin(np.arange(CV)) , use_features].copy()
train_data.reset_index(drop=True, inplace=True)
 trains.append(train_data)
 test_data = train.loc[train.cv_group==CV, use_features].copy()
test_data.reset_index(drop=True, inplace=True)
 if TUNE:
tune_data = train.loc[train.cv_group==CV+1, use_features].copy()
tune_data.reset_index(drop=True, inplace=True)
 else:
tune_data = pd.DataFrame()
del train, train_data
gc.collect()
train.drop('PassengerId',axis=1,inplace=True)
 ID = test['PassengerId']
test.drop('PassengerId',axis=1,inplace=True )<sort_values>
print("Training data")
 for i, train_data in enumerate(trains):
print("No.", i+1)
 print(train_data.shape, train_data.cv_group.unique() , train_data.user_id.nunique())
 display(train_data.head())
 print("Test data")
 print(test_data.shape, test_data.cv_group.unique() , test_data.user_id.nunique())
 display(test_data.head())
 if TUNE:
print("Tune data")
 print(tune_data.shape, tune_data.cv_group.unique() , tune_data.user_id.nunique())
 display(tune_data.head())
 del train_data
gc.collect()
def visualNA(df,perc=0):
NAN = [(c, df[c].isna().mean() *100)for c in df]
NAN = pd.DataFrame(NAN, columns=["column_name", "percentage"])
 NAN = NAN[NAN.percentage > perc]
print(NAN.sort_values("percentage", ascending=False))<count_values>
params1 = {
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
'learning_rate': 0.1,
"max_depth": 4,
"max_bin": 853,
"num_leaves": 492,
"min_data_in_leaf": 229,
"min_child_samples": 55,
"feature_fraction": 0.7894275993264507,
"bagging_fraction": 0.8305608180096481,
"bagging_freq": 6,
"lambda_l1": 0.00010208001912287773,
"lambda_l2": 9.315928833771094,
'early_stopping_rounds' : 100,
'random_state': 42,
"bagging_seed": 11,
"verbosity": -1}
params2 = {
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
'learning_rate': 0.05,
'max_depth': 4,
'max_bin': 700,
'num_leaves': 350,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.58,
'bagging_fraction': 0.58,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'early_stopping_rounds' : 100,
'random_state': 42,
"bagging_seed": 11,
"verbosity": -1}
params3 = {
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
'learning_rate': 0.05,
"max_depth": 9,
"max_bin": 872,
"num_leaves": 743,
"min_data_in_leaf": 274,
"min_child_samples": 27,
"feature_fraction": 0.5922083852690104,
"bagging_fraction": 0.7638997668812837,
"bagging_freq": 5,
"lambda_l1": 4.603407547894731e-07,
"lambda_l2": 4.833680631174746e-07,
'early_stopping_rounds' : 200,
'random_state': 42,
"bagging_seed": 11,
"verbosity": -1}
params4 = {
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
'learning_rate': 0.05,
"max_depth": 8,
"max_bin": 990,
"num_leaves": 885,
"min_data_in_leaf": 214,
"min_child_samples": 89,
"feature_fraction": 0.4990107527526446,
"bagging_fraction": 0.9513631160877628,
"bagging_freq": 7,
"lambda_l1": 4.856465786762882e-05,
"lambda_l2": 1.6804951000850403e-08,
'early_stopping_rounds' : 200,
'random_state': 42,
"bagging_seed": 11,
"verbosity": -1}
if TUNE:
params = [None] * MODEL_NUM
elif MODEL_NUM == 1:
params = [params3]
else:
params = [params3, params4]
tuning_type = "optuna"
train['Embarked'].value_counts()<data_type_conversions>
%%time
lgbms = []
for idx, train_data in enumerate(trains):
if DEBUG:
lgbm = LgbModel(train_data.sample(n=30000 ).reset_index(drop=True), test_data,
target, features, categoricals=categoricals,
task="binary", params=None, parameter_tuning=TUNE, tuning_type=tuning_type,
tune_df=tune_data, cv_method="GroupKFold", n_splits=4, group=group, verbose=False)
 else:
lgbm = LgbModel(train_data, test_data,
target, features, categoricals=categoricals,
task="binary", params=params[idx], parameter_tuning=TUNE, tuning_type=tuning_type,
tune_df=tune_data, cv_method="GroupKFold", n_splits=4, group=group, verbose=False)
 lgbms.append(lgbm)
 del lgbm
gc.collect()
train['Embarked'].fillna(value='S',inplace=True )<feature_engineering>
for lgbm in lgbms:
score = roc_auc_score(test_data.answered_correctly, lgbm.y_pred)
 fpr, tpr, thresholds = roc_curve(test_data.answered_correctly, lgbm.y_pred)
 print(score)
 plt.plot(fpr, tpr )
train[['Ticket_Class','Ticket_Number']]=train['Ticket'].str.split(" ",expand=True,n=1)
 train['Ticket_Class'],train['Ticket_Number'] = zip(*train[['Ticket_Class','Ticket_Number']].apply(( lambda x :(None,x['Ticket_Class'])if x['Ticket_Number'] is None else(x['Ticket_Class'],x['Ticket_Number'])) ,axis=1))<feature_engineering>
show_cols = ["correct","answered_correctly","predict","predict_prob","user_id"]+features
test_data["predict_prob"] = lgbm.y_pred
test_data["predict"] = test_data.predict_prob.apply(lambda x: 1 if x>0.5 else 0)
 test_data["correct"] = test_data[["answered_correctly","predict"]].apply(lambda x: 1 if x[0]==x[1] else 0, axis=1 )
train['Title'] = train['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]<data_type_conversions>
test_data.sort_values("predict_prob", ascending=False, inplace=True)
 test_data[show_cols].head()
train['Cabin'].fillna(value='0',inplace=True)
 train['Ticket_Class'].fillna(value='0',inplace=True )<categorify>
df_res_sorted = test_data.loc[(test_data.correct==0), show_cols].sort_values("predict_prob", ascending=False)
 df_res_sorted.head(20 )
les = LabelEncoder()
train['Sex'] = les.fit_transform(train['Sex'])
 let = LabelEncoder()
train['Title'] = let.fit_transform(train['Title'].astype(str))
letn = LabelEncoder()
train['Ticket_Number'] = letn.fit_transform(train['Ticket_Number'].astype(str))
letc = LabelEncoder()
train['Ticket_Class'] = letc.fit_transform(train['Ticket_Class'].astype(str))
lec = LabelEncoder()
train['Cabin'] = lec.fit_transform(train['Cabin'].astype(str))
lee = LabelEncoder()
train['Embarked'] = lee.fit_transform(train['Embarked'].astype(str)) 
<correct_missing_values>
for u in df_res_sorted.user_id.unique() [:3]:
test_data.loc[test_data.user_id==u, show_cols].to_csv("debug_"+str(u)+".csv", index=False )
train['Cabin'].replace(0,np.NaN,inplace=True )<prepare_x_and_y>
models = []
for lgbm in lgbms:
models += lgbm.models
print(len(models))
del lgbms, train_data, tune_data
gc.collect()
X = train[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch',
'Fare', 'Cabin', 'Embarked', 'Ticket_Class','Ticket_Number','Title']]
y = train['Survived']<feature_engineering>
def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask )
imputer = KNNImputer(n_neighbors=5)
 X = imputer.fit_transform(X )<train_model>
class FFN(nn.Module):
def __init__(self, state_size=200, forward_expansion=1, bn_size=(180-1), dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads=8, dropout=0.1, forward_expansion=1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion=forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.dropout(self.layer_normal_2(x + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=180, embed_dim=128, dropout=0.1, forward_expansion=1, num_layers=1, heads=8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill + 1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=180, embed_dim=128, dropout=0.1, forward_expansion=1, enc_layers=1, heads=8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim+1, 1)
 def forward(self, x1, x2, question_ids):
device = x1.device
x, att_weight = self.encoder(x1, question_ids)
 x = torch.cat([x, x2.unsqueeze(2)], dim=2 ).to(device)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
sm = SMOTE(random_state=42)
 X_res, y_res = sm.fit_resample(X, y )<train_model>
%%time
device = torch.device("cpu")
 skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-own/group.pkl.zip")
 lags = joblib.load("/kaggle/input/riiid-sakt-model-dataset-own/lags.pkl.zip")
 nn_model = SAKTModel(n_skill, max_seq=180, embed_dim=128, forward_expansion=1, enc_layers=1, heads=8, dropout=0.1)
 nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-own/sakt_model.pt", map_location='cpu'))
nn_model.to(device)
 nn_model.eval()
model_bc = BaggingClassifier(RandomForestClassifier(n_estimators=300, max_depth=5, random_state=42),n_estimators=100)
 model_bc.fit(X_res,y_res )<compute_test_metric>
def insert_new_data_to_hist(ary, ins):
ary = ary.copy()
ary = np.vstack([ary, ins])
 ary = ary[np.argsort(ary[:,0])]
return ary
def groupby_with_numpy(ary, t_idxs, qucnt):
ary = ary.copy()
g = sum([[i+1 for _ in range(c)] for i, c in enumerate(qucnt)], [])
 r = np.zeros(( qucnt.shape[0], t_idxs.shape[0]))
for i, t in enumerate(t_idxs):
r[:,i] = np.bincount(g, ary[:,t])[1:]
return r
def add_mistake_feat(df, t_col, n_col):
df = df.copy()
if df[n_col].min() == 0:
df[t_col] = df[n_col].apply(lambda x: 0.6*x**(-0.2)if x!=0 else 0.6)
 else:
df[t_col] = 0.6*df[n_col]**(-0.2)
 return df
def get_num_etimes(df):
num_etimes = []
for u, et in np.array(df[["user_id","prior_question_elapsed_time"]]):
if u in d.keys() :
etime_mean = d_etime[d[u][-1, 1]]
et_norm = et /(1000*3600)
 num_etimes.append(et_norm - etime_mean)
 else:
num_etimes.append(0)
 return num_etimes
def add_to_timedict(df):
for u, t in np.array(df[["user_id","timestamp"]]):
if u in d_timestamp.keys() :
d_timestamp[u] = np.unique(np.hstack([d_timestamp[u], np.array([t])]))
else:
d_timestamp[u] = np.array([t])
 def add_to_history(df, new_row_id):
for r in np.array(df[["user_id","question_id","part","tag1"]]):
if r[0] in d.keys() :
d[r[0]] = np.vstack([d[r[0]], np.hstack([np.array(new_row_id), r[1:], np.array(0)])])
 else:
d[r[0]] = np.array([np.hstack([np.array(new_row_id), r[1:], np.array(0)])])
 def upd_history_ans(df):
for r in np.array(df[["user_id","answered_correctly"]]):
if r[0] in d.keys() and r[1] == 1:
d[r[0]][-1,-1] = r[1]
feature_importances = np.mean([
tree.feature_importances_ for tree in model_bc.estimators_
], axis=0 )<feature_engineering>
df_lecs.drop(["part","starter"], axis=1, inplace=True)
 df_qs = df_qs.merge(df_question_rate, on=["question_id","bundle_id"] ).merge(df_part_rate, on="part" ).merge(df_etime_rate, on="question_id")
 df_qs.sort_values("question_id", inplace=True)
 df_qs.reset_index(drop=True, inplace=True)
 df_qs["LorR"] = 0
df_qs.loc[df_qs.part.isin([5,6,7]), "LorR"] = 1
print(df_lecs.shape, df_qs.shape)
 display(df_lecs.head(2))
display(df_qs.head(2))
test[['Ticket_Class','Ticket_Number']]=test['Ticket'].str.split(" ",expand=True,n=1)
 test['Ticket_Class'],test['Ticket_Number'] = zip(*test[['Ticket_Class','Ticket_Number']].apply(( lambda x :(None,x['Ticket_Class'])if x['Ticket_Number'] is None else(x['Ticket_Class'],x['Ticket_Number'])) ,axis=1))<feature_engineering>
sort_cols = ['user_id',"num_exp",'num_ans','num_devp','num_devm',"num_etime",'num_que'] + part_cols + abi_cols
ar_hist = np.array(df_history[sort_cols])
 ar_qs = np.array(df_qs)[:,0]
ar_ans_class = np.array(df_ans_class)[:,1]
test['Title'] = test['Name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]<feature_engineering>
iter_test = env.iter_test()
for i,lechange in [('Ticket_Class',letc),('Cabin',lec),('Embarked',lee), 
('Ticket_Number',letn),('Title',let)]:
test[i] = test[i].map(lambda s: '<unknown>' if s not in lechange.classes_ else s)
 le_classes = lechange.classes_.tolist()
le_classes.insert(len(le_classes), '<unknown>')
 lechange.classes_ = le_classes<data_type_conversions>
t_cols = ["ans_rate_class","study_rate_class"] + [f"part{p}_rate_class" for p in range(1,8)]
n_cols = ["num_que", "num_que"] + num_part_cols
r_cols = ['row_id','timestamp','user_id','question_id','content_type_id','task_container_id','prior_question_elapsed_time','prior_question_had_explanation']
df_test = pd.DataFrame()
<drop_column><EOS> 
%%time
for idx,(test_df, sample_prediction_df)in enumerate(iter_test):
test_df.prior_question_had_explanation =(test_df.prior_question_had_explanation.fillna(False)* 1 ).astype(int)
 test_df.prior_question_elapsed_time = test_df.prior_question_elapsed_time.fillna(0 ).astype(int)
 prior_ans = eval(test_df.prior_group_answers_correct.values[0])
 test_ary = np.array(test_df)[:,:8]
lec_ary = test_ary[test_ary[:,4]==1, 2:4]
test_ary = test_ary[test_ary[:,4]==0, :]
qusers, qucnt = np.unique(np.sort(test_ary[:,2]), return_counts=True)
 qusers = qusers.astype("int32")
 existing_qusers_bol = np.isin(ar_hist[:,0], qusers)
 not_existing_qusers = qusers[~np.isin(qusers, ar_hist[existing_qusers_bol, 0])]
questions = np.unique(test_ary[:,3].astype("int32"))
group_num = test_df.index[:test_ary.shape[0]]
if len(prior_ans)!= 0 and prior_ans is not None and prior_ans is not np.nan:
prior_ans = np.array([a for a in prior_ans if a != -1])
 if 0 < len(prior_ans)and len(prior_ans)== len(prior_test_user_ids)== len(prior_test_devp):
df_prev['answered_correctly'] = prior_ans
upd_history_ans(df_prev)
 if ENSEMBLE:
prev_group = df_prev.groupby('user_id' ).apply(lambda x:(x['question_id'].values,
x['answered_correctly'].values,
x["lagtime_mean"].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-180:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-180:])
 lags [prev_user_id] =(np.append(lags [prev_user_id], prev_group[prev_user_id][2])[-180:])
 else:
group[prev_user_id] =(prev_group[prev_user_id][0], prev_group[prev_user_id][1])
 lags [prev_user_id] = prev_group[prev_user_id][2]
ans_ary = np.array([prior_test_user_ids,
prior_ans,
prior_test_devp.fillna(9),
prior_test_devm,
prior_test_etime] ).T
ans_ary = ans_ary[ans_ary[:,2]!=9]
if 0 < ans_ary.shape[0]:
ans_ary[:,2] *= ans_ary[:,1]
ans_ary[:,3] = ans_ary[:,2] + np.where(ans_ary[:,1]==0, -1, 0)* ans_ary[:,3]
part_ans_ary = prior_test_part * ans_ary[:,1].reshape(-1,1)
 abi_ans_ary = prior_test_qc * ans_ary[:,1].reshape(-1,1)
 ans_ary = np.hstack([ans_ary,
np.ones(ans_ary.shape[0] ).reshape(-1,1),
part_ans_ary,
prior_test_part,
abi_ans_ary,
prior_test_qc])
 ans_ary = ans_ary[np.argsort(ans_ary[:,0])]
if len(prior_test_user_ids)!= len(np.unique(prior_test_user_ids)) :
_, qacnt = np.unique(ans_ary[:,0], return_counts=True)
 ans_ary = groupby_with_numpy(ans_ary, np.arange(1, ans_ary.shape[1]), qacnt)
 else:
ans_ary = ans_ary[:,1:]
ar_hist[prior_hist_equb, 2:] += ans_ary
if 0 < not_existing_qusers.shape[0]:
ins_exp = np.hstack([not_existing_qusers.reshape(-1,1),
np.zeros(( not_existing_qusers.shape[0], ar_hist.shape[1]-1)) ])
 ar_hist = insert_new_data_to_hist(ar_hist, ins_exp)
 existing_qusers_bol = np.isin(ar_hist[:,0], qusers)
 if 0 < test_ary.shape[0]:
ary = test_ary[:,[2,7]].copy()
ary = ary.astype(int)
 ary = ary[np.argsort(ary[:,0])]
ar_hist[existing_qusers_bol, 1] += groupby_with_numpy(ary, np.array([1]), qucnt ).flatten()
df_hist = pd.DataFrame(ar_hist[existing_qusers_bol], columns=sort_cols)
 df_test = pd.DataFrame(test_ary, columns=r_cols)
 df_test = df_test.merge(df_hist, on="user_id", sort=False)
 df_test = df_test.merge(df_qs[np.isin(ar_qs, questions)], on=["question_id","prior_question_had_explanation"], sort=False)
 df_test["ans_rate"] =(df_test.num_ans / df_test.num_que ).fillna(0)
 df_test["devp_rate"] = df_test.num_devp / df_test.num_que
df_test["devm_rate"] = df_test.num_devm / df_test.num_que
df_test["etime_rate"] = df_test.num_etime / df_test.num_que
ar_study_rate = np.array(df_test.num_exp / df_test.num_que)
 ar_study_rate[np.isnan(ar_study_rate)] = 0
ar_study_rate[np.isinf(ar_study_rate)] = 0
ar_study_rate[1 < ar_study_rate] = 1
df_test["study_rate"] = ar_study_rate
df_class = df_ans_class[np.isin(ar_ans_class, questions)].copy()
df_test = cut_and_merge_with_class(df_test, df_class, "ans_rate", "ans_rate_class")
 df_test = cut_and_merge_with_class(df_test, df_class, "study_rate", "study_rate_class")
 df_test = add_mistake_feat(df_test, "prob_mistake_que", "num_que")
 for p in range(0,10):
if p in [0,3,4,5,6,7,8,9]:
df_test[f"abi_rate{p}"] =(df_test[f"ans_abi{p}"] / df_test[f"num_abi{p}"] ).fillna(0)
 if p in [1,2,3,4,5,6,7]:
df_test[f"part{p}_rate"] =(df_test[f"ans_part{p}"] / df_test[f"num_part{p}"] ).fillna(0)
 df_test = add_mistake_feat(df_test, f"prob_mistake_part{p}", f"num_part{p}")
 df_test = cut_and_merge_with_class(df_test, df_class, f"part{p}_rate", f"part{p}_rate_class")
 for t, n in zip(t_cols, n_cols):
if df_test[n].min() < 5:
df_test = change_rate_with_num(df_test, t, n, method="respectively")
 df_test = change_rate_with_num(df_test, t, n, method="bundle")
 df_test["lagtime1"] =(df_test[["user_id","timestamp"]].apply(lambda x: get_lagtime(x[0], x[1], -1), axis=1)/(1000*3600)).astype("float32")
 df_test["lagtime2"] =(df_test[["user_id","timestamp"]].apply(lambda x: get_lagtime(x[0], x[1], -2), axis=1)/(1000*3600)).astype("float32")
 df_test["lagtime3"] =(df_test[["user_id","timestamp"]].apply(lambda x: get_lagtime(x[0], x[1], -3), axis=1)/(1000*3600)).astype("float32")
 df_test["lagtime_mean"] =(df_test.lagtime1 + df_test.lagtime2 + df_test.lagtime3)/ 3
df_test["timestamp"] =(df_test.timestamp /(1000*3600)).astype("float32")
 df_test["lagtime1"] =(df_test.timestamp - df_test.lagtime1 ).astype("float32")
 df_test["lagtime2"] =(df_test.timestamp - df_test.lagtime2 ).astype("float32")
 df_test["lagtime3"] =(df_test.timestamp - df_test.lagtime3 ).astype("float32")
 df_test["lagtime_mean"] = np.log(( df_test.timestamp - df_test.lagtime_mean)+ 1 ).astype("float32")
 df_test["noplay_days"] =(df_test[["user_id","timestamp"]].apply(lambda x: get_noplay_days(x[0], x[1]), axis=1)).astype("float32")
 df_test["elapsed_days"] =(df_test.timestamp / 24 ).astype("int32")
 df_test["noplay_rate"] =(df_test.noplay_days / df_test.num_que ).astype("float32")
 df_test["timestamp_rate"] =(df_test.timestamp / df_test.num_que ).astype("float32")
 df_test["elapsed_rate"] =(df_test.elapsed_days / df_test.num_que ).astype("float32")
 df_test["attempt_que"] = df_test[["user_id","question_id"]].apply(lambda x: get_attempt_que(x[0], 101230332+idx, x[1]), axis=1)
 df_test["attempt_tags"] = df_test[["user_id","tag1"]].apply(lambda x: get_attempt_tags(x[0], 101230332+idx, x[1]), axis=1)
 df_test["attempt_curtar"] = df_test[["user_id"]].apply(lambda x: get_attempt_curt_ans_rate(x[0], 101230332+idx), axis=1)
 all_prob_mistake = np.array(df_test[part_mistake_cols])
 all_part_rate = np.array(df_test[["part1_rate","part2_rate","part3_rate","part4_rate","part5_rate","part6_rate","part7_rate"]])
 all_part_ratec = np.array(df_test[["part1_rate_class","part2_rate_class","part3_rate_class","part4_rate_class","part5_rate_class","part6_rate_class","part7_rate_class"]])
 all_abi_rate = np.array(df_test[["abi_rate0","abi_rate3","abi_rate4","abi_rate5","abi_rate6","abi_rate7","abi_rate8","abi_rate9"]])
 one_hot_part = np.identity(8)[df_test.part][:,1:]
one_hot_ability = np.identity(10)[df_test.question_rate_class_exp][:,[0,3,4,5,6,7,8,9]]
df_test["prob_mistake_part_curt"] = np.nanmax(one_hot_part*all_prob_mistake, axis=1)
 df_test["part_rate_curt"] = np.nanmax(one_hot_part*all_part_rate, axis=1)
 df_test["part_ratec_curt"] = np.nanmax(one_hot_part*all_part_ratec, axis=1)
 df_test["abi_rate_curt"] = np.nanmax(one_hot_ability*all_abi_rate, axis=1)
 df_test["part_ratec_mean"] = np.nan_to_num(all_part_ratec ).sum(1)/ 7
df_test["harmonic_ar_prc"] = 2 *(df_test.ans_rate * df_test.part_rate_curt)/(df_test.ans_rate + df_test.part_rate_curt)
 df_test["harmonic_arc_qr"] = 2 *(df_test.ans_rate_class * df_test.question_rate_exp)/(df_test.ans_rate_class + df_test.question_rate_exp)
 df_test["harmonic_arc_prcc"] = 2 *(df_test.ans_rate_class * df_test.part_ratec_curt)/(df_test.ans_rate_class + df_test.part_ratec_curt)
 df_test["harmonic_prc_pr"] = 2 *(df_test.part_rate_curt * df_test.part_rate)/(df_test.part_rate_curt + df_test.part_rate)
 df_test["harmonic_prcc_qr"] = 2 *(df_test.part_ratec_curt * df_test.question_rate_exp)/(df_test.part_ratec_curt + df_test.question_rate_exp)
 df_test["harmonic_qr_pr"] = 2 *(df_test.part_rate * df_test.question_rate_exp)/(df_test.part_rate + df_test.question_rate_exp)
 df_test["harmonic_abrc_qr"] = 2 *(df_test.abi_rate_curt * df_test.question_rate_exp)/(df_test.abi_rate_curt + df_test.question_rate_exp)
 df_test["harmonic_arc_src"] = 2 *(df_test.ans_rate_class * df_test.study_rate_class)/(df_test.ans_rate_class + df_test.study_rate_class)
 df_test["harmonic_prcc_src"] = 2 *(df_test.part_ratec_curt * df_test.study_rate_class)/(df_test.part_ratec_curt + df_test.study_rate_class)
 df_test["harmonic_src_qr"] = 2 *(df_test.study_rate_class * df_test.question_rate_exp)/(df_test.study_rate_class + df_test.question_rate_exp)
 df_test["rate_arc_prcc"] = df_test.ans_rate_class / df_test.part_ratec_curt
df_test["rate_arc_src"] = df_test.ans_rate_class / df_test.study_rate_class
df_test["rate_prcc_src"] = df_test.part_ratec_curt / df_test.study_rate_class
df_test["rate_arc_qr"] = df_test.ans_rate_class / df_test.question_rate_exp
df_test["rate_qr_pr"] = df_test.part_rate / df_test.question_rate_exp
df_test["rate_prcc_prccm"] = df_test.part_ratec_curt / df_test.part_ratec_mean
prior_hist_equb = existing_qusers_bol
prior_test_user_ids = np.array(df_test.user_id ).astype("int32")
 prior_test_part = one_hot_part
prior_test_qc = one_hot_ability
prior_test_devp = df_test.deviation_p
prior_test_devm = df_test.deviation_m
prior_test_etime = get_num_etimes(df_test)
 df_test.fillna(0, inplace=True)
 df_test.sort_values("row_id", inplace=True)
 df_test.reset_index(drop=True, inplace=True)
 if idx < 1:
display(test_df)
 display(df_test[features+categoricals])
 reults = np.zeros(df_test.shape[0])
 for model in models:
reults += model.predict(df_test[features+categoricals], num_iteration=model.best_iteration)/ len(models)
 if ENSEMBLE:
outs = []
test_dataset = TestDataset(group, lags, df_test[["user_id","question_id","lagtime_mean"]])
 test_dataloader = DataLoader(test_dataset, batch_size=df_test.shape[0], shuffle=False)
 for item in test_dataloader:
x1 = item[0].to(device ).long()
x2 = item[1].to(device ).float()
target_id = item[2].to(device ).long()
with torch.no_grad() :
output, att_weight = nn_model(x1, x2, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 if ENSEMBLE:
df_test['answered_correctly'] = S * np.array(outs)+ L * reults
else:
df_test['answered_correctly'] = reults
add_to_timedict(test_df)
 add_to_history(df_test, 101230332+idx)
 df_prev = df_test[['user_id','question_id',"lagtime_mean"]].copy()
df_test = df_test[['row_id', 'answered_correctly']].copy()
df_test.index = group_num
env.predict(df_test )