Kaggle competition: Tabular Playground Series - Feb 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-feb-2021<data_type_conversions>
%matplotlib inline
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )<categorify>
train = pd.read_csv(DATA / "train.csv")
 test = pd.read_csv(DATA / "test.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 print("train: {}, test: {}, sample sub: {}".format(
 train.shape, test.shape, smpl_sub.shape 
))
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16')
<data_type_conversions>
def rmse(y_true, y_pred):

 return np.sqrt(np.mean(( y_true - y_pred)** 2))
<drop_column>
class TreeModel:

 def __init__(self, model_type: str):
self.model_type = model_type
self.trn_data = None
self.val_data = None
self.model = None
def train(self,
params: dict,
X_train: pd.DataFrame, y_train: np.ndarray,
X_val: pd.DataFrame, y_val: np.ndarray,
train_weight: tp.Optional[np.ndarray] = None,
val_weight: tp.Optional[np.ndarray] = None,
train_params: dict = None,
cat_cols: list = None, 
):
if self.model_type == "lgb":
self.trn_data = lgb.Dataset(X_train, label=y_train, weight=train_weight)
 self.val_data = lgb.Dataset(X_val, label=y_val, weight=val_weight)
 self.model = lgb.train(params=params,
train_set=self.trn_data,
valid_sets=[self.trn_data, self.val_data],
**train_params)
 elif self.model_type == "xgb":
self.trn_data = xgb.DMatrix(X_train, y_train, weight=train_weight)
 self.val_data = xgb.DMatrix(X_val, y_val, weight=val_weight)
 self.model = xgb.train(params=params,
dtrain=self.trn_data,
evals=[(self.trn_data, "train"),(self.val_data, "val")],
**train_params)
 elif self.model_type == "cat":
self.trn_data = Pool(
 X_train, label=y_train, cat_features=cat_cols)
 self.val_data = Pool(
 X_val, label=y_val, cat_features=cat_cols)
 self.model = CatBoost(params)
 self.model.fit(
 self.trn_data, eval_set=[self.val_data], use_best_model=True, **train_params)
 else:
raise NotImplementedError
def predict(self, X: pd.DataFrame):
if self.model_type == "lgb":
return self.model.predict(
 X, num_iteration=self.model.best_iteration)
 elif self.model_type == "xgb":
X_DM = xgb.DMatrix(X)
 return self.model.predict(
 X_DM, ntree_limit=self.model.best_ntree_limit)
 elif self.model_type == "cat":
return self.model.predict(X)
 else:
raise NotImplementedError
@property
def feature_names_(self):
if self.model_type == "lgb":
return self.model.feature_name()
elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).keys())
 elif self.model_type == "cat":
return self.model.feature_names_
else:
raise NotImplementedError
@property
def feature_importances_(self):
if self.model_type == "lgb":
return self.model.feature_importance(importance_type="gain")
 elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).values())
 elif self.model_type == "cat":
return self.model.feature_importances_
else:
raise NotImplementedError
del content_agg
del bundle_agg
del part_agg
gc.collect()<define_variables>
ID_COL = "id"
CAT_COLS= [f"cat{i}" for i in range(10)]
CONT_COLS = [f"cont{i}" for i in range(14)]
TGT_COL = "target"
N_SPLITS = 10
RANDOM_SEED_LIST = [
42,
]
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation',
]
features=list(features_dict.keys())
<init_hyperparams>
use_feat_cols = []
train_feat = train[[ID_COL]].copy()
test_feat = test[[ID_COL]].copy()
flag_lgbm=True
clfs = list()
params = {
'num_leaves': 200,
'max_bin':450,
'feature_fraction': 0.52,
'bagging_fraction': 0.52,
'objective': 'binary',
'learning_rate': 0.05,
"boosting_type": "gbdt",
"metric": 'auc',
}
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df=train_df_clf.sample(frac=0.1)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df)) 
<drop_column>
ord_enc = ce.OrdinalEncoder(cols=CAT_COLS)
 train_cat_feat = ord_enc.fit_transform(train[CAT_COLS])
 test_cat_feat = ord_enc.transform(test[CAT_COLS] )
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
train_feat = pd.concat([
train_feat, train_cat_feat], axis=1)
 test_feat = pd.concat([
test_feat, test_cat_feat], axis=1)
 use_feat_cols.extend(train_cat_feat.columns )
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=5000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect() 
<choose_model_class>
train_cont_feat = train[CONT_COLS]
test_cont_feat = test[CONT_COLS]
MAX_SEQ = 100
class FFN(nn.Module):
def __init__(self, state_size=200):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(state_size, state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=MAX_SEQ, embed_dim=128):
super(SAKTModel, self ).__init__()
self.n_skill = n_skill
self.embed_dim = embed_dim
self.embedding = nn.Embedding(2*n_skill+1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq-1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=8, dropout=0.2)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = x + pos_x
e = self.e_embedding(question_ids)
 x = x.permute(1, 0, 2)
 e = e.permute(1, 0, 2)
 att_mask = future_mask(x.size(0)).to(device)
 att_output, att_weight = self.multi_att(e, x, x, attn_mask=att_mask)
 att_output = self.layer_normal(att_output + e)
 att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.layer_normal(x + att_output)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load("/kaggle/input/riiid-sakt-model-dataset-public/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 nn_model = SAKTModel(n_skill, embed_dim=128)
 try:
nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt"))
except:
nn_model.load_state_dict(torch.load("/kaggle/input/riiid-sakt-model-dataset-public/sakt_model.pt", map_location='cpu'))
nn_model.to(device)
 nn_model.eval() 
<data_type_conversions>
train_feat = pd.concat([
train_feat, train_cont_feat], axis=1)
 test_feat = pd.concat([
test_feat, test_cont_feat], axis=1)
 use_feat_cols.extend(CONT_COLS )
user_sum_dict = user_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_count_dict = user_agg['count'].astype('int16' ).to_dict(defaultdict(int)) 
<data_type_conversions>
def run_train_and_inference(
 X, X_test, y, use_model, model_params, train_params, seed_list, n_splits, cat_cols=None 
):
oof_pred_arr = np.zeros(len(X))
test_pred_arr = np.zeros(len(X_test))
feature_importances = pd.DataFrame()
score_list = []
for seed in seed_list:
if use_model == "cat":
model_params['random_state'] = seed
else:
model_params["seed"] = seed
kf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
 tmp_oof_pred = np.zeros(len(X))
tmp_test_pred = np.zeros(len(X_test))
for fold,(trn_idx, val_idx)in enumerate(kf.split(X, y)) :
print("*" * 100)
 print(f"Seed: {seed} - Fold: {fold}")
 X_trn = X.loc[trn_idx].reset_index(drop=True)
 X_val = X.loc[val_idx].reset_index(drop=True)
 y_trn = y[trn_idx]
y_val = y[val_idx]
model = TreeModel(model_type=use_model)
 with timer(prefix="Model training"):
model.train(
 params=model_params, X_train=X_trn, y_train=y_trn,
X_val=X_val, y_val=y_val, train_params=train_params, cat_cols=cat_cols 
)
 with timer(prefix="Get Feature Importance"):
fi_tmp = pd.DataFrame()
fi_tmp["feature"] = model.feature_names_
fi_tmp["importance"] = model.feature_importances_
fi_tmp["fold"] = fold
fi_tmp["seed"] = seed
feature_importances = feature_importances.append(fi_tmp)
 with timer(prefix="Predict Valid"):
val_pred = model.predict(X_val)
 score = mean_squared_error(y_val, val_pred, squared=False)
 print(f"score: {score:.5f}")
 score_list.append([seed, fold, score])
 tmp_oof_pred[val_idx] = val_pred
tmp_test_pred += model.predict(X_test)
 oof_score = mean_squared_error(y, tmp_oof_pred, squared=False)
 print(f"oof score: {oof_score: 5f}")
 score_list.append([seed, "oof", oof_score])
 oof_pred_arr += tmp_oof_pred
test_pred_arr += tmp_test_pred / n_splits
oof_pred_arr /= len(seed_list)
 test_pred_arr /= len(seed_list)
 oof_score = mean_squared_error(y, oof_pred_arr, squared=False)
 score_list.append(["avg", "oof", oof_score])
 score_df = pd.DataFrame(
 score_list, columns=["seed", "fold", "rmse score"])
 return oof_pred_arr, test_pred_arr, score_df, feature_importances
del user_agg
gc.collect()
task_container_sum_dict = task_container_agg['sum'].astype('int32' ).to_dict(defaultdict(int))
task_container_count_dict = task_container_agg['count'].astype('int32' ).to_dict(defaultdict(int))
task_container_std_dict = task_container_agg['var'].astype('float16' ).to_dict(defaultdict(int))
explanation_sum_dict = explanation_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
explanation_count_dict = explanation_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del task_container_agg
del explanation_agg
gc.collect()<data_type_conversions>
X = train_feat[use_feat_cols]
X_test = test_feat[use_feat_cols]
y = train[TGT_COL].values
print(f"train_feat: {X.shape}, test_feat: {X_test.shape}" )
user_lecture_sum_dict = user_lecture_agg['sum'].astype('int16' ).to_dict(defaultdict(int))
user_lecture_count_dict = user_lecture_agg['count'].astype('int16' ).to_dict(defaultdict(int))
del user_lecture_agg
gc.collect()<categorify>
X_cat = X.copy()
X_cat[CAT_COLS] = train[CAT_COLS]
X_test_cat = X_test.copy()
X_test_cat = test[CAT_COLS]
max_timestamp_u_dict=max_timestamp_u.set_index('user_id' ).to_dict()
max_timestamp_u_dict2=max_timestamp_u2.set_index('user_id' ).to_dict()
max_timestamp_u_dict3=max_timestamp_u3.set_index('user_id' ).to_dict()
user_prior_question_elapsed_time_dict=user_prior_question_elapsed_time.set_index('user_id' ).to_dict()
del max_timestamp_u
del max_timestamp_u2
del max_timestamp_u3
del user_prior_question_elapsed_time
gc.collect()<categorify>
MODEL_PARAMS = {
"lgb": {
"objective": "root_mean_squared_error",
"boosting": "gbdt",
"max_depth": 8,
"learning_rate": 0.005,
"colsample_bytree": 0.2,
"subsample": 0.8,
"subsample_freq": 6,
"reg_alpha": 20,
"min_data_in_leaf": 200,
"n_jobs": 2,
"seed": RANDOM_SEED_LIST[0],
},
"xgb": {
"objective": "reg:squarederror",
"max_depth": 8,
"learning_rate": 0.003,
"colsample_bytree": 0.2,
"subsample": 0.8,
"reg_alpha" : 6,
"min_child_weight": 200,
"n_jobs": 2,
"seed": RANDOM_SEED_LIST[0],
'tree_method': "gpu_hist",
"gpu_id": 0,
},
"cat": {
'loss_function': 'RMSE',
"max_depth": 4,
'learning_rate': 0.03,
"bootstrap_type": 'Poisson',
"subsample": 0.8,
"border_count": 512,
"l2_leaf_reg": 200,
'random_state': RANDOM_SEED_LIST[0],
"thread_count": 2,
"task_type": "GPU",
"devices" : "0",
'num_boost_round': 50000,
}
}
TRAIN_PARAMS = {
"lgb": {
"num_boost_round": 50000,
"early_stopping_rounds": 200,
"verbose_eval": 200,
},
"xgb": {
"num_boost_round": 50000,
"early_stopping_rounds": 200,
"verbose_eval": 200,
},
"cat": {
'early_stopping_rounds': 200,
'verbose_eval': 200,
}
}
attempt_no_sum_dict = attempt_no_agg['sum'].to_dict(defaultdict(int))
del attempt_no_agg
gc.collect()<feature_engineering>
oof_pred_lgb, test_pred_lgb, score_lgb, feat_imps_lgb = run_train_and_inference(
 X, X_test, y, "lgb", MODEL_PARAMS["lgb"], TRAIN_PARAMS["lgb"], RANDOM_SEED_LIST, N_SPLITS )
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]<feature_engineering>
score_lgb.loc[score_lgb.fold == "oof"]
<define_variables>
oof_pred_xgb, test_pred_xgb, score_xgb, feat_imps_xgb = run_train_and_inference(
 X, X_test, y, "xgb", MODEL_PARAMS["xgb"], TRAIN_PARAMS["xgb"], RANDOM_SEED_LIST, N_SPLITS )
iter_test = env.iter_test()
prior_test_df = None
prev_test_df1 = None<define_search_space>
score_xgb.loc[score_xgb.fold == "oof"]
N=[0.4,0.6]<feature_engineering>
oof_pred_cat, test_pred_cat, score_cat, feat_imps_cat = run_train_and_inference(
 X, X_test, y, "cat", MODEL_PARAMS["cat"], TRAIN_PARAMS["cat"],
RANDOM_SEED_LIST, N_SPLITS, )
%%time
for(test_df, sample_prediction_df)in iter_test:
test_df1=test_df.copy()
if(prev_test_df1 is not None):
prev_test_df1['answered_correctly'] = eval(test_df1['prior_group_answers_correct'].iloc[0])
 prev_test_df1 = prev_test_df1[prev_test_df1.content_type_id == False]
prev_group = prev_test_df1[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prev_test_df1 = test_df1.copy()
test_df1 = test_df1[test_df1.content_type_id == False]
test_dataset = TestDataset(group, test_df1, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = nn_model(x, target_id)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prior_test_df = test_df.copy()
question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 sub_preds = np.zeros(test_df.shape[0])
 for i, model in enumerate(clfs, 1):
test_preds = model.predict(test_df[features])
 sub_preds += test_preds
o2=sub_preds / len(clfs)
 test_df[target]=0.5*np.array(outs)+0.5*np.array(o2)
 env.predict(test_df[['row_id', target]] )<define_variables>
score_cat.loc[score_cat.fold == "oof"]
sys.path.append('/kaggle/input/riiid-dataset/')
 question_metadata_dir = r'/kaggle/input/riiid-dataset/question_metadata.csv'
lesson_metadata_dir = r'/kaggle/input/riiid-dataset/lesson_metadata.csv'
pickle_dir= r'/kaggle/input/riiid-dataset/stage.pickle'
model_dir = r'/kaggle/input/riiid-dataset/classifier.model'
print("{} Submission Started".format(str(datetime.datetime.now())))
 try:
except ImportError as e:
print("{} Import Processed finished with error：{}".format(str(datetime.datetime.now()), e))
print("{} Import Completed".format(str(datetime.datetime.now())))
 env = trueskill.TrueSkill(mu=0.3, sigma=0.164486, beta=0.05, tau=0.00164, draw_probability=0)
 env.make_as_global()
def win_probability(team1, team2):

 delta_mu = team1.mu - team2.mu
sum_sigma = sum([team1.sigma ** 2, team2.sigma ** 2])
 size = 2
denom = math.sqrt(size *(0.05 * 0.05)+ sum_sigma)
 ts = trueskill.global_env()
return ts.cdf(delta_mu / denom)
 class user:

 def __init__(self):

 self.question_answered_num = 0
self.question_answered_num_agg_field = [0] * 7
self.question_answered_mean_accuracy = 0
self.question_answered_mean_accuracy_agg_field = [0] * 7
self.question_answered_mean_difficulty_weighted_accuracy = 0
self.question_answered_mean_difficulty_weighted_accuracy_agg_field = [0] * 7
self.max_solved_difficulty = 1
self.max_solved_difficulty_agg_field = [1] * 7
self.min_wrong_difficulty = 0
self.min_wrong_difficulty_agg_field = [0] * 7
self.lessons_overall = 0
self.lessons_overall_agg_field = [0] * 7
self.session_time = 0
self.since_last_session_time = 0
self._mmr_object = trueskill.setup(mu=0.3, sigma=0.164486, beta=0.05, tau=0.00164,
draw_probability=0 ).Rating()
self._mmr_object_agg_field = [trueskill.setup(mu=0.3, sigma=0.164486, beta=0.05, tau=0.00164,
draw_probability=0 ).Rating() ] * 7
self._most_liked_guess = [0] * 4
self._last_session_start_time = 0
self._first_action_time = 0
self._question_num_dict = {}
self._first_processed_flag = False
def update_user(self, data: pd.DataFrame):

 _temp = None
if data['content_type_id'] == 0:
self.question_answered_num = self.question_answered_num + 1
question_field = int(data['content_field'])
 self.question_answered_num_agg_field[question_field - 1] = int(self.question_answered_num_agg_field[
question_field - 1])+ 1
if data['answered_correctly'] == 1:
self.question_answered_mean_accuracy = \ 
(self.question_answered_mean_accuracy *(
 self.question_answered_num - 1)+ 1)/ self.question_answered_num
self.question_answered_mean_accuracy_agg_field[question_field - 1] = \ 
(self.question_answered_mean_accuracy_agg_field[question_field - 1] *(
 self.question_answered_num_agg_field[question_field - 1] - 1)+ 1)\
/ self.question_answered_num_agg_field[question_field - 1]
self.question_answered_mean_difficulty_weighted_accuracy = \ 
(self.question_answered_mean_difficulty_weighted_accuracy *(self.question_answered_num - 1)+(
 1 - data['mean_question_accuracy'])* 3)\
/ self.question_answered_num
self.question_answered_mean_difficulty_weighted_accuracy_agg_field[question_field - 1] = \ 
(self.question_answered_mean_difficulty_weighted_accuracy_agg_field[question_field - 1] *(
 self.question_answered_num_agg_field[question_field - 1] - 1)+(
 1 - data['mean_question_accuracy'])* 3)\
/ self.question_answered_num_agg_field[question_field - 1]
else:
self.question_answered_mean_accuracy = \ 
(self.question_answered_mean_accuracy *(
 self.question_answered_num - 1)) / self.question_answered_num
self.question_answered_mean_accuracy_agg_field[question_field - 1] = \ 
(self.question_answered_mean_accuracy_agg_field[question_field - 1] *(
 self.question_answered_num_agg_field[question_field - 1] - 1)) / \
self.question_answered_num_agg_field[question_field - 1]
self.question_answered_mean_difficulty_weighted_accuracy = \ 
(self.question_answered_mean_difficulty_weighted_accuracy *(self.question_answered_num - 1)) \
/ self.question_answered_num
self.question_answered_mean_difficulty_weighted_accuracy_agg_field[question_field - 1] = \ 
(self.question_answered_mean_difficulty_weighted_accuracy_agg_field[question_field - 1] *(
 self.question_answered_num_agg_field[question_field - 1] - 1)) \
/ self.question_answered_num_agg_field[question_field - 1]
if data['answered_correctly'] == 1:
if data['mean_question_accuracy'] < self.max_solved_difficulty:
self.max_solved_difficulty = data['mean_question_accuracy']
if data['mean_question_accuracy'] < self.max_solved_difficulty_agg_field[question_field - 1]:
self.max_solved_difficulty_agg_field[question_field - 1] = data['mean_question_accuracy']
else:
if data['mean_question_accuracy'] > self.min_wrong_difficulty:
self.min_wrong_difficulty = data['mean_question_accuracy']
if data['mean_question_accuracy'] > self.min_wrong_difficulty_agg_field[question_field - 1]:
self.min_wrong_difficulty_agg_field[question_field - 1] = data['mean_question_accuracy']
if data['answered_correctly'] == 0:
self._most_liked_guess[int(data['user_answer'])] = self._most_liked_guess[
int(data['user_answer'])] + 1
if self._first_action_time == 0:
self._first_action_time = data['timestamp']
self._last_session_start_time = data['timestamp']
else:
if data['timestamp'] - self._last_session_start_time >= 7200 * 1000:
self.since_last_session_time =(data[
'timestamp'] - self._last_session_start_time)/ 1000 / 3600
self._last_session_start_time = data['timestamp']
self.session_time = 0
else:
self.session_time =(data['timestamp'] - self._last_session_start_time)/ 1000 / 60
if str(data['content_id'])in self._question_num_dict:
self._question_num_dict[str(data['content_id'])] = self._question_num_dict[str(data['content_id'])] + 1
else:
self._question_num_dict[str(data['content_id'])] = 1
if data['answered_correctly'] == 1:
self._mmr_object, _temp = \
trueskill.rate_1vs1(self._mmr_object,
trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486,
beta=0.05, tau=0.00164, draw_probability=0 ).Rating())
 self._mmr_object_agg_field[question_field - 1], _temp = \
trueskill.rate_1vs1(self._mmr_object_agg_field[question_field - 1],
trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486,
beta=0.05,
tau=0.00164, draw_probability=0 ).Rating())
 else:
_temp, self._mmr_object = \
trueskill.rate_1vs1(trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486,
beta=0.05, tau=0.00164, draw_probability=0 ).Rating() ,
self._mmr_object)
 _temp, self._mmr_object_agg_field[question_field - 1] = \
trueskill.rate_1vs1(trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486,
beta=0.05,
tau=0.00164, draw_probability=0 ).Rating() ,
self._mmr_object_agg_field[question_field - 1])
 else:
self.lessons_overall = self.lessons_overall + 1
lesson_field = int(data['content_field'])
 self.lessons_overall_agg_field[lesson_field - 1] = self.lessons_overall_agg_field[lesson_field - 1] + 1
def process_output(self, data):

 output_dict = {}
output_dict['question_answered_num'] = self.question_answered_num
output_dict['question_answered_num_agg_field'] = self.question_answered_num_agg_field[
int(data['content_field'])- 1]
output_dict['question_answered_mean_accuracy'] = self.question_answered_mean_accuracy
output_dict['question_answered_mean_accuracy_agg_field'] = self.question_answered_mean_accuracy_agg_field[
int(data['content_field'])- 1]
output_dict[
'question_answered_mean_difficulty_weighted_accuracy'] = self.question_answered_mean_difficulty_weighted_accuracy
output_dict['question_answered_mean_difficulty_weighted_accuracy_agg_field'] = \
self.question_answered_mean_difficulty_weighted_accuracy_agg_field[int(data['content_field'])- 1]
output_dict['max_solved_difficulty'] = self.max_solved_difficulty
output_dict['max_solved_difficulty_agg_field'] = self.max_solved_difficulty_agg_field[
int(data['content_field'])- 1]
output_dict['min_wrong_difficulty'] = self.min_wrong_difficulty
output_dict['min_wrong_difficulty_agg_field'] = self.min_wrong_difficulty_agg_field[
int(data['content_field'])- 1]
output_dict['lessons_overall'] = self.lessons_overall
output_dict['lessons_overall_agg_field'] = self.lessons_overall_agg_field[int(data['content_field'])- 1]
if output_dict['lessons_overall_agg_field'] > 0:
output_dict['field_learnt'] = 1
else:
output_dict['field_learnt'] = 0
output_dict['session_time'] = self.session_time
output_dict['time_to_last_session'] = self.since_last_session_time
output_dict['task_id'] = data['task_container_id']
output_dict['prior_time'] = data['prior_question_elapsed_time']
output_dict['mean_question_accuracy'] = data['mean_question_accuracy']
output_dict['std_question_accuracy'] = data['std_accuracy']
output_dict['question_id'] = data['content_id']
output_dict['mmr_overall'] = self._mmr_object.mu
output_dict['mmr_overall_agg_field'] = self._mmr_object_agg_field[int(data['content_field'])- 1].mu
output_dict['mmr_confidence'] = self._mmr_object.sigma
output_dict['mmr_overall_agg_field'] = self._mmr_object_agg_field[int(data['content_field'])- 1].sigma
output_dict['mmr_win_prob'] = win_probability(self._mmr_object,
trueskill.setup(mu=1 - data['mean_question_accuracy'],
sigma=0.164486,
beta=0.05, tau=0.00164,
draw_probability=0 ).Rating())
 output_dict['mmr_win_prob_agg_field'] = win_probability(
 self._mmr_object_agg_field[int(data['content_field'])- 1],
trueskill.setup(mu=1 - data['mean_question_accuracy'], sigma=0.164486, beta=0.05,
tau=0.00164, draw_probability=0 ).Rating())
 output_dict['user_id'] = data['user_id']
output_dict['tag_1'] = data['tag_1']
output_dict['tag_2'] = data['tag_2']
output_dict['tags_encoded'] = data['tags_encoded']
if not pd.isna(['prior_question_had_explanation']):
output_dict['previous_explained'] = data['prior_question_had_explanation']
else:
output_dict['previous_explained'] = False
if str(data['content_id'])in self._question_num_dict:
output_dict['question_seen'] = 1
else:
output_dict['question_seen'] = 0
max_choice = 0
max_choice_num = 0
i = 0
for item in self._most_liked_guess:
if item > max_choice_num:
max_choice_num = item
max_choice = i
i = i + 1
if output_dict['mmr_win_prob'] <= 0.4:
if max_choice == data['correct_answer']:
output_dict['most_liked_guess_correct'] = True
else:
output_dict['most_liked_guess_correct'] = False
else:
output_dict['most_liked_guess_correct'] = True
return output_dict
question_metadata = pd.read_csv(question_metadata_dir)
 lesson_metadata = pd.read_csv(lesson_metadata_dir)
 print("{} Metadata Imported".format(str(datetime.datetime.now())))
 question_metadata = question_metadata.set_index(keys=['content_id'])
 lesson_metadata = lesson_metadata.set_index(keys=['content_id'])
 print("{} Metadata Indexed".format(str(datetime.datetime.now())))
 with open(pickle_dir, 'rb')as fo:
user_pickle = pickle.load(fo)
 print("{} Pickle Object Imported".format(str(datetime.datetime.now())))
 for user_id,user_info in user_pickle.items() :
user_pickle[user_id]._mmr_object = trueskill.setup(mu=user_pickle[user_id]._mmr_object[0],
sigma=user_pickle[user_id]._mmr_object[1],
beta=0.05, tau=0.00164,
draw_probability=0 ).Rating()
for i in range(0, 7):
user_pickle[user_id]._mmr_object_agg_field[i] = trueskill.setup(mu=user_pickle[user_id]._mmr_object_agg_field[i][0],
sigma=user_pickle[user_id]._mmr_object_agg_field[i][1],
beta=0.05, tau=0.00164,
draw_probability=0 ).Rating()
print("{} Pickle Trueskill Rebuilt".format(str(datetime.datetime.now())))
 model = lgb.Booster(model_file=model_dir)
 print("{} Model Imported".format(str(datetime.datetime.now())))
 env = riiideducation.make_env()
iter_test = env.iter_test()
print("{} Init Competiton Environment".format(str(datetime.datetime.now())))
 rows_accum = 0
first_submission = True
model_prd = [0]
true_value = []
last_df = pd.DataFrame()
print("{} Init Done!".format(str(datetime.datetime.now())))
 for(test_df, sample_prediction_df)in iter_test:
if first_submission == False:
last_df['answered_correctly'] = eval(test_df.iloc[0]['prior_group_answers_correct'])
 last_df['user_answer'] = eval(test_df.iloc[0]['prior_group_responses'])
 true_value.extend(eval(test_df.iloc[0]['prior_group_answers_correct']))
for index,row in last_df.iterrows() :
user_pickle[row['user_id']].update_user(row)
 rows_accum = rows_accum + test_df.shape[0]
if first_submission == False:
1+1
test_df['answered_correctly'] = 0.6524
st = float(time.time())
 try:
sub_1 = test_df[test_df['content_type_id'] == False]
sub_2 = test_df[test_df['content_type_id'] == True]
del test_df
sub_1 = sub_1.merge(question_metadata, on="content_id", how="left")
 sub_2 = sub_2.merge(lesson_metadata, on="content_id", how="left")
 test_df = pd.DataFrame()
test_df = pd.concat([sub_1,sub_2])
 except Exception:
pass
for index, row in test_df.iterrows() :
try:
if row['user_id'] not in user_pickle:
user_pickle[row['user_id']] = user()
if row['content_type_id'] == 0:
predict_dict = user_pickle[row['user_id']].process_output(row)
 l = []
for i,v in predict_dict.items() :
l.append(v)
 prd_value = float(model.predict([l])[0])
 test_df.loc[test_df.row_id == row.row_id, 'answered_correctly'] = prd_value
model_prd.append(prd_value)
 except Exception as e:
print(e)
 pass
time_taken = float(time.time())- st
print("{} Based on current speed，it would take {} minutes to complete".format(
 str(datetime.datetime.now()),int(time_taken / test_df.shape[0] * 2500000 / 60)))
 if first_submission == True:
first_submission = False
last_df = test_df
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
<install_modules>
model_names = ["lgb", "xgb", "cat"]
!pip install.. /input/python-datatable/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl > /dev/null 2>&1<set_options>
pd.DataFrame(
 np.corrcoef([
oof_pred_lgb,
oof_pred_xgb,
oof_pred_cat
]),
columns=model_names, index=model_names )
_ = np.seterr(divide='ignore', invalid='ignore' )<set_options>
pd.DataFrame(
 np.corrcoef([
test_pred_lgb,
test_pred_xgb,
test_pred_cat
]),
columns=model_names, index=model_names )
%matplotlib inline 
<define_variables>
oof_pred_avg =(oof_pred_lgb + oof_pred_xgb + oof_pred_cat)/ 3
oof_score_avg = mean_squared_error(y, oof_pred_avg, squared=False)
 print(f"oof score avg: {oof_score_avg:.5f}")
 test_pred_avg =(test_pred_lgb + test_pred_xgb + test_pred_cat)/ 3
SEED = 123
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 seed_everything(SEED )<define_variables>
weights = [0.5, 0.4, 0.1]
oof_pred_wavg = weights[0] * oof_pred_lgb + weights[1] * oof_pred_xgb + weights[2] * oof_pred_cat
oof_score_wavg = mean_squared_error(y, oof_pred_wavg, squared=False)
 print(f"oof score weighted avg: {oof_score_wavg:.6f}")
 test_pred_wavg = weights[0] * test_pred_lgb + weights[1] * test_pred_xgb + weights[2] * test_pred_cat
<load_from_csv><EOS> 
sub = smpl_sub.copy()
sub[TGT_COL] = test_pred_wavg
sub.to_csv("submission.csv", index=False)
 sub.head()