Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<normalization>
_ = np.seterr(divide='ignore', invalid='ignore' )
X_test = imputer.transform(X_test )<predict_on_test>
with open('.. /input/file-collected-for-inference/baggingfeat/attempt_no_sum_dict.pickle', 'rb')as handle:
attempt_no_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_prior_question_elapsed_time_dict.pickle', 'rb')as handle:
user_prior_question_elapsed_time_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/max_timestamp_u_dict3.pickle', 'rb')as handle:
max_timestamp_u_dict3 = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/max_timestamp_u_dict2.pickle', 'rb')as handle:
max_timestamp_u_dict2 = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/max_timestamp_u_dict.pickle', 'rb')as handle:
max_timestamp_u_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_lecture_count_dict.pickle', 'rb')as handle:
user_lecture_count_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_lecture_sum_dict.pickle', 'rb')as handle:
user_lecture_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/explanation_count_dict.pickle', 'rb')as handle:
explanation_count_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/explanation_sum_dict.pickle', 'rb')as handle:
explanation_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/task_container_std_dict.pickle', 'rb')as handle:
task_container_std_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/task_container_count_dict.pickle', 'rb')as handle:
task_container_count_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/task_container_sum_dict.pickle', 'rb')as handle:
task_container_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_sum_dict.pickle', 'rb')as handle:
user_sum_dict = pickle.load(handle)
 with open('.. /input/file-collected-for-inference/baggingfeat/user_count_dict.pickle', 'rb')as handle:
user_count_dict = pickle.load(handle)
 lgb_model = joblib.load('.. /input/file-collected-for-inference/lgbmmodel/lgb.pkl')
 prior_question_elapsed_time_mean = 25423.810042960275
lagtime_mean2,lagtime_mean3,lagtime_mean =(39610585.65514838, 58879590.7244385, 20061832.085456338)
 delta_prior_question_elapsed_time_mean = -4.699396307374765
questions_df = pd.read_csv('.. /input/file-collected-for-inference/baggingfeat/questions_df.csv')
 N=[0.4,0.6]
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16',
}
features=list(features_dict.keys())
 target = 'answered_correctly'
def get_max_attempt(user_id,content_id):
k =(user_id,content_id)
 if k in attempt_no_sum_dict.keys() :
attempt_no_sum_dict[k]+=1
return attempt_no_sum_dict[k]
attempt_no_sum_dict[k] = 1
return attempt_no_sum_dict[k]
y_pred_bc = model_bc.predict(X_test )<choose_model_class>
MAX_SEQ = 240
DROPOUT=0.1
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class FFN0(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN0, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.layer_normal = nn.LayerNorm(state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 x=self.layer_normal(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask = np.triu(np.ones(( seq_length, seq_length)) , k=1 ).astype('bool')
 return torch.from_numpy(future_mask)
 class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.ffn0 = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x1 = self.ffn0(att_output)
 x = self.dropout(self.layer_normal_2(x + x1 + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
skills = joblib.load(".. /input/file-collected-for-inference/sakt100/skills.pkl.zip")
 n_skill = len(skills)
 group = joblib.load(".. /input/file-collected-for-inference/sakt100/group.pkl.zip")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 sakt_model = SAKTModel(n_skill, max_seq=240, embed_dim=256, forward_expansion=1, enc_layers=1, heads=4, dropout=0.1)
 sakt_model.load_state_dict(torch.load(".. /input/v4-fork-of-riiid-sakt-model-full/sakt_model.pt", map_location='cpu'))
sakt_model.to(device)
 sakt_model.eval()
class TestDataset(Dataset):
def __init__(self, samples, test_df, skills, max_seq=MAX_SEQ):
super(TestDataset, self ).__init__()
self.samples = samples
self.user_ids = [x for x in test_df["user_id"].unique() ]
self.test_df = test_df
self.skills = skills
self.n_skill = len(skills)
 self.max_seq = max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info["user_id"]
target_id = test_info["content_id"]
q = np.zeros(self.max_seq, dtype=int)
 qa = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
q_, qa_ = self.samples[user_id]
seq_len = len(q_)
 if seq_len >= self.max_seq:
q = q_[-self.max_seq:]
qa = qa_[-self.max_seq:]
else:
q[-seq_len:] = q_
qa[-seq_len:] = qa_
x = np.zeros(self.max_seq-1, dtype=int)
 x = q[1:].copy()
x +=(qa[1:] == 1)* self.n_skill
questions = np.append(q[2:], [target_id])
 return x, questions
lgb = LGBMClassifier()<train_model>
cat_model = CatBoostClassifier()
cat_model.load_model('.. /input/file-collected-for-inference/catmodel/catboost_model' )
lgb.fit(X_res,y_res )<predict_on_test>
DROPOUT=0.1
MAX_SEQ2 = 170
ACCEPTED_USER_CONTENT_SIZE = 5
EMBED_SIZE = 128
class FFN2(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size=MAX_SEQ2 - 1, dropout=0.2):
super(FFN2, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN2(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x = self.dropout(self.layer_normal_2(x + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel2(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel2, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
def create_model() :
return SAKTModel2(n_skill, max_seq=MAX_SEQ2, embed_dim=EMBED_SIZE, forward_expansion=3, enc_layers=1, heads=8, dropout=0.1)
 sakt_model2 = create_model()
sakt_model2.load_state_dict(torch.load('.. /input/file-collected-for-inference/sakt170/sakt.pth',map_location = 'cpu'))
sakt_model2.eval()
y_pred_lgb = lgb.predict(X_test )<predict_on_test>
env = riiideducation.make_env()
iter_test = env.iter_test()
prior_test_df = None
Proba_bc=model_bc.predict_proba(X_test)
 Proba_lgb=lgb.predict_proba(X_test )<prepare_output>
for(test_df, sample_prediction_df)in iter_test:
if prior_test_df is not None:
prior_test_df[target] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prior_test_df = prior_test_df[prior_test_df[target] != -1].reset_index(drop=True)
 prior_test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 prior_test_df.prior_question_had_explanation=prior_test_df.prior_question_had_explanation.astype('int8')
 user_ids = prior_test_df['user_id'].values
targets = prior_test_df[target].values
for user_id, answered_correctly in zip(user_ids,targets):
user_sum_dict[user_id] += answered_correctly
user_count_dict[user_id] += 1
prev_group = prior_test_df[['user_id', 'content_id', 'answered_correctly']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 prior_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_dataset = TestDataset(group, test_df, skills)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 sakt_preds = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = sakt_model(x, target_id)
 sakt_preds.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 sakt_preds2 = []
test_dataset = TestDataset(group, test_df, skills,max_seq = MAX_SEQ2)
 test_dataloader = DataLoader(test_dataset, batch_size= len(test_df), shuffle=False)
 for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, _ = sakt_model2(x, target_id)
 sakt_preds2.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 question_len=len(test_df[test_df['content_type_id'] == 0])
 test_df['prior_question_had_explanation'].fillna(False, inplace=True)
 test_df.prior_question_had_explanation=test_df.prior_question_had_explanation.astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace=True)
 user_lecture_sum = np.zeros(question_len, dtype=np.int16)
 user_lecture_count = np.zeros(question_len, dtype=np.int16)
 user_sum = np.zeros(question_len, dtype=np.int16)
 user_count = np.zeros(question_len, dtype=np.int16)
 task_container_sum = np.zeros(question_len, dtype=np.int32)
 task_container_count = np.zeros(question_len, dtype=np.int32)
 task_container_std = np.zeros(question_len, dtype=np.float16)
 explanation_sum = np.zeros(question_len, dtype=np.int32)
 explanation_count = np.zeros(question_len, dtype=np.int32)
 delta_prior_question_elapsed_time = np.zeros(question_len, dtype=np.int32)
 attempt_no_count = np.zeros(question_len, dtype=np.int16)
 lagtime = np.zeros(question_len, dtype=np.float32)
 lagtime2 = np.zeros(question_len, dtype=np.float32)
 lagtime3 = np.zeros(question_len, dtype=np.float32)
 i=0
for j,(user_id,prior_question_had_explanation,content_type_id,prior_question_elapsed_time,timestamp, content_id,task_container_id)in enumerate(zip(test_df['user_id'].values,test_df['prior_question_had_explanation'].values,test_df['content_type_id'].values,test_df['prior_question_elapsed_time'].values,test_df['timestamp'].values, test_df['content_id'].values, test_df['task_container_id'].values)) :
user_lecture_sum_dict[user_id] += content_type_id
user_lecture_count_dict[user_id] += 1
if(content_type_id==1):
x=1
if(content_type_id==0):
user_lecture_sum[i] = user_lecture_sum_dict[user_id]
user_lecture_count[i] = user_lecture_count_dict[user_id]
user_sum[i] = user_sum_dict[user_id]
user_count[i] = user_count_dict[user_id]
task_container_sum[i] = task_container_sum_dict[task_container_id]
task_container_count[i] = task_container_count_dict[task_container_id]
task_container_std[i]=task_container_std_dict[task_container_id]
explanation_sum_dict[user_id] += prior_question_had_explanation
explanation_count_dict[user_id] += 1
explanation_sum[i] = explanation_sum_dict[user_id]
explanation_count[i] = explanation_count_dict[user_id]
if user_id in max_timestamp_u_dict['max_time_stamp'].keys() :
lagtime[i]=timestamp-max_timestamp_u_dict['max_time_stamp'][user_id]
if(max_timestamp_u_dict2['max_time_stamp2'][user_id]==lagtime_mean2):
lagtime2[i]=lagtime_mean2
lagtime3[i]=lagtime_mean3
else:
lagtime2[i]=timestamp-max_timestamp_u_dict2['max_time_stamp2'][user_id]
if(max_timestamp_u_dict3['max_time_stamp3'][user_id]==lagtime_mean3):
lagtime3[i]=lagtime_mean3
else:
lagtime3[i]=timestamp-max_timestamp_u_dict3['max_time_stamp3'][user_id]
max_timestamp_u_dict3['max_time_stamp3'][user_id]=max_timestamp_u_dict2['max_time_stamp2'][user_id]
max_timestamp_u_dict2['max_time_stamp2'][user_id]=max_timestamp_u_dict['max_time_stamp'][user_id]
max_timestamp_u_dict['max_time_stamp'][user_id]=timestamp
else:
lagtime[i]=lagtime_mean
max_timestamp_u_dict['max_time_stamp'].update({user_id:timestamp})
 lagtime2[i]=lagtime_mean2
max_timestamp_u_dict2['max_time_stamp2'].update({user_id:lagtime_mean2})
 lagtime3[i]=lagtime_mean3
max_timestamp_u_dict3['max_time_stamp3'].update({user_id:lagtime_mean3})
 if user_id in user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].keys() :
delta_prior_question_elapsed_time[i]=prior_question_elapsed_time-user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'][user_id]=prior_question_elapsed_time
else:
delta_prior_question_elapsed_time[i]=delta_prior_question_elapsed_time_mean
user_prior_question_elapsed_time_dict['prior_question_elapsed_time'].update({user_id:prior_question_elapsed_time})
 i=i+1
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 test_df=test_df.merge(questions_df.loc[questions_df.index.isin(test_df['content_id'])],
how='left', on='content_id', right_index=True)
 test_df['user_lecture_lv'] = user_lecture_sum / user_lecture_count
test_df['user_lecture_sum'] = user_lecture_sum
test_df['user_interaction_count'] = user_lecture_count
test_df['user_interaction_timestamp_mean'] = test_df['timestamp']/user_lecture_count
test_df['user_correctness'] = user_sum / user_count
test_df['user_uncorrect_count'] =user_count-user_sum
test_df['user_correct_count'] =user_sum
test_df['task_container_correctness'] = task_container_sum / task_container_count
test_df['task_container_cor_count'] = task_container_sum
test_df['task_container_uncor_count'] =task_container_count-task_container_sum
test_df['task_container_std'] = task_container_std
test_df['explanation_mean'] = explanation_sum / explanation_count
test_df['explanation_true_count'] = explanation_sum
test_df['explanation_false_count'] = explanation_count-explanation_sum
test_df['delta_prior_question_elapsed_time'] = delta_prior_question_elapsed_time
test_df["attempt_no"] = test_df[["user_id", "content_id"]].apply(lambda row: get_max_attempt(row["user_id"], row["content_id"]), axis=1)
 test_df["lagtime"]=lagtime
test_df["lagtime2"]=lagtime2
test_df["lagtime3"]=lagtime3
test_df['timestamp']=test_df['timestamp']/(1000*3600)
 test_df.timestamp=test_df.timestamp.astype('float16')
 test_df['lagtime']=test_df['lagtime']/(1000*3600)
 test_df.lagtime=test_df.lagtime.astype('float32')
 test_df['lagtime2']=test_df['lagtime2']/(1000*3600)
 test_df.lagtime2=test_df.lagtime2.astype('float32')
 test_df['lagtime3']=test_df['lagtime3']/(1000*3600)
 test_df.lagtime3=test_df.lagtime3.astype('float32')
 test_df['user_interaction_timestamp_mean']=test_df['user_interaction_timestamp_mean']/(1000*3600)
 test_df.user_interaction_timestamp_mean=test_df.user_interaction_timestamp_mean.astype('float32')
 test_df['user_correctness'].fillna(0.67, inplace=True)
 lgb_preds = lgb_model.predict(test_df[features])
 cat_preds = cat_model.predict_proba(test_df[features])[:,1]
test_df[target] = 0.20*(lgb_preds)+ 0.30*cat_preds+ 0.22*np.array(sakt_preds2)+ 0.28*np.array(sakt_preds)
 env.predict(test_df[['row_id', target]] )