Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<categorify>
env = riiideducation.make_env()
iter_test = env.iter_test()
labelencoder_X = LabelEncoder()
X[:, 1] = labelencoder_X.fit_transform(X[:, 1] )<categorify>
import sys
import numpy as np
labelencoder_test_data_X = LabelEncoder()
test_data_X[:, 1] = labelencoder_test_data_X.fit_transform(test_data_X[:, 1] )<split>
warnings.filterwarnings("ignore")
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25, random_state = 0 )<normalization>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 q_pad = 13523
a_pad = 3
start_token = 2
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
 X_test = sc_X.transform(X_test)
 test_data_X = sc_X.transform(test_data_X )<train_model>
group = pd.read_pickle(".. /input/groups/group.pandas" )
classifier = SVC(kernel = 'rbf', random_state = 0)
 classifier.fit(X_train, y_train )<predict_on_test>
features_1_path = '.. /input/get-features-1/'
que_data = pd.read_pickle(features_1_path + "que_data.pickle")
 difficulty =(np.round(que_data.que_correct_per, 1)*10 ).astype("int8" ).values
difficulty = torch.Tensor(difficulty ).long().to(device)
 unique_tags = pd.concat([que_data.tags1,que_data.tags2, que_data.tags3, que_data.tags4,que_data.tags5,que_data.tags6] ).unique()
tags_n = len(unique_tags)
 unk_tag = tags_n-1
que_data = que_data.replace(-1, unk_tag)
 part_valus = torch.from_numpy(que_data.part.values ).long().to(device)
 que_data = que_data.to_dict("index")
 que_arr = np.zeros(( np.array(list(que_data.keys())).shape[0], 6))
for i in que_data:
a = que_data[i]
que_arr[i] = [a['tags1'],a['tags2'],a['tags3'],a['tags4'],a['tags5'],a['tags6']]
with open('.. /input/hashtable/user_info', 'rb')as handle:
user_info = pickle.load(handle )
y_pred = classifier.predict(test_data_X )<save_to_csv>
st_user_info = {}
for i in user_info:
st_user_info[i] = {"timestamp_ms":user_info[i]["first_timestamp"]}
del user_info
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': y_pred})
 output.to_csv('my_submission.csv', index=False )<import_modules>
class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 self.scale = nn.Parameter(torch.ones(1))
pe = torch.zeros(max_len, d_model)
 position = torch.arange(0, max_len, dtype=torch.float ).unsqueeze(1)
 div_term = torch.exp(torch.arange(
 0, d_model, 2 ).float() *(-math.log(10000.0)/ d_model))
pe[:, 0::2] = torch.sin(position * div_term)
 pe[:, 1::2] = torch.cos(position * div_term)
 pe = pe.unsqueeze(0 ).transpose(0, 1)
 self.register_buffer('pe', pe)
 def forward(self, x):
x = x + self.scale * self.pe[:x.size(0), :]
return self.dropout(x )
sns.set(color_codes=True)
 %matplotlib inline
print(check_output(["ls", ".. /input"] ).decode("utf8")) 
<load_from_csv>
class EmbedTag(nn.Module):
def __init__(self, d_model, que_arr, tags_n):
super(EmbedTag, self ).__init__()
self.que_arr = torch.LongTensor(que_arr ).to(device)
 self.embedding = nn.Embedding(tags_n, d_model)
 def forward(self, x):
x = self.que_arr[x, :]
x = self.embedding(x)
 return torch.sum(x, dim=-2 )
train = pd.read_csv(".. /input/train.csv", dtype={"Age": np.float64},)
 test = pd.read_csv(".. /input/test.csv", dtype={"Age": np.float64}, )<count_missing_values>
class TransformerModel(nn.Module):
def __init__(self, intoken, outtoken, hidden, que_arr, part_arr, difficulty, enc_layers=4, dec_layers=4, dropout=0.1, ts_unique=70, prior_unique=50):
super(TransformerModel, self ).__init__()
nhead = hidden//64
self.encoder = nn.Embedding(intoken, hidden)
 self.pos_encoder = PositionalEncoding(hidden, dropout)
 self.decoder = nn.Embedding(outtoken, hidden)
 self.pos_decoder = PositionalEncoding(hidden, dropout)
 self.tagsEmbedder = EmbedTag(hidden, que_arr, tags_n)
 self.transformer = nn.Transformer(d_model=hidden, nhead=nhead, num_encoder_layers=enc_layers, num_decoder_layers=dec_layers, dim_feedforward=hidden*4, dropout=dropout, activation='relu')
 self.fc_out = nn.Linear(hidden, 1)
 self.src_mask = None
self.trg_mask = None
self.memory_mask = None
self.part_embedding = nn.Embedding(7,hidden)
 self.part_arr = part_arr
self.ts_embedding = nn.Embedding(ts_unique, hidden)
 self.prior_embedding = nn.Embedding(prior_unique, hidden)
 self.task_container_embedding = nn.Embedding(10000, hidden)
 self.user_answer_embedding = nn.Embedding(5, hidden)
 self.difficulty = difficulty
self.difficulty_embedding = nn.Embedding(11, hidden)
 self.dropout_7 = nn.Dropout(dropout)
 self.dropout_1 = nn.Dropout(dropout)
 self.dropout_2 = nn.Dropout(dropout)
 self.dropout_3 = nn.Dropout(dropout)
 self.dropout_4 = nn.Dropout(dropout)
 self.dropout_5 = nn.Dropout(dropout)
 self.dropout_6 = nn.Dropout(dropout)
 self.explan_embedding = nn.Embedding(3, hidden)
 self.dropout_9 = nn.Dropout(dropout)
 def generate_square_subsequent_mask(self, sz, sz1=None):
if sz1 == None:
mask = torch.triu(torch.ones(sz, sz), 1)
 else:
mask = torch.triu(torch.ones(sz, sz1), 1)
 return mask.masked_fill(mask==1, float('-inf'))
def make_len_mask_a(self, inp):
return(inp == a_pad ).transpose(0, 1)
 def make_len_mask_q(self, inp):
return(inp == q_pad ).transpose(0, 1)
 def forward(self, src, trg, ts, prior, task_container, user_answer, explan):
if self.trg_mask is None or self.trg_mask.size(0)!= len(trg):
self.trg_mask = self.generate_square_subsequent_mask(len(trg)).to(trg.device)
 if self.src_mask is None or self.src_mask.size(0)!= len(src):
self.src_mask = self.generate_square_subsequent_mask(len(src)).to(trg.device)
 if self.memory_mask is None or self.memory_mask.size(0)!= len(trg)or self.memory_mask.size(1)!= len(src):
self.memory_mask = self.generate_square_subsequent_mask(len(trg),len(src)).to(trg.device)
 src_pad_mask = self.make_len_mask_q(src)
 trg_pad_mask = self.make_len_mask_a(trg)
 part_emb = self.dropout_1(self.part_embedding(self.part_arr[src]-1))
ts_emb = self.dropout_3(self.ts_embedding(ts))
user_answer_emb = self.dropout_5(self.user_answer_embedding(user_answer))
src = self.encoder(src)
 src = torch.add(src, part_emb)
 src = torch.add(src, ts_emb)
 src = self.pos_encoder(src)
 trg = self.decoder(trg)
 trg = torch.add(trg, user_answer_emb)
 trg = self.pos_decoder(trg)
 output = self.transformer(src, trg, src_mask=self.src_mask, tgt_mask=self.trg_mask, memory_mask=self.memory_mask,
src_key_padding_mask=src_pad_mask, tgt_key_padding_mask=trg_pad_mask, memory_key_padding_mask=src_pad_mask)
 output = self.fc_out(output)
 return output
train.isnull().sum(axis=0 )<count_values>
d_model = 128
INPUT_DIM = q_pad+1
OUTPUT_DIM = 4
model_saint = TransformerModel(INPUT_DIM, OUTPUT_DIM, hidden=d_model, que_arr=que_arr,part_arr=part_valus, difficulty=difficulty ).to(device)
 weights = torch.load(".. /input/last-saint/last.torch", map_location=torch.device(device))
model_saint.load_state_dict(weights)
 model_saint.to(device)
 model_saint.eval()
test['Sex'].value_counts()<prepare_x_and_y>
def pred_users(vals):
eval_batch = vals.shape[0]
tensor_question = np.zeros(( eval_batch, 100), dtype=np.long)
 tensor_answers = np.zeros(( eval_batch, 100), dtype=np.long)
 tensor_ts = np.zeros(( eval_batch, 100), dtype=np.long)
 tensor_user_answer = np.zeros(( eval_batch, 100), dtype=np.long)
 val_len = []
preds = []
group_index = group.index
for i, line in enumerate(vals):
if line[2] == True:
val_len.append(0)
 continue
user_id = line[0]
question_id = line[1]
timestamp = get_timestamp(line[4], user_id)
 prior = get_prior(line[5])
 task_container_id = line[3]
que_history = np.array([], dtype=np.int32)
 answers_history = np.array([], dtype=np.int32)
 ts_history = np.array([], dtype=np.int32)
 user_answer_history = np.array([], dtype=np.int32)
 if user_id in group_index:
cap = 99
que_history, answers_history, ts_history, user_answer_history = group[user_id]
que_history = que_history[-cap:]
answers_history = answers_history[-cap:]
ts_history = ts_history[-cap:]
user_answer_history = user_answer_history[-cap:]
a_token = 2
user_a_token = 4
answers_history = np.concatenate(( [a_token],answers_history))
user_answer_history = np.concatenate(( [user_a_token],user_answer_history))
que_history = np.concatenate(( que_history, [question_id]))
ts_history = np.concatenate(( ts_history, [timestamp]))
tensor_question[i][:len(que_history)] = que_history
tensor_answers[i][:len(que_history)] = answers_history
tensor_ts[i][:len(que_history)] = ts_history
tensor_user_answer[i][:len(que_history)] = user_answer_history
val_len.append(len(que_history))
tensor_question = torch.from_numpy(tensor_question ).long().T.to(device)
 tensor_answers = torch.from_numpy(tensor_answers ).long().T.to(device)
 tensor_ts = torch.from_numpy(tensor_ts ).long().T.to(device)
 tensor_user_answer = torch.from_numpy(tensor_user_answer ).long().T.to(device)
 with torch.no_grad() :
out = F.sigmoid(model_saint(tensor_question, tensor_answers, tensor_ts, None, None, tensor_user_answer,2)).squeeze(dim=-1 ).T
for j in range(len(val_len)) :
preds.append(out[j][val_len[j]-1].item())
 return preds
test_id = test['PassengerId']
target = train['Survived']<data_type_conversions>
def split_preds(preds):
if preds.shape[0] > 1000:
ret = []
for i in np.array_split(preds, preds.shape[0]//1000):
ret.extend(pred_users(i))
return ret
else:
return pred_users(preds )
train["Fare"].fillna(train["Fare"].median() , inplace=True)
 test["Fare"].fillna(test["Fare"].median() , inplace=True )<set_options>
def update_group_var(vals):
global group
for i, line in enumerate(vals):
user_id = line[0]
question_id = line[1]
content_type_id = line[2]
ts = get_timestamp(line[4], user_id)
 correct = line[6]
user_answer = line[7]
if content_type_id == True:
continue
if st_user_info.get(user_id, -1)== -1:
st_user_info[user_id] = {"timestamp_ms":0}
else:
st_user_info[user_id]["timestamp_ms"] = line[4]
if user_id in group.index:
questions= np.append(group[user_id][0],[question_id])
 answers= np.append(group[user_id][1],[correct])
 ts= np.append(group[user_id][2],[ts])
 user_answer= np.append(group[user_id][3],[user_answer])
 group[user_id] =(questions, answers, ts, user_answer)
 else:
group[user_id] =(np.array([question_id], dtype=np.int32), np.array([correct], dtype=np.int32), np.array([ts], dtype=np.int32)
 ,np.array([user_answer], dtype=np.int32))
train["Embarked"].fillna(train['Embarked'].mode() [0], inplace=True)
 test["Embarked"].fillna(test['Embarked'].mode() [0], inplace=True )<data_type_conversions>
ordinal_enc = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6, 7: 7, 8: 8, 9: 9, 10: 10, 11: 11, 12: 12, 13: 13, 14: 14, 15: 15, 16: 16, 17: 17, 18: 18, 19: 19, 20: 20, 30: 21, 40: 22, 50: 23, 60: 24, 70: 25, 80: 26, 90: 27, 100: 28, 110: 29, 120: 30, 130: 31, 140: 32, 150: 33, 160: 34, 170: 35, 180: 36, 190: 37, 200: 38, 210: 39, 220: 40, 230: 41, 240: 42, 250: 43, 260: 44, 270: 45, 280: 46, 290: 47, 300: 48}
boundaries = [120,600,1800,3600,10800,43200,86400,259200,604800]
def get_prior(prior):
if prior != prior:
return 0
prior /= 1000
if prior > 20:
prior = np.round(prior, decimals=-1)
 return ordinal_enc.get(round(prior),0)
 def get_timestamp(ts, user_id):
if st_user_info.get(user_id, -1)== -1:
return 0
diff =(ts - st_user_info[user_id]["timestamp_ms"])/1000
if diff < 0:
return 0
if diff <= 60:
return int(diff)
 for i, boundary in enumerate(boundaries):
if boundary > diff:
break
if i == len(boundaries)- 1:
return 60+i+1
return 60+i
train["Age"].fillna(train["Age"].median() , inplace=True)
 test["Age"].fillna(test["Age"].median() , inplace=True )<data_type_conversions>
prior_part_mean_dict = {1: 22166.159642501425,
2: 18714.69673913695,
3: 23620.317746179924,
4: 23762.753651169547,
5: 25094.620302855932,
6: 32417.37918735745,
7: 47444.16407400242}
<string_transform>
with open('.. /input/lgbm-test/repeated_que_count', 'rb')as handle:
repeated_que_count = pickle.load(handle)
 with open('.. /input/lgbm-test/user_info', 'rb')as handle:
user_info = pickle.load(handle)
 with open('.. /input/lgbm-test/watched_tags', 'rb')as handle:
watched_tags = pickle.load(handle)
 with open('.. /input/lgbm-test/containers_mean', 'rb')as handle:
containers_mean = pickle.load(handle)
 with open('.. /input/lgbm-test/hardest', 'rb')as handle:
hard_questions = pickle.load(handle)
 with open('.. /input/lgbm-test/easiest', 'rb')as handle:
easy_questions = pickle.load(handle)
 with open('.. /input/lgbm-test/que_2_k', 'rb')as handle:
que_2_k = pickle.load(handle)
 gc.collect()
def substrings_in_string(big_string, substrings):
for substring in substrings:
if big_string.find(substring)!= -1:
return substring
return np.nan<categorify>
for u in user_info:
user_info[u]["count_2"] = user_info[u]["count"]
user_info[u]["part_count_2"] = user_info[u]["part_count"].copy()
user_info[u]["last_part"] = 1
def replace_titles(x):
title=x['Title']
if title in ['Don', 'Major', 'Capt', 'Jonkheer', 'Rev', 'Col']:
return 'Mr'
elif title in ['Countess', 'Mme']:
return 'Mrs'
elif title in ['Mlle', 'Ms']:
return 'Miss'
elif title =='Dr':
if x['Sex']=='Male':
return 'Mr'
else:
return 'Mrs'
else:
return title<define_variables>
groups = pd.read_pickle(".. /input/lgbm-test/groups")
 def numpy_ewma_vectorized_v2(data, window):
alpha = 2 /(window + 1.0)
 alpha_rev = 1-alpha
n = data.shape[0]
pows = alpha_rev**(np.arange(n+1))
scale_arr = 1/pows[:-1]
offset = data[0]*pows[1:]
pw0 = alpha*alpha_rev**(n-1)
 mult = data*pw0*scale_arr
cumsums = mult.cumsum()
out = offset + cumsums*scale_arr[::-1]
return out
title_list=['Mrs', 'Mr', 'Master', 'Miss', 'Major', 'Rev',
'Dr', 'Ms', 'Mlle','Col', 'Capt', 'Mme', 'Countess',
'Don', 'Jonkheer']<feature_engineering>
features_1_path = '.. /input/get-features-1/'
que_data = pd.read_pickle(features_1_path + "que_data.pickle")
 questions = que_data.drop(columns=["options_number","correctness_number", "correct_answer","tags6","tags5", "tags4"] ).to_dict("index")
 questions1 = que_data[["tags1", "tags2", "tags3","tags4","tags5", "tags6"]].to_dict("index")
 parts = que_data.part.to_dict()
train['Title']=train['Name'].map(lambda x: substrings_in_string(x, title_list))
test['Title']=test['Name'].map(lambda x: substrings_in_string(x, title_list))<feature_engineering>
lec_data = pd.read_csv(".. /input/riiid-test-answer-prediction/lectures.csv")
 lec_dict = lec_data[["lecture_id", "tag"]].set_index("lecture_id" ).tag.to_dict()
features = [
'task_container_id', "ts_diff_shifted", "watched","ts_diff_shifted_2",
'content_id', "k", "k_acc", "el_avg", "wut",
'prior_question_elapsed_time', "time_diff2", "rolling_mean_5", "rolling_mean_10", "rolling_mean_15", "prior_question_had_explanation_u_part_avg",
'prior_question_had_explanation', "hard_ratio_opp", "easy_ratio_opp", "correct_recency", "prior_question_elapsed_time_u_part_avg", "ewm_mean_10", "rolling_mean_5_prior_question",
'last_lecture', "part_mean", "opp_mean", "mean_pause", "timestamp", "prior_part_mean",
"container_mean", "lecs_per", "hard_ratio", "easy_ratio",
'que_count_user', 'question_repeated', "rolling_mean","time_diff3", "time_diff4",
'user_mean', "time_diff1", "time_diff", "sessions", "session_count", "prior_question_had_explanation_ratio"
] + que_data.columns.tolist() [:-1]
features.remove("options_number")
 features.remove("correct_answer")
 features.remove("tags6")
 features.remove("tags5")
 features.remove("tags4")
 test_cols = ['row_id','timestamp','user_id','content_id','content_type_id','task_container_id','prior_question_elapsed_time',
'prior_question_had_explanation','prior_group_answers_correct','prior_group_responses']
train['Title']=train.apply(replace_titles, axis=1)
 test['Title']=test.apply(replace_titles, axis=1 )<data_type_conversions>
stack_features = [
'task_container_id', "ts_diff_shifted", "watched","ts_diff_shifted_2",
'content_id', "k", "k_acc", "el_avg", "wut", "lgb_preds", "st_preds",
'prior_question_elapsed_time', "time_diff2", "rolling_mean_5", "rolling_mean_10", "rolling_mean_15", "prior_question_had_explanation_u_part_avg",
'prior_question_had_explanation', "hard_ratio_opp", "easy_ratio_opp", "correct_recency", "prior_question_elapsed_time_u_part_avg", "ewm_mean_10", "rolling_mean_5_prior_question",
'last_lecture', "part_mean", "opp_mean", "mean_pause", "timestamp", "prior_part_mean",
"container_mean", "lecs_per", "hard_ratio", "easy_ratio",
'que_count_user', 'question_repeated', "rolling_mean","time_diff3", "time_diff4",
'user_mean', "time_diff1", "time_diff", "sessions", "session_count", "prior_question_had_explanation_ratio"
] + que_data.columns.tolist() [:-1]
stack_features.remove("options_number")
 stack_features.remove("correct_answer")
 stack_features.remove("tags6")
 stack_features.remove("tags5")
 stack_features.remove("tags4")
 lgb_preds_idx = stack_features.index("lgb_preds")
 st_preds_idx = stack_features.index("st_preds" )
train['Family_Size']=train['SibSp']+train['Parch']
test['Family_Size']=test['SibSp']+test['Parch']<feature_engineering>
k_size = 20
cols = {test_cols[k]:k for k in range(len(test_cols)) }
features_dict = {features[k]:k for k in range(len(features)) }
train['Age*Class']=train['Age']*train['Pclass']
test['Age*Class']=test['Age']*test['Pclass']<feature_engineering>
new_user = {'count': 0, 'mean_acc':0.5, 'correct_count': 0, 'last_lec':0, 'tmp':0,"first_timestamp":0, "second_timestamp":0,
"third_timestamp":0, "fourth_timestamp":0, "fifth_timestamp":0, "lecs_n":0,"interaction_n":0, "ts_diff_shifted":0.,
"part_corr":np.zeros(( 7), dtype=np.uint16), "part_count":np.zeros(( 7), dtype=np.uint16), "hard_ct":0, "hard_cr":0, "easy_ct":0, "easy_cr":0,
"sessions":0, "session_count":0, "sum_pauses":0., "had_exp":0, "el_sum":0, "part_et": np.zeros(( 7), dtype=np.float),
"part_explan": np.zeros(( 7), dtype=np.uint16), "k_count": np.zeros(( k_size), dtype=np.uint16), "k_corr": np.zeros(( k_size), dtype=np.uint16),
"recent_corr":0, "priors_5": [], "ts_diff_shifted_2":0., "count_2":0, "part_count_2":np.zeros(( 7), dtype=np.uint16), "last_part":1}
train['Fare_Per_Person']=train['Fare']/(train['Family_Size']+1)
 test['Fare_Per_Person']=test['Fare']/(test['Family_Size']+1 )<data_type_conversions>
def get_meta_data(data_1):
user_id = data_1[cols['user_id']]
content_type_id = data_1[cols['content_type_id']]
content_id = data_1[cols['content_id']]
prior_group_answers_correct = data_1[cols['prior_group_answers_correct']]
timestamp = data_1[cols['timestamp']]
task_container_id = data_1[cols['task_container_id']]
prior_question_had_explanation = data_1[cols['prior_question_had_explanation']]
elapsed = data_1[cols['prior_question_elapsed_time']]
return user_id, content_type_id, content_id, prior_group_answers_correct, timestamp,task_container_id,prior_question_had_explanation, elapsed
train["Embarked"].fillna(0, inplace=True)
 test["Embarked"].fillna(0, inplace=True )<categorify>
def add_user(user_id):
user_info[user_id] = copy.deepcopy(new_user)
 repeated_que_count[user_id] = {}
groups[user_id] = []
lb_make = LabelEncoder()
train["Embarked"] = lb_make.fit_transform(train["Embarked"])
 train["Sex"] = lb_make.fit_transform(train["Sex"])
 train["Title"] = lb_make.fit_transform(train["Title"])
 test["Embarked"] = lb_make.fit_transform(test["Embarked"])
 test["Sex"] = lb_make.fit_transform(test["Sex"])
 test["Title"] = lb_make.fit_transform(test["Title"] )<drop_column>
def update_user_part_acc(user_id, question, answered_correctly, elapsed, explan):
part = parts.get(question, -1)
 user_info[user_id]["part_count"][part-1] += 1
user_info[user_id]["part_corr"][part-1] += answered_correctly
if not isinstance(explan, pd._libs.missing.NAType)and explan == explan:
user_info[user_id]["had_exp"] += explan
train = train.drop(['Name', 'PassengerId', 'Survived', 'Ticket', 'Cabin'], axis=1)
 test = test.drop(['Name', 'PassengerId', 'Ticket', 'Cabin'], axis=1 )<import_modules>
def update_user(user_id, had_exp, elapsed, content_id ,answered_correctly, timestamp):
user_info[user_id]['count'] += 1
if repeated_que_count[user_id].get(content_id, -1)== -1:
repeated_que_count[user_id][content_id] = 1
else:
repeated_que_count[user_id][content_id] += 1
if answered_correctly:
user_info[user_id]['correct_count'] += 1
user_info[user_id]['mean_acc'] = user_info[user_id]['correct_count']/user_info[user_id]['count']
update_user_part_acc(user_id, content_id, answered_correctly, elapsed, had_exp)
 if hard_questions.get(content_id, False):
user_info[user_id]["hard_ct"] += 1
user_info[user_id]["hard_cr"] += answered_correctly
if easy_questions.get(content_id, False):
user_info[user_id]["easy_ct"] += 1
user_info[user_id]["easy_cr"] += answered_correctly
k = que_2_k[content_id]
user_info[user_id]["k_count"][k] += 1
user_info[user_id]["k_corr"][k] += answered_correctly
if answered_correctly:
user_info[user_id]["recent_corr"] = timestamp
import xgboost as xgb<choose_model_class>
def update_lec_data(user_id, content_id):
if watched_tags.get(str(user_id), -1)== -1:
watched_tags[str(user_id)] = {}
if user_info.get(user_id, -1)== -1:
add_user(user_id)
 user_info[user_id]["lecs_n"] += 1
lec_tag = lec_dict[content_id]
watched_tags[str(user_id)][str(lec_tag)] = 1
user_info[user_id]['last_lec'] = content_id
vote_est = [ 
('ada', AdaBoostClassifier()), 
('bc', BaggingClassifier()), 
('etc',ExtraTreesClassifier()), 
('gbc', GradientBoostingClassifier()), 
('rfc', RandomForestClassifier()), 
('gpc', GaussianProcessClassifier()), 
('lr', LogisticRegressionCV()), 
('bnb', BernoulliNB()), 
('gnb', GaussianNB()), 
('knn', KNeighborsClassifier()), 
('svc', SVC(probability=True)) , 
('xgb', xgb.XGBClassifier())
 ]
model = VotingClassifier(estimators = vote_est , voting = 'soft' )<compute_train_metric>
def non_lag_update(user_id, timestamp, elapsed, explan, lec):
timestamp = timestamp/8.64e+7
diff_timestamp_1 = timestamp - user_info[user_id]["first_timestamp"]
diff_timestamp_2 = timestamp - user_info[user_id]["second_timestamp"]
diff_timestamp_3 = timestamp - user_info[user_id]["third_timestamp"]
diff_timestamp_4 = timestamp - user_info[user_id]["fourth_timestamp"]
diff_timestamp_5 = timestamp - user_info[user_id]["fifth_timestamp"]
user_info[user_id]["fifth_timestamp"] = user_info[user_id]["fourth_timestamp"]
user_info[user_id]["fourth_timestamp"] = user_info[user_id]["third_timestamp"]
user_info[user_id]["third_timestamp"] = user_info[user_id]["second_timestamp"]
user_info[user_id]["second_timestamp"] = user_info[user_id]["first_timestamp"]
user_info[user_id]["first_timestamp"] = timestamp
if(user_info[user_id]["second_timestamp"] - user_info[user_id]["third_timestamp"])!= 0:
user_info[user_id]["ts_diff_shifted"] = user_info[user_id]["second_timestamp"]*8.64e+7 - user_info[user_id]["third_timestamp"]*8.64e+7
if(user_info[user_id]["second_timestamp"] - user_info[user_id]["fourth_timestamp"])!= 0:
user_info[user_id]["ts_diff_shifted_2"] = user_info[user_id]["second_timestamp"]*8.64e+7 - user_info[user_id]["fourth_timestamp"]*8.64e+7
if not isinstance(explan, pd._libs.missing.NAType)and explan == explan and not lec:
user_info[user_id]["priors_5"].append(explan)
 if diff_timestamp_1 > 0.083:
user_info[user_id]["sessions"] += 1
user_info[user_id]["session_count"] = 0
user_info[user_id]["sum_pauses"] += diff_timestamp_1
user_info[user_id]["session_count"] += 1
return diff_timestamp_1, diff_timestamp_2, diff_timestamp_3, diff_timestamp_4, diff_timestamp_5
print(cross_val_score(model, train, target, cv=5, scoring=make_scorer(accuracy_score)) )<predict_on_test>
def update_data(prior_group_answers_correct):
global tmp_data
arr = np.array(ast.literal_eval(prior_group_answers_correct))
for i, line in enumerate(tmp_data):
user_id = line[cols['user_id']]
content_type_id = line[cols['content_type_id']]
content_id = line[cols['content_id']]
timestamp = line[cols['timestamp']]
task_container_id = line[cols['task_container_id']]
explan = line[cols['prior_question_had_explanation']]
if isinstance(explan, pd._libs.missing.NAType)or explan == explan:
explan = 0
elapsed = line[cols['prior_question_elapsed_time']]
if isinstance(elapsed, pd._libs.missing.NAType)or elapsed == elapsed:
elapsed = 0
answered_correctly = arr[i]
user_arr = groups[user_id]
user_arr.insert(len(user_arr), answered_correctly)
 groups[user_id] = user_arr
if content_type_id == False:
if user_info.get(user_id, -1)== -1:
add_user(user_id)
 update_user(user_id, explan, elapsed, content_id, answered_correctly, timestamp)
 tmp_data = []
model.fit(train,target)
 pred = model.predict(test )<save_to_csv>
tmp_data = []
submission = pd.DataFrame({
"PassengerId": test_id,
"Survived": pred
})
 submission.to_csv('titanic_result_pandas.csv', index=False )<set_options>
def preprocess_1(chunk):
data_1 = chunk.values
out = np.zeros(( data_1.shape[0], len(features)))
 batch_counts = data_1[:, [cols["user_id"],cols["content_type_id"]]]
batch_counts = Counter(batch_counts[batch_counts[:, 1] == False][:, 0])
 global tmp_data
for i in range(data_1.shape[0]):
user_id, content_type_id, content_id ,prior_group_answers_correct, timestamp, task_container_id, explan, elapsed = get_meta_data(data_1[i])
 part = parts.get(content_id, -1)
 task_count = batch_counts[user_id]
if prior_group_answers_correct == prior_group_answers_correct and prior_group_answers_correct != '[]':
update_data(prior_group_answers_correct)
 tmp_data.append(data_1[i].tolist())
 if content_type_id:
update_lec_data(user_id, content_id)
 if user_info.get(user_id, -1)== -1:
add_user(user_id)
 user_info[user_id]["interaction_n"] += 1
_ = non_lag_update(user_id, timestamp, elapsed, explan, content_type_id)
 diff_timestamp_1, diff_timestamp_2, diff_timestamp_3, diff_timestamp_4, diff_timestamp_5 = _
if not content_type_id:
if not isinstance(elapsed, pd._libs.missing.NAType)and elapsed == elapsed:
user_info[user_id]["el_sum"] += elapsed
else:
elapsed = 0
if isinstance(explan, pd._libs.missing.NAType)or explan != explan:
explan = 0
if user_info[user_id]["count_2"] != 0:
last_part = user_info[user_id]["last_part"]
user_info[user_id]["part_et"][last_part-1] += elapsed
user_info[user_id]["part_explan"][last_part-1] += explan
user_info[user_id]["last_part"] = part
user_info[user_id]["part_count_2"][part-1] += 1
user_info[user_id]["count_2"] += 1
out[i, features_dict['content_id']] = content_id
out[i, features_dict['task_container_id']] = data_1[i, cols['task_container_id']]
out[i, features_dict['prior_question_elapsed_time']] = elapsed
out[i, features_dict['el_avg']] =(user_info[user_id]["el_sum"]/(user_info[user_id]["count_2"])) /1000
out[i, features_dict['prior_question_elapsed_time_u_part_avg']] =(user_info[user_id]["part_et"][part-1])/(user_info[user_id]["part_count_2"][part-1])
 out[i, features_dict['prior_question_had_explanation']] = explan
out[i, features_dict['prior_question_had_explanation_u_part_avg']] = user_info[user_id]["part_explan"][part-1]/(user_info[user_id]["part_count_2"][part-1]+1)
 out[i, features_dict['bundle_id']:] = np.array(list(questions.get(content_id ).values()))
out[i, features_dict['que_count_user']] = user_info.get(user_id, {} ).get('count',0)
 out[i, features_dict['question_repeated']] = repeated_que_count.get(user_id, {} ).get(content_id, 0)+ 1
m = user_info.get(user_id, {} ).get('mean_acc', 0)
 if m == 0:
out[i, features_dict['user_mean']] = 0.55
else:
out[i, features_dict['user_mean']] = m
out[i, features_dict['opp_mean']] = 1 - out[i, features_dict['user_mean']]
out[i, features_dict['last_lecture']] = user_info.get(user_id, {} ).get('last_lec',0)
 out[i, features_dict['time_diff']] = diff_timestamp_1
out[i, features_dict['time_diff1']] = diff_timestamp_2
out[i, features_dict['time_diff2']] = diff_timestamp_3
out[i, features_dict['time_diff3']] = diff_timestamp_4
out[i, features_dict['time_diff4']] = diff_timestamp_5
out[i, features_dict['timestamp']] = timestamp/8.64e+7
out[i, features_dict['correct_recency']] =(timestamp - user_info[user_id]["recent_corr"])/8.64e+7
kk = user_info[user_id]["priors_5"][-5:]
if len(kk)!= 0:
out[i, features_dict['rolling_mean_5_prior_question']] = np.array(kk ).mean()
out[i, features_dict['ts_diff_shifted']] = user_info[user_id]["ts_diff_shifted"]
out[i, features_dict['ts_diff_shifted_2']] = user_info[user_id]["ts_diff_shifted_2"]
out[i, features_dict['container_mean']] = containers_mean[task_container_id]
out[i, features_dict['lecs_per']] = user_info[user_id]["lecs_n"]/user_info[user_id]["interaction_n"]*100
out[i, features_dict['sessions']] = user_info[user_id]["sessions"]
out[i, features_dict['session_count']] = user_info[user_id]["session_count"]
if user_info[user_id]["count"] != 0:
out[i, features_dict['prior_question_had_explanation_ratio']] = user_info[user_id]["had_exp"]/user_info[user_id]["count"]
if user_info[user_id]["sessions"] != 0:
out[i, features_dict['mean_pause']] = user_info[user_id]["sum_pauses"]/user_info[user_id]["sessions"]
nn = user_info.get(user_id, {} ).get('easy_ct',0)
 if nn != 0:
out[i, features_dict['easy_ratio']] = user_info.get(user_id, {} ).get('easy_cr',0)/nn
out[i, features_dict['easy_ratio_opp']] = 1 - out[i, features_dict['easy_ratio']]
nn = user_info.get(user_id, {} ).get('hard_ct',0)
 if nn != 0:
out[i, features_dict['hard_ratio']] = user_info.get(user_id, {} ).get('hard_cr',0)/nn
out[i, features_dict['hard_ratio_opp']] = 1 - out[i, features_dict['hard_ratio']]
nn = user_info.get(user_id, {} ).get('part_count',[])
 if nn != []:
part = int(out[i, features_dict['part']] - 1)
 ct = nn[part]
cr = user_info[user_id]['part_corr'][part]
if ct != 0:
out[i, features_dict['part_mean']] = cr/ct
if groups.get(user_id, -1)!= -1 and groups[user_id] != []:
last_arr = np.array(groups[user_id][-40:])
 out[i, features_dict['rolling_mean']] = numpy_ewma_vectorized_v2(np.array(last_arr[-30:]),5)[-1]
last_arr = last_arr[last_arr != -1]
out[i, features_dict['ewm_mean_10']] = numpy_ewma_vectorized_v2(np.array(last_arr[-30:]),5)[-1]
out[i, features_dict['rolling_mean_10']] = np.array(last_arr[-10:] ).mean()
out[i, features_dict['rolling_mean_15']] = np.array(last_arr[-15:] ).mean()
out[i, features_dict['rolling_mean_5']] = np.array(last_arr[-5:] ).mean()
out[i, features_dict['prior_part_mean']] = prior_part_mean_dict[int(out[i, features_dict['part']])]
k = que_2_k[content_id]
out[i, features_dict['k']] = k
if user_info[user_id]["k_count"][k] != 0:
out[i, features_dict['k_acc']] = user_info[user_id]["k_corr"][k]/user_info[user_id]["k_count"][k]
out[i, features_dict['wut']] = user_info[user_id]["ts_diff_shifted"] - elapsed*task_count
if out[i, features_dict['wut']] < 0:
out[i, features_dict['wut']] = 0
usr = watched_tags.get(str(user_id), -1)
 if usr != -1:
nn = 0
for k in range(6):
nn += usr.get(str(int(questions1[content_id]["tags"+str(k+1)])) , 0)
 out[i, features_dict['watched']] = nn
return out
%matplotlib inline
warnings.filterwarnings('ignore' )<predict_on_test>
model_lgbm = lgb.Booster(model_file='.. /input/lgbm-test/lgb_classifier.txt')
 stack_lgbm = lgb.Booster(model_file='.. /input/lgbm-test/lgb_stack.txt' )
def fill_missing_data_basic_reg(X_train, y_train, X_test):
rf_reg_est = ensemble.RandomForestRegressor(random_state = 42)
 rf_reg_est.fit(X_train, y_train)
 y_test = rf_reg_est.predict(X_test)
 return y_test
def fill_missing_data_basic_class(X_train, y_train, X_test):
rf_class_est = ensemble.RandomForestClassifier(random_state = 42)
 rf_class_est.fit(X_train, y_train)
 y_test = rf_class_est.predict(X_test)
 return y_test<load_from_csv>
vals = 0
<count_missing_values><EOS> 
%%time
for(test_data,sample_prediction_df)in iter_test:
if not isinstance(vals, int):
if test_data.iloc[0].prior_group_answers_correct == test_data.iloc[0].prior_group_answers_correct:
past_vals = np.array(ast.literal_eval(test_data.iloc[0].prior_group_answers_correct))
past_answers = np.array(ast.literal_eval(test_data.iloc[0].prior_group_responses))
past_vals = np.concatenate(( vals, past_vals.reshape(len(past_vals),1)) , axis=1)
 past_vals = np.concatenate(( past_vals, past_answers.reshape(len(past_answers),1)) , axis=1)
 update_group_var(past_vals)
 vals = test_data[["user_id","content_id", "content_type_id", "task_container_id","timestamp","prior_question_elapsed_time"]].values
test_transform = preprocess_1(test_data)
 lgbm_predic = model_lgbm.predict(test_transform)
 st_predic = np.array(split_preds(vals))
test_data['answered_correctly'] =(lgbm_predic*0.85 + 1.15*st_predic)/2
env.predict(test_data.loc[test_data['content_type_id'] == 0, ['row_id', 'answered_correctly']] )