Kaggle competition: Titanic - Machine Learning from Disaster
def read_and_preprocess(feature_engineering = False, n_split = 3):
train_pickle = '.. /input/riiid-cross-validation-files/cv1_train.pickle'
valid_pickle = '.. /input/riiid-cross-validation-files/cv1_valid.pickle'
question_metadata_file = '.. /input/question-metadate-new-new/question_metadata_new_new.csv'
question_data_file = '.. /input/riiid-test-answer-prediction/questions.csv'
feld_needed = ['timestamp',
'user_id',
'answered_correctly',
'content_id',
'content_type_id',
'prior_question_elapsed_time',
'prior_question_had_explanation',
'task_container_id']
train = pd.read_pickle(train_pickle)[feld_needed]
valid = pd.read_pickle(valid_pickle)[feld_needed]
if feature_engineering:
train = train.iloc[-1000000:]
valid = valid.iloc[-1000000:]
train = train.loc[train.content_type_id == False].reset_index(drop = True)
 valid = valid.loc[valid.content_type_id == False].reset_index(drop = True)
 print('train size: ')
 print(train.shape)
 print('valid size: ')
 print(valid.shape)
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8')
 prior_question_elapsed_time_mean = train['prior_question_elapsed_time'].dropna().mean()
train['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)
 valid['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)
 questions_df = pd.read_csv(question_metadata_file)
 questions_df_ = pd.read_csv(question_data_file)
 questions_df_.rename(columns = {'question_id': 'content_id'}, inplace = True)
 questions_df = questions_df.merge(questions_df_[['content_id', 'tags']], on = 'content_id', how = 'left')
 del questions_df_
gc.collect()
train = train.merge(questions_df[['content_id', 'tags', 'content_field', 'mean_question_accuracy']], on = 'content_id', how = 'left')
 valid = valid.merge(questions_df[['content_id', 'tags', 'content_field', 'mean_question_accuracy']], on = 'content_id', how = 'left')
 answered_correctly_u_count = defaultdict(int)
 answered_correctly_u_sum = defaultdict(int)
 elapsed_time_u_sum = defaultdict(int)
 explanation_u_sum = defaultdict(int)
 timestamp_u = defaultdict(list)
 timestamp_u_incorrect = defaultdict(list)
 latest_u_theta = defaultdict(np.float32)
 answered_correctly_u_count_field = {}
answered_correctly_u_sum_field = {}
answered_correctly_difficulty_weighted_sum = defaultdict(np.float32)
 answered_correctly_difficulty_weighted_sum_field = {}
max_solved_difficulty = defaultdict(np.float32)
 max_solved_difficulty_field = {}
min_wrong_difficulty = defaultdict(np.float32)
 min_wrong_difficulty_field = {}
session_time = defaultdict(np.float32)
 since_last_session_time = defaultdict(np.float32)
 last_session_start_time = defaultdict(np.float32)
 first_action_time = defaultdict(np.float32)
 answered_correctly_q_count = defaultdict(int)
 answered_correctly_q_sum = defaultdict(int)
 elapsed_time_q_sum = defaultdict(int)
 explanation_q_sum = defaultdict(int)
 explanation_qtrue_sum = defaultdict(int)
 explanation_qtrue_count = defaultdict(int)
 latest_q_beta = defaultdict(np.float32)
 answered_correctly_uq = {}
question_avg_sum_u = defaultdict(np.float32)
 task_container_sum = defaultdict(int)
 task_container_count = defaultdict(int)
 tags_sum = defaultdict(int)
 tags_count = defaultdict(int)
 l = int(len(train)// n_split)
 for n in range(n_split - 1):
train.iloc[int(l * n): int(l *(n + 1)) ].to_csv('train_%s.csv' % str(n))
train.iloc[int(l *(n_split - 1)) :].to_csv('train_%s.csv' % str(n+1))
del train
gc.collect()
print('User feature calculation started...')
 print('
')
 for n in range(n_split):
train = pd.read_csv('train_%s.csv' % str(n))
print(train.shape)
 add_train_features(train,
answered_correctly_u_count,
answered_correctly_u_sum,
elapsed_time_u_sum,
explanation_u_sum,
timestamp_u,
timestamp_u_incorrect,
latest_u_theta,
answered_correctly_q_count,
answered_correctly_q_sum,
elapsed_time_q_sum,
explanation_q_sum,
explanation_qtrue_sum,
explanation_qtrue_count,
latest_q_beta,
answered_correctly_uq,
question_avg_sum_u,
task_container_sum,
task_container_count,
tags_sum,
tags_count,
answered_correctly_u_count_field,
answered_correctly_u_sum_field,
answered_correctly_difficulty_weighted_sum,
answered_correctly_difficulty_weighted_sum_field,
max_solved_difficulty,
max_solved_difficulty_field,
min_wrong_difficulty,
min_wrong_difficulty_field,
session_time,
since_last_session_time,
last_session_start_time,
first_action_time, 
)
 gc.collect()
del train
gc.collect()
print(n + 1, '/', n_split)
 print('Finish train feature calculation!')
 add_train_features(valid,
answered_correctly_u_count,
answered_correctly_u_sum,
elapsed_time_u_sum,
explanation_u_sum,
timestamp_u,
timestamp_u_incorrect,
latest_u_theta,
answered_correctly_q_count,
answered_correctly_q_sum,
elapsed_time_q_sum,
explanation_q_sum,
explanation_qtrue_sum,
explanation_qtrue_count,
latest_q_beta,
answered_correctly_uq,
question_avg_sum_u,
task_container_sum,
task_container_count,
tags_sum,
tags_count,
answered_correctly_u_count_field,
answered_correctly_u_sum_field,
answered_correctly_difficulty_weighted_sum,
answered_correctly_difficulty_weighted_sum_field,
max_solved_difficulty,
max_solved_difficulty_field,
min_wrong_difficulty,
min_wrong_difficulty_field,
session_time,
since_last_session_time,
last_session_start_time,
first_action_time, 
)
 print('Finish valid feature calculation!')
 del valid
gc.collect()
print('User feature calculation completed...')
 print('
')
 features_dicts = {
'answered_correctly_u_count': answered_correctly_u_count,
'answered_correctly_u_sum': answered_correctly_u_sum,
'elapsed_time_u_sum': elapsed_time_u_sum,
'explanation_u_sum': explanation_u_sum,
'timestamp_u': timestamp_u,
'timestamp_u_incorrect': timestamp_u_incorrect,
'latest_u_theta': latest_u_theta,
'answered_correctly_q_count': answered_correctly_q_count,
'answered_correctly_q_sum': answered_correctly_q_sum,
'elapsed_time_q_sum': elapsed_time_q_sum,
'explanation_q_sum': explanation_q_sum,
'explanation_qtrue_sum': explanation_qtrue_sum,
'explanation_qtrue_count': explanation_qtrue_count,
'latest_q_beta': latest_q_beta,
'answered_correctly_uq': answered_correctly_uq,
'question_avg_sum_u': question_avg_sum_u,
'task_container_sum': task_container_sum,
'task_container_count': task_container_count,
'tags_sum': tags_sum,
'tags_count': tags_count,
'answered_correctly_u_count_field': answered_correctly_u_count_field,
'answered_correctly_u_sum_field': answered_correctly_u_sum_field,
'answered_correctly_difficulty_weighted_sum': answered_correctly_difficulty_weighted_sum,
'answered_correctly_difficulty_weighted_sum_field': answered_correctly_difficulty_weighted_sum_field,
'max_solved_difficulty': max_solved_difficulty,
'max_solved_difficulty_field': max_solved_difficulty_field,
'min_wrong_difficulty': min_wrong_difficulty,
'min_wrong_difficulty_field': min_wrong_difficulty_field,
'session_time': session_time,
'since_last_session_time': since_last_session_time,
'last_session_start_time': last_session_start_time,
'first_action_time': first_action_time,
}
questions_df['tags'] = questions_df['tags'].\
apply(lambda ts: [int(x)for x in str(ts ).split() if x != 'nan'])
 return questions_df, prior_question_elapsed_time_mean, features_dicts<load_pretrained>
def feature_engg_train(df):
df['Title'] = df['Name'].map(lambda name:name.split('.')[0].split(',')[1].strip())
 titles_dict_train = {}
for title in ['Capt','Col','Major']:
titles_dict_train[title] = 'Officer'
for title in ['Rev','Dr']:
titles_dict_train[title] = 'Other'
for title in ['Don','the Countess','Jonkheer','Lady','Sir']:
titles_dict_train[title] = 'Royalty'
for title in ['Ms','Miss','Mlle']:
titles_dict_train[title] = 'Miss'
for title in ['Mrs','Mme']:
titles_dict_train[title] = 'Mrs'
titles_dict_train['Mr'] = 'Mr'
titles_dict_train['Master'] = 'Master'
df['Title'] = df['Title'].map(titles_dict_train)
 title_dummies = pd.get_dummies(df['Title'],prefix='Title')
 df = pd.concat([df,title_dummies],axis=1)
 grouped_sex_class = df.groupby(['Sex','Pclass','Title'])
 def impute_median(series):
return(series.fillna(series.median()))
df['Age'] = grouped_sex_class['Age'].transform(impute_median)
 df['Fare'].fillna(df['Fare'].mean() ,inplace=True)
 df['Embarked'].fillna('S',inplace=True)
 embarked_dummies = pd.get_dummies(df['Embarked'],prefix='Embarked')
 df = pd.concat([df,embarked_dummies],axis=1)
 df['Cabin'].fillna('X',inplace=True)
 df['Cabin_clean'] = df['Cabin'].apply(lambda x: x[0])
 cabin_dummies = pd.get_dummies(df['Cabin_clean'],prefix='Cabin')
 df = pd.concat([df,cabin_dummies],axis=1)
 sex_dict = dict(zip(df['Sex'].unique() ,range(0,2)))
 df['Sex'] = df['Sex'].map(sex_dict)
 pclass_dummies = pd.get_dummies(df['Pclass'],prefix='Pclass')
 df = pd.concat([df,pclass_dummies],axis=1)
 df['Family'] = df['SibSp'] + df['Parch'] + 1
df['Family_single'] = df['Family'].map(lambda x: 1 if x==1 else 0)
 df['Family_small'] = df['Family'].map(lambda x: 1 if 2<=x<=4 else 0)
 df['Family_large'] = df['Family'].map(lambda x: 1 if x>=4 else 0)
 df.drop(['Name','Pclass','Ticket','Cabin','Cabin_clean',\
'Embarked','Title'],axis=1,inplace=True)
 return(df)
 def feature_engg_test(df):
df['Title'] = df['Name'].map(lambda name:name.split('.')[0].split(',')[1].strip())
 titles_dict_test = { 'Col': 'Officer',
'Dona': 'Royalty','Dr': 'Other', 'Master': 'Master', 'Miss': 'Miss',\
'Mr': 'Mr', 'Mrs': 'Mrs', 'Ms': 'Miss', 'Rev': 'Other'}
df['Title'] = df['Title'].map(titles_dict_test)
 test_dummies_title = pd.get_dummies(df['Title'],prefix='Title')
 df = pd.concat([df,test_dummies_title],axis=1)
 df['Fare'].fillna(df['Fare'].mean() ,inplace=True)
 grouped_sex_class_test = df.groupby(['Sex','Pclass','Title'])
 def impute_median(series):
return series.fillna(series.median())
 df['Age'] = grouped_sex_class_test['Age'].transform(impute_median)
 df['Embarked'].fillna('S',inplace=True)
 test_dummies_embarked = pd.get_dummies(df['Embarked'],prefix='Embarked')
 df = pd.concat([df,test_dummies_embarked],axis=1)
 df['Cabin'].fillna('X',inplace=True)
 df['Cabin_clean'] = df['Cabin'].apply(lambda x:x[0])
 test_dummies_cabin = pd.get_dummies(df['Cabin_clean'],prefix='Cabin')
 df = pd.concat([df,test_dummies_cabin],axis=1)
 sex_dict = dict(zip(df['Sex'].unique() ,range(0,2)))
 df['Sex'] = df['Sex'].map(sex_dict)
 test_dummies_pclass = pd.get_dummies(df['Pclass'],prefix='Pclass')
 df = pd.concat([df,test_dummies_pclass],axis=1)
 df['Family'] = df['Parch'] + df['SibSp'] + 1
df['Family_single'] = df['Family'].map(lambda x: 1 if x==1 else 0)
 df['Family_small'] = df['Family'].map(lambda x: 1 if 2<=x<4 else 0)
 df['Family_large'] = df['Family'].map(lambda x: 1 if x>=4 else 0)
 df.drop(['Name','Pclass','Ticket','Cabin','Cabin_clean',\
'Embarked','Title'],axis=1,inplace=True)
 return(df)
questions_df, prior_question_elapsed_time_mean, features_dicts = read_and_preprocess(feature_engineering = False, n_split = 25)
 LGBM_model = lgb.Booster(model_file = '.. /input/lgbm-v921-01/lgbm_model_V921_0.1.lgb' )<define_variables>
%matplotlib inline
sns.set_style('white')
answered_correctly_u_count = features_dicts['answered_correctly_u_count']
answered_correctly_u_sum = features_dicts['answered_correctly_u_sum']
elapsed_time_u_sum = features_dicts['elapsed_time_u_sum']
explanation_u_sum = features_dicts['explanation_u_sum']
answered_correctly_q_count = features_dicts['answered_correctly_q_count']
answered_correctly_q_sum = features_dicts['answered_correctly_q_sum']
elapsed_time_q_sum = features_dicts['elapsed_time_q_sum']
explanation_q_sum = features_dicts['explanation_q_sum']
explanation_qtrue_sum = features_dicts['explanation_qtrue_sum']
explanation_qtrue_count = features_dicts['explanation_qtrue_count']
answered_correctly_uq = features_dicts['answered_correctly_uq']
timestamp_u = features_dicts['timestamp_u']
timestamp_u_incorrect = features_dicts['timestamp_u_incorrect']
question_avg_sum_u = features_dicts['question_avg_sum_u']
task_container_sum = features_dicts['task_container_sum']
task_container_count = features_dicts['task_container_count']
latest_u_theta = features_dicts['latest_u_theta']
latest_q_beta = features_dicts['latest_q_beta']
tags_sum = features_dicts['tags_sum']
tags_count = features_dicts['tags_count']
answered_correctly_u_count_field = features_dicts['answered_correctly_u_count_field']
answered_correctly_u_sum_field = features_dicts['answered_correctly_u_sum_field']
answered_correctly_difficulty_weighted_sum = features_dicts['answered_correctly_difficulty_weighted_sum']
answered_correctly_difficulty_weighted_sum_field = features_dicts['answered_correctly_difficulty_weighted_sum_field']
max_solved_difficulty = features_dicts['max_solved_difficulty']
max_solved_difficulty_field = features_dicts['max_solved_difficulty_field']
min_wrong_difficulty = features_dicts['min_wrong_difficulty']
min_wrong_difficulty_field = features_dicts['min_wrong_difficulty_field']
session_time = features_dicts['session_time']
since_last_session_time = features_dicts['since_last_session_time']
last_session_start_time = features_dicts['last_session_start_time']
first_action_time =features_dicts['first_action_time']
del features_dicts
gc.collect()<init_hyperparams>
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )
MAX_SEQ = 240
ACCEPTED_USER_CONTENT_SIZE = 2
EMBED_SIZE = 256
BATCH_SIZE = 64+32
DROPOUT = 0.1
n_skill = 13523
print(n_skill)
 group = joblib.load(".. /input/new-sakt-dataset/group.pkl.zip")
 class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class FFN0(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN0, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.layer_normal = nn.LayerNorm(state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 x=self.layer_normal(x)
 return self.dropout(x)
 def future_mask(seq_length):
future_mask =(np.triu(np.ones([seq_length, seq_length]), k = 1)).astype('bool')
 return torch.from_numpy(future_mask)
 class TransformerBlock(nn.Module):
def __init__(self, embed_dim, heads = 8, dropout = DROPOUT, forward_expansion = 1):
super(TransformerBlock, self ).__init__()
self.multi_att = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=heads, dropout=dropout)
 self.dropout = nn.Dropout(dropout)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.ffn0 = FFN0(embed_dim, forward_expansion = forward_expansion, dropout=dropout)
 self.layer_normal_2 = nn.LayerNorm(embed_dim)
 def forward(self, value, key, query, att_mask):
att_output, att_weight = self.multi_att(value, key, query, attn_mask=att_mask)
 att_output = self.dropout(self.layer_normal(att_output + value))
att_output = att_output.permute(1, 0, 2)
 x = self.ffn(att_output)
 x1 = self.ffn0(att_output)
 x = self.dropout(self.layer_normal_2(x + x1 + att_output))
return x.squeeze(-1), att_weight
class Encoder(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, num_layers=1, heads = 8):
super(Encoder, self ).__init__()
self.n_skill, self.embed_dim = n_skill, embed_dim
self.embedding = nn.Embedding(2 * n_skill + 1, embed_dim)
 self.pos_embedding = nn.Embedding(max_seq - 1, embed_dim)
 self.e_embedding = nn.Embedding(n_skill+1, embed_dim)
 self.layers = nn.ModuleList([TransformerBlock(embed_dim, forward_expansion = forward_expansion)for _ in range(num_layers)])
 self.dropout = nn.Dropout(dropout)
 def forward(self, x, question_ids):
device = x.device
x = self.embedding(x)
 pos_id = torch.arange(x.size(1)).unsqueeze(0 ).to(device)
 pos_x = self.pos_embedding(pos_id)
 x = self.dropout(x + pos_x)
 x = x.permute(1, 0, 2)
 e = self.e_embedding(question_ids)
 e = e.permute(1, 0, 2)
 for layer in self.layers:
att_mask = future_mask(e.size(0)).to(device)
 x, att_weight = layer(e, x, x, att_mask=att_mask)
 x = x.permute(1, 0, 2)
 x = x.permute(1, 0, 2)
 return x, att_weight
class SAKTModel(nn.Module):
def __init__(self, n_skill, max_seq=100, embed_dim=128, dropout = DROPOUT, forward_expansion = 1, enc_layers=1, heads = 8):
super(SAKTModel, self ).__init__()
self.encoder = Encoder(n_skill, max_seq, embed_dim, dropout, forward_expansion, num_layers=enc_layers)
 self.pred = nn.Linear(embed_dim, 1)
 def forward(self, x, question_ids):
x, att_weight = self.encoder(x, question_ids)
 x = self.pred(x)
 return x.squeeze(-1), att_weight
class TestDataset(Dataset):
def __init__(self, samples, test_df, n_skill, max_seq=100):
super(TestDataset, self ).__init__()
self.samples, self.user_ids, self.test_df = samples, [x for x in test_df["user_id"].unique() ], test_df
self.n_skill, self.max_seq = n_skill, max_seq
def __len__(self):
return self.test_df.shape[0]
def __getitem__(self, index):
test_info = self.test_df.iloc[index]
user_id = test_info['user_id']
target_id = test_info['content_id']
content_id_seq = np.zeros(self.max_seq, dtype=int)
 answered_correctly_seq = np.zeros(self.max_seq, dtype=int)
 if user_id in self.samples.index:
content_id, answered_correctly = self.samples[user_id]
seq_len = len(content_id)
 if seq_len >= self.max_seq:
content_id_seq = content_id[-self.max_seq:]
answered_correctly_seq = answered_correctly[-self.max_seq:]
else:
content_id_seq[-seq_len:] = content_id
answered_correctly_seq[-seq_len:] = answered_correctly
x = content_id_seq[1:].copy()
x +=(answered_correctly_seq[1:] == 1)* self.n_skill
questions = np.append(content_id_seq[2:], [target_id])
 return x, questions
def create_model() :
return SAKTModel(n_skill, max_seq=MAX_SEQ, embed_dim=EMBED_SIZE, forward_expansion=1, enc_layers=1, heads=4, dropout=0.1)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 MODEL_PATH = '.. /input/new-sakt-dataset/sakt_model.pt'
SAKT_model = create_model()
SAKT_model.load_state_dict(torch.load(MODEL_PATH, map_location='cpu'))
SAKT_model.to(device )<predict_on_test>
df_train = feature_engg_train(df_train)
 df_test = feature_engg_test(df_test)
 features_train = df_train.drop(['Survived','Cabin_T'],axis=1)
 target_train = df_train['Survived']
features_test = df_test
env = riiideducation.make_env()
iter_test = env.iter_test()
set_predict = env.predict<feature_engineering>
clf=RandomForestClassifier(n_estimators=100,random_state=42)
 clf=clf.fit(features_train,target_train )
%%time
w = 0.225
previous_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if previous_test_df is not None:
previous_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 update_features(previous_test_df,
answered_correctly_u_sum,
answered_correctly_q_sum,
timestamp_u_incorrect,
explanation_qtrue_sum,
task_container_sum,
latest_u_theta,
latest_q_beta,
answered_correctly_u_count,
answered_correctly_q_count,
tags_sum,
answered_correctly_u_sum_field,
answered_correctly_difficulty_weighted_sum,
answered_correctly_difficulty_weighted_sum_field,
max_solved_difficulty,
max_solved_difficulty_field,
min_wrong_difficulty,
min_wrong_difficulty_field)
 prev_group = previous_test_df[previous_test_df['content_type_id'] == 0][['user_id', 'content_id', 'answered_correctly']].\
groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1] 
)
 test_df['prior_question_had_explanation'] = test_df.prior_question_had_explanation.\
fillna(False ).astype('int8')
 test_df['prior_question_elapsed_time'].fillna(prior_question_elapsed_time_mean, inplace = True)
 test_df = test_df.merge(questions_df, on = 'content_id', how = 'left')
 previous_test_df = test_df.copy()
test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop = True)
 test_df[TARGET] = 1/4
test_df = add_features(test_df,
answered_correctly_u_count,
answered_correctly_u_sum,
elapsed_time_u_sum,
explanation_u_sum,
timestamp_u,
timestamp_u_incorrect,
latest_u_theta,
answered_correctly_q_count,
answered_correctly_q_sum,
elapsed_time_q_sum,
explanation_q_sum,
explanation_qtrue_sum,
explanation_qtrue_count,
latest_q_beta,
answered_correctly_uq,
question_avg_sum_u,
task_container_sum,
task_container_count,
tags_sum,
tags_count,
answered_correctly_u_count_field,
answered_correctly_u_sum_field,
answered_correctly_difficulty_weighted_sum,
answered_correctly_difficulty_weighted_sum_field,
max_solved_difficulty,
max_solved_difficulty_field,
min_wrong_difficulty,
min_wrong_difficulty_field,
session_time,
since_last_session_time,
last_session_start_time,
first_action_time,
update = False)
 test_df[TARGET] = LGBM_model.predict(test_df[FEATURES])
 test_dataset = TestDataset(group, test_df, n_skill, max_seq=MAX_SEQ)
 test_dataloader = DataLoader(test_dataset, batch_size=len(test_df), shuffle=False)
 SAKT_outs = []
for item in test_dataloader:
x = item[0].to(device ).long()
target_id = item[1].to(device ).long()
with torch.no_grad() :
output, att_weight = SAKT_model(x, target_id)
 output = torch.sigmoid(output)
 output = output[:, -1]
SAKT_outs.extend(output.view(-1 ).data.cpu().numpy())
 test_df[TARGET] = test_df[TARGET] *(1 - w)+ np.array(SAKT_outs)* w
set_predict(test_df[['row_id', TARGET]])
 print('Job Done' )<define_variables>
model = SelectFromModel(clf, prefit=True,threshold='median')
 train_reduced = model.transform(features_train)
 train_reduced.shape
DEFAULT_SEED = 42
PLOT_SHAP = False
MAX_QUESTIONS = 14000
VAL_SIZE = 2500000
USE_DATA_RATIO = 0.5
TS_SCALING = 1000*3600
LEARNING_RATE = 0.1
MAX_BIN = 364
NUM_LEAVES = 445
FEATURE_FRACTION = 0.639
BAGGING_FRACTION = 0.842
BAGGING_FREQ = 19
NUM_BOOST_ROUNDS = 10000
EARLY_STOP_ROUNDS = 20
VERBOSE_EVAL = 50
TRAIN_FILE_PATH = ".. /input/riiid-test-answer-prediction/train.csv"
QUESTIONS_FILE_PATH = ".. /input/riiid-test-answer-prediction/questions.csv"
LECTURES_FILE_PATH = ".. /input/riiid-test-answer-prediction/lectures.csv"
@contextmanager
def trace_mem(title):
t0 = time()
p = psutil.Process(os.getpid())
 m0 = p.memory_info() [0] / 2.** 30
yield
m1 = p.memory_info() [0] / 2.** 30
delta = m1 - m0
sign = '+' if delta >= 0 else '-'
delta = math.fabs(delta)
 print(f"[{m1:.1f}GB({sign}{delta:.2f}GB): {time() - t0:.1f}s] {title} ", file=sys.stderr)
 def seed_everything(seed=DEFAULT_SEED):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 seed_everything()
TRA_FEATURES = [
'prior_question_elapsed_time',
'prior_question_had_explanation',
'bundle_id',
'part',
'tags_encoded',
'tags_first1',
'tags_last2',
'content_count',
'content_correctness',
'user_correctness',
'user_correct_cumsum',
'residual_user_mean',
'pq_elapsed_time_user_mean',
'explanation_user_cumsum',
'lag_time',
'lag_time2',
'user_content_attempted',
]
cat_features = ['bundle_id', 'part', 'tags_encoded', 'tags_first1', 'tags_last2'] 
<data_type_conversions>
X_train,X_test,y_train,y_test = train_test_split(train_reduced,target_train,random_state=42)
 param_grid_rf = {'n_estimators':[100,500],'min_samples_split':[2,5],\
'max_depth':[5,10],'min_samples_leaf':[5,10]}
grid_rf = GridSearchCV(estimator=RandomForestClassifier(random_state=42, oob_score=True, warm_start=True),param_grid=param_grid_rf,cv=10)
 grid_rf.fit(X_train,y_train )
target_col = 'answered_correctly'
data_types_dict = {
'row_id': 'int64',
'timestamp': 'int64',
'user_id': 'int32',
'content_id': 'int16',
'content_type_id': 'int8',
'task_container_id': 'int16',
'answered_correctly': 'int8',
'prior_question_elapsed_time': 'float32',
'prior_question_had_explanation': 'int8'
}
with trace_mem("Loading train file, fillna, astype"):
df_train = dt.fread(TRAIN_FILE_PATH, columns=set(data_types_dict.keys())).to_pandas()
df_train['prior_question_had_explanation'].fillna(False, inplace=True)
 mean_prior_question_elapsed_time = df_train['prior_question_elapsed_time'].mean()
df_train['prior_question_elapsed_time'].fillna(mean_prior_question_elapsed_time, inplace=True)
 assert(0 == df_train.isnull().sum().sum())
 df_train = df_train.astype(data_types_dict)
 del data_types_dict
gc.collect()
with trace_mem("df_train drop lecture rows"):
df_train = df_train[df_train['content_type_id'] == 0].reset_index(drop=True)
 with trace_mem("df_train drop useless columns"):
df_train.drop(columns=['content_type_id'], inplace=True)
 with trace_mem("Generate feature 'lag_time2'"):
df_train['user_id_copy'] = df_train['user_id']
timestamp_groupby_user_task = df_train[['user_id', 'task_container_id', 'row_id', 'user_id_copy', 'timestamp']].\
groupby(['user_id', 'task_container_id'], sort=False)['row_id', 'user_id_copy', 'timestamp']
ts_user_task_agg = timestamp_groupby_user_task.first()
del timestamp_groupby_user_task
gc.collect()
ts_user_task_agg.set_index('row_id', inplace=True)
 df_train.set_index('row_id', inplace=True)
 ts_user_task_agg['timestamp'] = ts_user_task_agg['timestamp']/TS_SCALING
ts_groupby = ts_user_task_agg.groupby('user_id_copy')['timestamp']
ts_user_task_agg['timestamp_1'] = ts_groupby.shift(1)
 ts_user_task_agg['timestamp_1'].fillna(0, inplace=True)
 ts_user_task_agg['timestamp_1'] = ts_user_task_agg['timestamp_1'].astype('float32')
 user_max_timestamp_2_agg = ts_user_task_agg.groupby(['user_id_copy'])['timestamp_1'].max()
ts_user_task_agg.drop(columns=['timestamp_1'], inplace=True)
 ts_user_task_agg['timestamp'] = ts_groupby.shift(2)
 ts_user_task_agg['timestamp'].fillna(0, inplace=True)
 df_train.drop(columns=['user_id_copy'], inplace=True)
 del ts_groupby
gc.collect()
df_train['lag_time2'] = 0
df_train.loc[ts_user_task_agg.index, 'lag_time2'] = ts_user_task_agg['timestamp']
ts_user_task_agg.drop(columns=['user_id_copy', 'timestamp'], inplace=True)
 del ts_user_task_agg
gc.collect()
df_train['lag_time2'] = df_train['timestamp']/TS_SCALING - df_train['lag_time2']
df_train['lag_time2'] = df_train['lag_time2'].astype('float32')
 lag2_groupby_user_task = df_train[['user_id', 'task_container_id', 'lag_time2']].\
groupby(['user_id', 'task_container_id'])['lag_time2']
df_train['lag_time2'] = lag2_groupby_user_task.transform('first')
 del lag2_groupby_user_task
gc.collect()
lag2_groupby_user = df_train[['user_id', 'lag_time2']].groupby('user_id')['lag_time2']
user_last_lagtime_2_agg = lag2_groupby_user.last()
del lag2_groupby_user
gc.collect()
with trace_mem("Calculate Train/Val splitting via 'timestamp'"):
user_max_timestamp = df_train[['user_id', 'timestamp']].groupby(['user_id'] ).agg(['max'] ).reset_index()
user_max_timestamp.columns = ['user_id', 'max_time_stamp']
MAX_TIME_STAMP = user_max_timestamp['max_time_stamp'].max()
def rand_time(max_time_stamp):
interval = MAX_TIME_STAMP - max_time_stamp
rand_time_stamp = random.randint(0, interval)
 return rand_time_stamp
user_max_timestamp['rand_time_stamp'] = user_max_timestamp['max_time_stamp'].apply(rand_time)
 user_max_timestamp.drop(columns=['max_time_stamp'], inplace=True)
 df_train = df_train.merge(user_max_timestamp, on='user_id', how='left')
 del user_max_timestamp
df_train['virtual_time_stamp'] = df_train['timestamp'] + df_train['rand_time_stamp']
ts_arr =(df_train['timestamp']/TS_SCALING ).to_numpy()
df_train.drop(columns=['timestamp', 'rand_time_stamp'], inplace=True)
 df_train['val'] = pd.Series([0]*len(df_train), dtype=np.int8)
 use_data_size = int(len(df_train)*USE_DATA_RATIO)
 print(f"Use {USE_DATA_RATIO:.0%} of train data: {use_data_size} rows")
 df_train.iloc[df_train['virtual_time_stamp'].nlargest(use_data_size ).index, -1] = 1
df_train.iloc[df_train['virtual_time_stamp'].nlargest(VAL_SIZE ).index, -1] = 2
df_train.drop(columns=['virtual_time_stamp'], inplace=True)
 gc.collect()

 with trace_mem("Generate feature 'lag_time'"):
df_train['timestamp'] = ts_arr
del ts_arr
gc.collect()
timestamp_groupby_user = df_train[['user_id', 'timestamp']].groupby('user_id')
 df_train['lag_time'] = timestamp_groupby_user['timestamp'].shift(1)
 user_max_timestamp_1_agg = timestamp_groupby_user['timestamp'].max().astype('float32')
 del timestamp_groupby_user
gc.collect()
df_train['lag_time'].fillna(0, inplace=True)
 df_train['lag_time'] = df_train['timestamp'] - df_train['lag_time']
df_train['lag_time'] = df_train['lag_time'].astype('float32')
 lagtime_groupby = df_train[['user_id', 'task_container_id', 'lag_time']].\
groupby(['user_id', 'task_container_id'])
 df_train['lag_time'] = lagtime_groupby['lag_time'].transform('first')
 df_train.drop(columns=['timestamp'], inplace=True)
 del lagtime_groupby
gc.collect()
lag_task_groupby_user = df_train[['user_id', 'lag_time', 'task_container_id']].groupby('user_id')
 user_last_lagtime_1_agg = lag_task_groupby_user['lag_time'].last()
user_last_task_agg = lag_task_groupby_user['task_container_id'].last()
df_train.drop(columns=['task_container_id'], inplace=True)
 del lag_task_groupby_user
gc.collect()
with trace_mem("Generate user_content_attempted_dict"):
user_content_attempted_dict = dict()
for _user, _content in zip(df_train['user_id'].to_numpy() , df_train['content_id'].to_numpy()):
if _user not in user_content_attempted_dict:
a = bitarray(MAX_QUESTIONS, endian='little')
 a.setall(False)
 a[_content] = True
user_content_attempted_dict[_user] = a
else:
user_content_attempted_dict[_user][_content] = True
del a
gc.collect()
def get_user_content_attempted(_user_id, _content_id):
if _user_id in user_content_attempted_dict:
_attempted = user_content_attempted_dict[_user_id][_content_id]
if not _attempted:
user_content_attempted_dict[_user_id][_content_id] = True
return _attempted
else:
_a = bitarray(MAX_QUESTIONS, endian='little')
 _a.setall(False)
 _a[_content_id] = True
user_content_attempted_dict[_user_id] = _a
return False
with trace_mem("Generate feature 'user_content_attempted'"):
df_train["user_content_attempted"] = pd.Series([1]*len(df_train), dtype=np.int8)
 df_train["user_content_attempted"] = df_train.loc[:, ["user_id", "content_id", 'user_content_attempted']].\
groupby(["user_id", "content_id"])["user_content_attempted"].cumsum().clip(0, 2)- 1
with trace_mem("Calculate aggregating values"):
groupby_user = df_train[['user_id', 'prior_question_elapsed_time', target_col]].groupby('user_id')
 groupby_content = df_train[['content_id', target_col, 'prior_question_elapsed_time',
'prior_question_had_explanation']].groupby('content_id')
 user_ques_elapsed_time_sum_agg = groupby_user['prior_question_elapsed_time'].sum()
user_target_sum_agg = groupby_user[target_col].sum().astype('int16')
 user_count_agg = groupby_user[target_col].count().astype('int16')
 content_target_sum_agg = groupby_content[target_col].sum().astype('int32')
 content_count_agg = groupby_content[target_col].count().astype('int32')
 del groupby_user, groupby_content
gc.collect()
with trace_mem("Generate feature 'residual_user_mean'"):
df_train.reset_index(drop=True, inplace=True)
 df_train['residual'] = df_train[target_col] - \
df_train['content_id'].map(content_target_sum_agg/content_count_agg)
 df_train['residual'] = df_train['residual']
residual_groupby_user = df_train[['user_id', 'residual']].groupby('user_id', sort=False)['residual']
df_train['lag'] = residual_groupby_user.shift()
df_train['lag'].fillna(0, inplace=True)
 lag_groupby_user = df_train[['user_id', 'lag']].groupby('user_id', sort=False)['lag']
lag_cumsum, lag_cumcount = lag_groupby_user.cumsum() , lag_groupby_user.cumcount()
df_train['residual_user_mean'] = lag_cumsum / lag_cumcount
df_train['residual_user_mean'].fillna(0, inplace=True)
 df_train['residual_user_mean'] = df_train['residual_user_mean'].astype('float32')
 user_residual_sum_agg = residual_groupby_user.sum().astype('float32')
 df_train.drop(columns=['residual', 'lag'], inplace=True)
 del residual_groupby_user, lag_groupby_user, lag_cumsum, lag_cumcount
gc.collect()
with trace_mem("Generate feature 'user_correctness' and 'user_correct_cumsum'"):
df_train['lag'] = df_train[['user_id', target_col]].groupby('user_id')[target_col].shift()
df_train['lag'].fillna(0, inplace=True)
 df_train['lag'] = df_train['lag'].astype('int8')
 lag_groupby_user_id = df_train.loc[:, ['user_id', 'lag']].groupby(['user_id'])['lag']
lag_cumsum, lag_cumcount = lag_groupby_user_id.cumsum().astype('int16'), \
lag_groupby_user_id.cumcount().astype('int16')
 df_train['user_correct_cumsum'] = lag_cumsum
df_train['user_correctness'] = lag_cumsum / lag_cumcount
df_train['user_correctness'].fillna(0, inplace=True)
 df_train['user_correctness'] = df_train['user_correctness'].astype('float32')
 del lag_cumsum, lag_cumcount, lag_groupby_user_id
df_train.drop(columns=['lag'], inplace=True)
 gc.collect()
with trace_mem("Generate feature 'pq_elapsed_time_user_mean'"):
df_train['lag'] = df_train[['user_id', 'prior_question_elapsed_time']].\
groupby('user_id', sort=False)['prior_question_elapsed_time'].shift()
df_train['lag'].fillna(0, inplace=True)
 lag_groupby_user = df_train[['user_id', 'lag']].groupby('user_id', sort=False)['lag']
lag_cumsum, lag_cumcount = lag_groupby_user.cumsum() , lag_groupby_user.cumcount()
df_train['pq_elapsed_time_user_mean'] = lag_cumsum / lag_cumcount
df_train['pq_elapsed_time_user_mean'].fillna(0, inplace=True)
 df_train['pq_elapsed_time_user_mean'] = df_train['pq_elapsed_time_user_mean'].astype('float32')
 df_train.drop(columns=['lag'], inplace=True)
 del lag_groupby_user, lag_cumsum, lag_cumcount
gc.collect()
with trace_mem("Generate feature 'explanation_user_cumsum'"):
explanation_groupby_userid = df_train[['user_id', 'prior_question_had_explanation']].\
groupby('user_id')['prior_question_had_explanation']
df_train['lag'] = explanation_groupby_userid.shift()
lag_cumsum = df_train[['user_id', 'lag']].groupby('user_id')['lag'].cumsum()
df_train['lag'] = lag_cumsum
df_train.rename(columns={'lag': 'explanation_user_cumsum'}, inplace=True)
 df_train['explanation_user_cumsum'].fillna(0, inplace=True)
 df_train['explanation_user_cumsum'] = df_train['explanation_user_cumsum'].astype('int16')
 explanation_sum_agg = explanation_groupby_userid.sum().astype('int16')
 del explanation_groupby_userid, lag_cumsum
gc.collect()
with trace_mem("Dropping rows with 'val' == 0"):
df_train = df_train[df_train['val'] != 0].reset_index(drop=True)
 with trace_mem("Generate feature 'content_count', 'content_correctness'"):
df_train['content_count'] = df_train['content_id'].map(content_count_agg ).astype('int32')
 df_train['content_correctness'] = df_train['content_id'].\
map(content_target_sum_agg/content_count_agg ).astype('float32')
 with trace_mem("Process questions.csv"):
NULL_TAG = '255'
data_types_dict = {'question_id': 'int16',
'part': 'int8',
'bundle_id': 'int16',
'tags': 'string'}
df_questions = pd.read_csv(QUESTIONS_FILE_PATH, usecols=data_types_dict.keys() , dtype=data_types_dict)
 df_questions['tags'].fillna(NULL_TAG, inplace=True)
 del data_types_dict
assert(0 == df_questions.isnull().sum().sum())
 df_questions.rename(columns={'question_id': 'content_id'}, inplace=True)

unique_tags_combos_keys = {value: idx for idx, value in enumerate(df_questions['tags'].unique())}
df_questions['tags_encoded'] = df_questions['tags'].apply(lambda x: unique_tags_combos_keys[x] ).astype('int16')
 del unique_tags_combos_keys
question_tags_list = df_questions['tags'].apply(lambda x: x.split())
 df_questions['tags_first1'] = df_questions['tags'].apply(lambda x: x.split() [0])
 unique_tags_combos_keys = {value: idx for idx, value in enumerate(df_questions['tags_first1'].unique())}
df_questions['tags_first1'] = df_questions['tags_first1'].apply(lambda x: unique_tags_combos_keys[x] ).astype('int16')
 del unique_tags_combos_keys
df_questions['tags_last2'] = df_questions['tags'].\
apply(lambda x: x.split() [-1] if len(x.split())== 1 else x.split() [-2]+x.split() [-1])
 unique_tags_combos_keys = {value: idx for idx, value in enumerate(df_questions['tags_last2'].unique())}
df_questions['tags_last2'] = df_questions['tags_last2'].apply(lambda x: unique_tags_combos_keys[x] ).astype('int16')
 df_questions.drop(columns=['tags'], inplace=True)
 del unique_tags_combos_keys, question_tags_list
gc.collect()
with trace_mem("Merge df_questions to df_train"):
df_train = pd.merge(df_train, df_questions, on='content_id', how='left')
 assert(0 == df_train.isnull().sum().sum())
 with trace_mem("Split to train/val dataset"):
df_val = df_train[df_train['val'] == 2].reset_index(drop=True)
 df_train = df_train[df_train['val'] != 2].reset_index(drop=True)
 df_val.drop(columns=['user_id', 'content_id', 'val'], inplace=True)
 df_train.drop(columns=['user_id', 'content_id', 'val'], inplace=True)
 print(f"df_train`s shape {df_train.shape}, df_val`s shape: {df_val.shape}")
 with trace_mem("Save User_feats_dict and Ques_feats_dict for online inference"):
def clip(count):
return np.clip(count, 1, np.inf)
 class UserFeats(object):
def __init__(self, user_target_sum=0, user_count=0, explanation_sum=0, prior_question_elapsed_time_sum=0,
user_residual_sum=0, user_max_ts_1=0, user_max_ts_2=0, last_lagtime_1=0,
last_lagtime_2=0, last_task_container=0):
self.ans_corr_cnt = user_target_sum
self.user_cnt = user_count
self.user_acc = user_target_sum/clip(user_count)
 self.user_explanation_sum = explanation_sum
self.elapsed_time_sum = prior_question_elapsed_time_sum
self.residual_sum = user_residual_sum
self.residual_mean = user_residual_sum/clip(user_count)
 self.user_max_timestamp_1 = user_max_ts_1
self.user_max_timestamp_2 = user_max_ts_2
self.user_last_lagtime_1 = last_lagtime_1
self.user_last_lagtime_2 = last_lagtime_2
self.last_task_id = last_task_container
def update_user_acc(self, _ans_corr):
self.ans_corr_cnt += _ans_corr
self.user_cnt += 1
self.user_acc = self.ans_corr_cnt/clip(self.user_cnt)
 def get_user_acc(self):
return self.user_acc
def get_explanation_cumsum(self, current_explanation):
current_explanation_sum = self.user_explanation_sum
self.user_explanation_sum += current_explanation
return current_explanation_sum
def get_elapsed_time_mean(self, _elapsed_time):
current_elapsed_mean = self.elapsed_time_sum/clip(self.user_cnt)
 self.elapsed_time_sum += _elapsed_time
return current_elapsed_mean
def update_user_residual(self, new_residual):
self.residual_sum += new_residual
self.residual_mean = self.residual_sum/self.user_cnt
def get_residual_mean(self):
return self.residual_mean
def get_lagtime(self, new_timestamp, new_task_id):
if new_task_id == self.last_task_id:
return self.user_last_lagtime_1, self.user_last_lagtime_2
self.last_task_id = new_task_id
lag_time1, lag_time2 = new_timestamp/TS_SCALING - self.user_max_timestamp_1, \
new_timestamp/TS_SCALING - self.user_max_timestamp_2
self.user_max_timestamp_2, self.user_last_lagtime_2 = self.user_max_timestamp_1, lag_time2
self.user_max_timestamp_1, self.user_last_lagtime_1 = new_timestamp/TS_SCALING, lag_time1
return lag_time1, lag_time2
User_feats_dict = defaultdict(UserFeats)
 for _user_id in user_target_sum_agg.index:
_user_feats = UserFeats(user_target_sum=user_target_sum_agg[_user_id], user_count=user_count_agg[_user_id],
explanation_sum=explanation_sum_agg[_user_id],
prior_question_elapsed_time_sum=user_ques_elapsed_time_sum_agg[_user_id],
user_residual_sum=user_residual_sum_agg[_user_id],
user_max_ts_1=user_max_timestamp_1_agg[_user_id],
user_max_ts_2=user_max_timestamp_2_agg[_user_id],
last_lagtime_1=user_last_lagtime_1_agg[_user_id],
last_lagtime_2=user_last_lagtime_2_agg[_user_id],
last_task_container=user_last_task_agg[_user_id])
 User_feats_dict[_user_id] = _user_feats
class QuesFeats(object):
def __init__(self, content_target_sum=0, content_count=0, ques_bundle=1, ques_part=1, ques_tags_encoded=0,
ques_tags_first1=0, ques_tags_last2=0):
self.corr_cnt = content_target_sum
self.ques_cnt = content_count
self.ques_acc = content_target_sum/clip(content_count)
 self.bundle = ques_bundle
self.part = ques_part
self.tags_encoded = ques_tags_encoded
self.tags_first1 = ques_tags_first1
self.tags_last2 = ques_tags_last2
def update_ques_acc(self, _ans_corr):
self.corr_cnt += _ans_corr
self.ques_cnt += 1
self.ques_acc = self.corr_cnt/clip(self.ques_cnt)
 def get_ques_acc(self):
return self.ques_acc
Ques_feats_dict = defaultdict(QuesFeats)
 df_ques_tmp = df_questions.set_index('content_id')
 for _content_id in content_target_sum_agg.index:
_ques_feats = QuesFeats(content_target_sum=content_target_sum_agg[_content_id],
content_count=content_count_agg[_content_id],
ques_bundle=df_ques_tmp.loc[_content_id, 'bundle_id'],
ques_part=df_ques_tmp.loc[_content_id, 'part'],
ques_tags_encoded=df_ques_tmp.loc[_content_id, 'tags_encoded'],
ques_tags_first1=df_ques_tmp.loc[_content_id, 'tags_first1'],
ques_tags_last2=df_ques_tmp.loc[_content_id, 'tags_last2'], 
)
 Ques_feats_dict[_content_id] = _ques_feats
del df_ques_tmp, _user_feats, _ques_feats
del user_target_sum_agg, user_count_agg, explanation_sum_agg, content_target_sum_agg, content_count_agg, \
user_residual_sum_agg, user_ques_elapsed_time_sum_agg, user_max_timestamp_1_agg, user_max_timestamp_2_agg, \
user_last_lagtime_1_agg, user_last_lagtime_2_agg, user_last_task_agg
gc.collect()
class FeatureEngineer(object):
def __init__(self, user_feats_dict, ques_feats_dict, all_elapsed_time_mean):
self._user_feats_dict = user_feats_dict
self._ques_feats_dict = ques_feats_dict
self._elapsed_time_mean = all_elapsed_time_mean
self._init_feat_extract_fns()
self._prior_user_ids = []
self._prior_content_ids = []
self._prior_content_type_ids = []
def _init_feat_extract_fns(self):
name_to_extract_fn = {
'prior_question_elapsed_time': lambda u_feats, q_feats, row, cache: self._elapsed_time_mean \
if cache['elapsed_time_isna'] else row['prior_question_elapsed_time'],
'prior_question_had_explanation': lambda u_feats, q_feats, row, cache: cache['current_explanation'],
'bundle_id': lambda u_feats, q_feats, row, cache: q_feats.bundle,
'part': lambda u_feats, q_feats, row, cache: q_feats.part,
'tags_encoded': lambda u_feats, q_feats, row, cache: q_feats.tags_encoded,
'tags_first1': lambda u_feats, q_feats, row, cache: q_feats.tags_first1,
'tags_last2': lambda u_feats, q_feats, row, cache: q_feats.tags_last2,
'content_count': lambda u_feats, q_feats, row, cache: q_feats.ques_cnt,
'content_correctness': lambda u_feats, q_feats, row, cache: q_feats.get_ques_acc() ,
'user_correctness': lambda u_feats, q_feats, row, cache: u_feats.get_user_acc() ,
'user_correct_cumsum': lambda u_feats, q_feats, row, cache: u_feats.ans_corr_cnt,
'residual_user_mean': lambda u_feats, q_feats, row, cache: u_feats.get_residual_mean() ,
'pq_elapsed_time_user_mean': lambda u_feats, q_feats, row, cache: \
u_feats.get_elapsed_time_mean(0 if cache['elapsed_time_isna']
else row['prior_question_elapsed_time']),
'explanation_user_cumsum': lambda u_feats, q_feats, row, cache: \
u_feats.get_explanation_cumsum(cache['current_explanation']),
'lag_time': lambda u_feats, q_feats, row, cache: cache['lag_times'][0],
'lag_time2': lambda u_feats, q_feats, row, cache: cache['lag_times'][1],
'user_content_attempted': lambda u_feats, q_feats, row, cache: \
get_user_content_attempted(row['user_id'], row['content_id']),
}
self._feat_extract_fns = [name_to_extract_fn[name] for name in TRA_FEATURES]
def _get_row_fvs(self, row: Series, u_feats: UserFeats, q_feats: QuesFeats)-> List[float]:
cache = {'current_explanation': 0 if pd.isna(row['prior_question_had_explanation'])
 else row['prior_question_had_explanation'],
'elapsed_time_isna': True if pd.isna(row['prior_question_elapsed_time'])else False,
'lag_times': u_feats.get_lagtime(row['timestamp'], row['task_container_id']),
}
return [fn(u_feats, q_feats, row, cache)for fn in self._feat_extract_fns]
def _reset_prior_lists(self):
self._prior_user_ids, self._prior_content_ids, self._prior_content_type_ids = [], [], []
def _process_prior_df(self, _prior_targets):
len_targets = len(_prior_targets)
 if len_targets == 0:
self._reset_prior_lists()
return
len_prior_user_ids, len_prior_content_ids = len(self._prior_user_ids), len(self._prior_content_ids)
 if(len_targets != len_prior_user_ids)or(len_prior_user_ids != len_prior_content_ids):
print(f"length not match, len_targets: {len_targets}, len_prior_user_ids: {len_prior_user_ids}, "
f"len_prior_content_ids: {len_prior_content_ids}")
 self._reset_prior_lists()
return
for _uid, _cid, _target, _content_type in zip(self._prior_user_ids, self._prior_content_ids, _prior_targets,
self._prior_content_type_ids):
if _content_type == 0:
self._user_feats_dict[_uid].update_user_acc(_target)
 self._ques_feats_dict[_cid].update_ques_acc(_target)
 self._user_feats_dict[_uid].update_user_residual(_target - self._ques_feats_dict[_cid].get_ques_acc())
 self._reset_prior_lists()
def get_feat_values(self, df: DataFrame):
prior_targets_list = json.loads(df['prior_group_answers_correct'].iloc[0])
 self._process_prior_df(prior_targets_list)
 fvs, _ques_row_ids = [], []
for _, row in df.iterrows() :
self._prior_user_ids.append(row['user_id'])
 self._prior_content_ids.append(row['content_id'])
 self._prior_content_type_ids.append(row['content_type_id'])
 if row['content_type_id'] == 0:
_ques_row_ids.append(row['row_id'])
 u_feats = self._user_feats_dict[row['user_id']]
q_feats = self._ques_feats_dict[row['content_id']]
fvs.append(self._get_row_fvs(row, u_feats, q_feats))
return np.array(_ques_row_ids), np.array(fvs)
 def make_x_y_np(df):
y = df[target_col].values.astype(np.float32)
 if not PLOT_SHAP:
df.drop(columns=[target_col], inplace=True)
 x = np.ndarray(shape=(df.shape[0], len(TRA_FEATURES)) , dtype=np.float32)
 for idx_, feature_ in enumerate(TRA_FEATURES):
x[:, idx_] = df[feature_].values.astype(np.float32)
 if not PLOT_SHAP:
df.drop(columns=[feature_], inplace=True)
 return x, y
with trace_mem("Prepare float32 ndarray for LGBM"):
X_tra, y_tra = make_x_y_np(df_train)
 X_val, y_val = make_x_y_np(df_val)
 if PLOT_SHAP:
del df_train
else:
del df_train, df_val
gc.collect()
with trace_mem("Prepare datasets for LGBM"):
lgb_train = lgb.Dataset(X_tra, y_tra, feature_name=TRA_FEATURES)
 lgb_val = lgb.Dataset(X_val, y_val, feature_name=TRA_FEATURES)
 with trace_mem("del X_tra, y_tra, X_val, y_val"):
del X_tra, y_tra, X_val, y_val
gc.collect()
params = {'objective': 'binary',
'seed': DEFAULT_SEED,
'learning_rate': LEARNING_RATE,
'metric': 'auc',
'max_bin': MAX_BIN,
'num_leaves': NUM_LEAVES,
'feature_fraction': FEATURE_FRACTION,
'bagging_fraction': BAGGING_FRACTION,
'bagging_freq': BAGGING_FREQ,
}
time_begin = time()
lgb_model = lgb.train(params,
lgb_train,
valid_sets=[lgb_val],
verbose_eval=VERBOSE_EVAL,
num_boost_round=NUM_BOOST_ROUNDS,
early_stopping_rounds=EARLY_STOP_ROUNDS,
categorical_feature=cat_features)
 print(f"Training lgb_model elapsed {str(timedelta(seconds=time() -time_begin)) }")
 with trace_mem("del lgb_train, lgb_val"):
del lgb_train, lgb_val
gc.collect()
feat_eng = FeatureEngineer(User_feats_dict, Ques_feats_dict, mean_prior_question_elapsed_time)
 env = riiideducation.make_env()
iter_test = env.iter_test()
for(test_df, _)in iter_test:
ques_row_ids, X_test = feat_eng.get_feat_values(test_df)
 target_preds = lgb_model.predict(X_test)
 submit_df = pd.DataFrame({'row_id': ques_row_ids, target_col: target_preds})
 env.predict(submit_df )<import_modules>
print("Best parameters : {}".format(grid_rf.best_params_))
print("Best cross-validation score : {:.2f}".format(grid_rf.best_score_))
<import_modules><EOS> 
clf_rf = grid_rf.best_estimator_
clf_rf.fit(train_reduced,target_train)
 target_test = clf_rf.predict(test_reduced)
 df_test['Survived'] = target_test
df_test[['PassengerId','Survived']].to_csv('rf-kaggle-submit.csv',index=False )