Kaggle competition: Mechanisms of Action (MoA) Prediction
n_skill = 13523
group = joblib.load(".. /input/saint-plus-data-new/group_20210102.pkl.zip" )<load_from_csv>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_rows', 100, 'display.max_columns', 900)
questions_df = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/questions.csv' )<load_pretrained>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug= pd.read_csv('.. /input/lish-moa/train_drug.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
time_dict = joblib.load(".. /input/saint-plus-data-new/time_dict.pkl.zip" )<load_pretrained>
genes=[col for col in train_features.columns if col.startswith('g-')]
cells=[col for col in train_features.columns if col.startswith('c-')]
print(genes)
 print(' <----------------------------------------------------------------------------------------------------------------->')
 print(cells)
 print('
')
 print(len(genes))
print('
')
 print(len(cells))
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model = SAINTModel(n_skill, n_part, embed_dim= D_MODEL)
 try:
model.load_state_dict(torch.load(".. /input/saint-plus-model/saint_plus_model_20210108_v1.pt"))
except:
model.load_state_dict(torch.load(".. /input/saint-plus-model/saint_plus_model_20210108_v1.pt", map_location='cpu'))
model.to(device)
 model.eval()<feature_engineering>
print(train_features.isna().sum().sum())
 print(test_features.isna().sum().sum())
 print(train_drug.isna().sum().sum())
 print(train_targets_scored.isna().sum().sum())
 print(train_targets_nonscored.isna().sum().sum() )
prev_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if(prev_test_df is not None)&(psutil.virtual_memory().percent < 90):
prev_test_df['answered_correctly'] = eval(test_df['prior_group_answers_correct'].iloc[0])
 prev_test_df = prev_test_df[prev_test_df.content_type_id == False]
prev_test_df = feature_time_lag(prev_test_df, time_dict)
 prev_group = prev_test_df[['user_id', 'content_id', 'answered_correctly', 'part', 'prior_question_elapsed_time', 'time_lag', 'prior_question_had_explanation']].groupby('user_id' ).apply(lambda r:(
 r['content_id'].values,
r['answered_correctly'].values,
r['part'].values,
r['prior_question_elapsed_time'].values,
r['time_lag'].values,
r['prior_question_had_explanation'].values))
for prev_user_id in prev_group.index:
if prev_user_id in group.index:
group[prev_user_id] =(
 np.append(group[prev_user_id][0], prev_group[prev_user_id][0])[-MAX_SEQ:],
np.append(group[prev_user_id][1], prev_group[prev_user_id][1])[-MAX_SEQ:],
np.append(group[prev_user_id][2], prev_group[prev_user_id][2])[-MAX_SEQ:],
np.append(group[prev_user_id][3], prev_group[prev_user_id][3])[-MAX_SEQ:],
np.append(group[prev_user_id][4], prev_group[prev_user_id][4])[-MAX_SEQ:],
np.append(group[prev_user_id][5], prev_group[prev_user_id][5])[-MAX_SEQ:] 
)
 else:
group[prev_user_id] =(
 prev_group[prev_user_id][0],
prev_group[prev_user_id][1],
prev_group[prev_user_id][2],
prev_group[prev_user_id][3],
prev_group[prev_user_id][4],
prev_group[prev_user_id][5] 
)
 test_df.prior_question_elapsed_time = test_df.prior_question_elapsed_time.fillna(0)
 test_df['prior_question_had_explanation'] = test_df['prior_question_had_explanation'].fillna(value = False ).astype(int)
 test_df = test_df.merge(questions_df[["question_id","part"]], how = "left",left_on = 'content_id', right_on = 'question_id')
 prev_test_df = test_df.copy()
test_df = test_df[test_df.content_type_id == False]
test_df = feature_time_lag(test_df, time_dict)
 test_dataset = TestDataset(group, test_df, n_skill)
 test_dataloader = DataLoader(test_dataset, batch_size=51200, shuffle=False)
 outs = []
for item in test_dataloader:
exercise = item[0].to(device ).long()
part = item[1].to(device ).long()
response = item[2].to(device ).long()
elapsed_time = item[3].to(device ).long()
lag_time = item[4].to(device ).long()
pri_exp = item[5].to(device ).long()
with torch.no_grad() :
output = model(exercise, part, response, elapsed_time, lag_time, pri_exp)
 outs.extend(torch.sigmoid(output)[:, -1].view(-1 ).data.cpu().numpy())
 test_df['answered_correctly'] = outs
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']] )<set_options>
train_features2=train_features.copy()
test_features2=test_features.copy()
warnings.filterwarnings('ignore' )<define_variables>
g_features = [feature for feature in train_features.columns if feature.startswith('g-')]
c_features = [feature for feature in train_features.columns if feature.startswith('c-')]
other_features = [feature for feature in train_features.columns if feature not in g_features and
feature not in c_features and
feature not in train_targets_scored and
feature not in train_targets_nonscored]
print(f'Number of g- Features: {len(g_features)}')
 print(f'Number of c- Features: {len(c_features)}')
 print(f'Number of Other Features: {len(other_features)}({other_features})' )
TARGET = 'answered_correctly'
feats = ['content_id', 'task_container_id', 'prior_question_elapsed_time', 'prior_question_had_explanation', 'part', 'content_id_target_mean', 'user_count', 'user_correct_mean', 'user_question_count', 'user_question_correct_mean', 'last_time_user', 'last_time_question_user', 'last_time_user_inter', 'user_last_n_correct', 'answer1', 'answer2', 'answer3', 'answer4', 'user_last_n_time', 'user_last_n_time2', 'user_last_n_time3', 'user_part_count', 'user_part_correct_mean', 'user_part1_mean', 'user_part2_mean', 'user_part3_mean', 'user_part4_mean', 'user_part5_mean', 'user_part6_mean', 'user_part7_mean', 'task_container_eq1', 'task_container_eq2', 'user_hardness_count', 'user_hardness_mean', 'user_hardness_inter']<define_variables>
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[g_features+c_features]), pd.DataFrame(test_features[g_features+c_features])])
 data2 = qt.fit_transform(data[g_features+c_features])
 train_features[g_features+c_features] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[g_features+c_features] = pd.DataFrame(data2[-test_features.shape[0]:] )
model_path = '.. /input/lgb-baseline-riid/lgb_0.7936991.dat'
questions_file = '.. /input/riiid-test-answer-prediction/questions.csv'
lectures_file = '.. /input/riiid-test-answer-prediction/lectures.csv'
train_file = '.. /input/riiid-test-answer-prediction/train.csv'
test_file = '.. /input/riiid-test-answer-prediction/example_test.csv'<merge>
for col in(genes + cells):
transformer = QuantileTransformer(n_quantiles=100,random_state=42, output_distribution='normal')
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
def get_and_merge_feat(trn, target, feat):
feat_name = '{}_target_mean'.format(feat)
 mean = trn[[feat, target]].groupby([feat] ).agg(['mean'])
 mean.columns = [feat_name]
trn = pd.merge(trn, mean, on=feat, how='left')
 feat_dict = mean.astype('float32' ).to_dict() [feat_name]
return trn, feat_dict<data_type_conversions>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
def get_answer_feats(trn):
answer_counts = trn.groupby('content_id')['user_answer'].value_counts(normalize=True)
 answer_counts_unstack = answer_counts.unstack().reset_index(drop=True ).astype(np.float32)
 answer_counts_unstack.columns = ['answer1', 'answer2', 'answer3', 'answer4']
answer_counts_unstack = answer_counts_unstack.rename_axis('content_id' ).reset_index()
answers = answer_counts_unstack.values[:, -4:].astype(np.float32)
 answers.sort(axis=1)
 answer_counts_unstack[['answer1', 'answer2', 'answer3', 'answer4']] = answers
answer_counts_unstack = answer_counts_unstack.astype(np.float32)
 return answer_counts_unstack<feature_engineering>
n_comp = 650
data = pd.concat([pd.DataFrame(train_features[genes]), pd.DataFrame(test_features[genes])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[genes]))
train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )
def get_q_hardness_bins(df):
df['question_hardness'] = 0
df.question_hardness.values[df.content_id_target_mean >= 0.9] = -10
df.question_hardness.values[(df.content_id_target_mean >= 0.7)&(df.content_id_target_mean < 0.9)] = -11
df.question_hardness.values[(df.content_id_target_mean >= 0.5)&(df.content_id_target_mean < 0.7)] = -12
df.question_hardness.values[df.content_id_target_mean < 0.5] = -13
return df<load_pretrained>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[cells]), pd.DataFrame(test_features[cells])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[cells]))
train2 = data2[:train_features.shape[0]]
test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )
needed_cols = ['user_id', 'content_id', 'timestamp', 'part', 'task_container_id', TARGET, 'user_answer']
print("Get content mean feature...")
 train = pd.read_pickle('.. /input/local-training-file-pickled/train_all.pkl')[needed_cols]
train = train[train.answered_correctly != -1]
train, content_dict = get_and_merge_feat(train, TARGET, 'content_id')
 train = get_q_hardness_bins(train)
 print("Get question answer distribution...")
 answer_counts = get_answer_feats(train)
 del train['user_answer']
gc.collect()
model = pickle.load(open(model_path, "rb"))
user_idx = pd.concat([ train[['user_id']].drop_duplicates(keep='first'), train[['user_id']].drop_duplicates(keep='last'), ] ).reset_index().sort_values(['user_id', 'index'] ).groupby(['user_id'])['index'].agg([list] ).to_dict() ['list']<define_variables>
var_thresh = VarianceThreshold(0.845)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
answer_dist_dict = {}
for idx, row in enumerate(answer_counts.values):
answer_dist_dict[int(row[0])] = [row[1], row[2], row[3], row[4]]
del answer_counts<feature_engineering>
def feat_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = g_features
features_c = c_features
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=feat_cluster(train_features2,test_features2 )
def update_dicts(row, count_dict, correct_dict, time_dict, last_n_dict):
user = int(row[0])
 question = int(row[1])
 timestamp = int(row[2])
 part = int(-row[3])
 task_id = int(row[4])
 hardness = int(row[5])
 correct = int(row[6])
 if user in count_dict:
count_dict[user]['sum'] += 1
correct_dict[user]['sum'] += correct
time_dict[user]['last'] = timestamp
last_n_dict[user]['last_task2'] = last_n_dict[user]['last_task']
last_n_dict[user]['last_task'] = task_id
last_n_dict[user]['last_n'].append(correct)
 correction = last_n_dict[user]['last_n'].pop(0)
 last_n_dict[user]['sum'] -= correction
last_n_dict[user]['sum'] += correct
last_n_dict[user]['last_n_time'].append(timestamp)
 last_n_dict[user]['time_sum'] = last_n_dict[user]['last_n_time'].pop(0)
 last_n_dict[user]['time_sum2'] = last_n_dict[user]['last_n_time'][9]
last_n_dict[user]['time_sum3'] = last_n_dict[user]['last_n_time'][14]
if hardness in count_dict[user]:
count_dict[user][hardness] += 1
correct_dict[user][hardness] += correct
else:
count_dict[user][hardness] = 1
correct_dict[user][hardness] = correct
if part in count_dict[user]:
count_dict[user][part] += 1
correct_dict[user][part] += correct
else:
count_dict[user][part] = 1
correct_dict[user][part] = correct
if question in count_dict[user]:
count_dict[user][question] += 1
correct_dict[user][question] += correct
time_dict[user][question] = timestamp
else:
count_dict[user][question] = 1
correct_dict[user][question] = correct
time_dict[user][question] = timestamp
else:
count_dict[user] = {'sum': 1, question: 1, part: 1, hardness: 1}
correct_dict[user] = {'sum': correct, question: correct, part: correct, hardness: correct}
time_dict[user] = {'last': timestamp, question: timestamp}
last_n_dict[user] = {'sum': correct, 'time_sum': 0, 'time_sum2': 0, 'time_sum3': 0,
'last_n': [0, 0, 0, 0, correct], 'last_task': task_id, 'last_task2': np.nan,
'last_n_time': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, timestamp]}
return count_dict, correct_dict, time_dict, last_n_dict
def get_row_values(row, count_dict, correct_dict, time_dict, last_n_dict):
feats = np.full(23, fill_value=np.nan, dtype=np.float32)
 user = int(row[0])
 question = int(row[1])
 timestamp = int(row[2])
 part = int(-row[3])
 task_id = int(row[4])
 hardness = int(row[5])
 if user in count_dict:
feats[0] = count_dict[user]['sum']
feats[1] = correct_dict[user]['sum']
feats[4] = timestamp - time_dict[user]['last']
feats[6] = last_n_dict[user]['sum']
feats[7] = timestamp - last_n_dict[user]['time_sum']
feats[8] = timestamp - last_n_dict[user]['time_sum2']
feats[9] = timestamp - last_n_dict[user]['time_sum3']
feats[19] = task_id - last_n_dict[user]['last_task']
feats[20] = task_id - last_n_dict[user]['last_task2']
if hardness in count_dict[user]:
feats[21] = count_dict[user][hardness]
feats[22] = correct_dict[user][hardness]
if part in count_dict[user]:
feats[10] = count_dict[user][part]
feats[11] = correct_dict[user][part]
for i, p in enumerate([-1, -2, -3, -4, -5, -6, -7]):
if p in count_dict[user]:
feats[12 + i] = correct_dict[user][p] / count_dict[user][p]
if question in count_dict[user]:
feats[2] = count_dict[user][question]
feats[3] = correct_dict[user][question]
feats[5] = timestamp - time_dict[user][question]
return feats
def get_user_feats(trn, val, save_dicts=False):
trn, count_dict, correct_dict, time_dict, last_n_dict = calc_dicts_and_add(trn)
 val, count_dict, correct_dict, time_dict, last_n_dict = calc_dicts_and_add(val, count_dict, correct_dict,
time_dict, last_n_dict)
 if save_dicts:
print("Save count dict...")
 with open(configs.count_dict_path, 'wb')as f:
pickle.dump(count_dict, f, pickle.HIGHEST_PROTOCOL)
 print("Save correct dict...")
 with open(configs.correct_dict_path, 'wb')as f:
pickle.dump(correct_dict, f, pickle.HIGHEST_PROTOCOL)
 print("Save time dict...")
 with open(configs.time_dict_path, 'wb')as f:
pickle.dump(time_dict, f, pickle.HIGHEST_PROTOCOL)
 print("Save last n dict...")
 with open(configs.last_n_dict_path, 'wb')as f:
pickle.dump(last_n_dict, f, pickle.HIGHEST_PROTOCOL)
 del count_dict, correct_dict, time_dict, last_n_dict
gc.collect()
return trn, val
def calc_feats_from_stats(df, user_feats):
user_feats[:, 1] = user_feats[:, 1] / user_feats[:, 0]
user_feats[:, 3] = user_feats[:, 3] / user_feats[:, 2]
user_feats[:, 11] = user_feats[:, 11] / user_feats[:, 10]
user_feats[:, 22] = user_feats[:, 22] / user_feats[:, 21]
df['user_count'] = user_feats[:, 0].astype(np.float32)
 df['user_correct_mean'] = user_feats[:, 1].astype(np.float32)
 df['user_question_count'] = user_feats[:, 2].astype(np.float32)
 user_feats[:, 3][user_feats[:, 3] == -np.inf] = 0
df['user_question_correct_mean'] = user_feats[:, 3].astype(np.float32)
 df['last_time_user'] = user_feats[:, 4].astype(np.float32)
 df['last_time_question_user'] = user_feats[:, 5].astype(np.float32)
 df['last_time_user_inter'] = df['last_time_user'].astype(np.float32)- df['last_time_question_user'].astype(
 np.float32)
 df['user_last_n_correct'] = user_feats[:, 6].astype(np.float32)
 df['user_last_n_time'] = user_feats[:, 7].astype(np.float32)
 df['user_last_n_time2'] = user_feats[:, 8].astype(np.float32)
 df['user_last_n_time3'] = user_feats[:, 9].astype(np.float32)
 df['user_part_count'] = user_feats[:, 10].astype(np.float32)
 df['user_part_correct_mean'] = user_feats[:, 11].astype(np.float32)
 df['user_part1_mean'] = user_feats[:, 12].astype(np.float32)
 df['user_part2_mean'] = user_feats[:, 13].astype(np.float32)
 df['user_part3_mean'] = user_feats[:, 14].astype(np.float32)
 df['user_part4_mean'] = user_feats[:, 15].astype(np.float32)
 df['user_part5_mean'] = user_feats[:, 16].astype(np.float32)
 df['user_part6_mean'] = user_feats[:, 17].astype(np.float32)
 df['user_part7_mean'] = user_feats[:, 18].astype(np.float32)
 df['task_container_eq1'] = user_feats[:, 19].astype(np.float32)
 df['task_container_eq2'] = user_feats[:, 20].astype(np.float32)
 df['user_hardness_count'] = user_feats[:, 21].astype(np.float32)
 df['user_hardness_mean'] = user_feats[:, 22].astype(np.float32)
 df['user_hardness_inter'] = df['user_hardness_mean'] - df['content_id_target_mean']
return df
def calc_dicts_and_add(df, count_dict=None, correct_dict=None, time_dict=None, last_n_dict=None):
if not count_dict:
count_dict = {}
if not correct_dict:
correct_dict = {}
if not time_dict:
time_dict = {}
if not last_n_dict:
last_n_dict = {}
user_feats = np.full(( len(df), 23), fill_value=np.nan, dtype=np.float32)
 prev_row = None
feat_iterator = df[['user_id', 'content_id', 'timestamp', 'part', 'task_container_id', 'question_hardness',
'answered_correctly']].values
del df['timestamp']
del df['user_id']
del df['row_id']
for row_id, curr_row in enumerate(tqdm(feat_iterator)) :
if prev_row is not None:
count_dict, correct_dict, time_dict, last_n_dict = update_dicts(prev_row, count_dict, correct_dict,
time_dict, last_n_dict)
 user_row_values = get_row_values(curr_row, count_dict, correct_dict, time_dict, last_n_dict)
 user_feats[row_id] = user_row_values
prev_row = curr_row
del feat_iterator
df = calc_feats_from_stats(df, user_feats)
 return df, count_dict, correct_dict, time_dict, last_n_dict<load_from_csv>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
def prepare_questions() :
questions = pd.read_csv(questions_file,
dtype={
'question_id': 'int16',
'bundle_id': 'int16',
'correct_answer': 'int8',
'part': 'int8'
})
 questions.drop(['bundle_id', 'correct_answer'], axis=1, inplace=True)
 questions.rename(columns={'question_id': 'content_id'}, inplace=True)
 questions['content_type_id'] = 0
return questions
def prepare_lectures() :
lectures = pd.read_csv(lectures_file,
dtype={
'lecture_id': 'int16',
'tag': 'int16',
'part': 'int8'
})
 lectures.drop(['type_of'], axis=1, inplace=True)
 lectures.rename(columns={'lecture_id': 'content_id', 'tag': 'tags'}, inplace=True)
 lectures['content_type_id'] = 1
return lectures
def prepare_meta_feats() :
questions = prepare_questions()
lectures = prepare_lectures()
meta = questions.append(lectures)
 return meta<data_type_conversions>
def feat_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = feat_cluster_pca(train_pca,test_pca )
def replace_bools(df):
df.loc[:, 'prior_question_had_explanation'] = df.loc[:, 'prior_question_had_explanation'].map(
 {False: 0, True: 1} 
)
 df['prior_question_had_explanation'] = df['prior_question_had_explanation'].astype(np.float16)
 return df<merge>
train_cluster_pca = train_cluster_pca.iloc[:,700:]
test_cluster_pca = test_cluster_pca.iloc[:,700:]
def merge_feat_val(key_feat, feat_dict):
add_feat = np.zeros(( len(key_feat)))
 key_feat = key_feat.values
for row_id, row in enumerate(key_feat):
key = key_feat[row_id]
if key in feat_dict:
add_feat[row_id] = feat_dict[key]
else:
add_feat[row_id] = -1
return add_feat<data_type_conversions>
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
def get_answer_feats(row, ans_dict):
feats = np.zeros(4)
 question = int(row[1])
 feats[0] = ans_dict[question][0]
feats[1] = ans_dict[question][1]
feats[2] = ans_dict[question][2]
feats[3] = ans_dict[question][3]
return feats
def add_answer_feats(df, ans_feats):
df['answer1'] = ans_feats[:, 0].astype(np.float32)
 df['answer2'] = ans_feats[:, 1].astype(np.float32)
 df['answer3'] = ans_feats[:, 2].astype(np.float32)
 df['answer4'] = ans_feats[:, 3].astype(np.float32)
 return df<predict_on_test>
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167',\
'g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486',\
'g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298',\
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
meta_features = prepare_meta_feats()
env = riiideducation.make_env()
set_predict = env.predict
iter_test = env.iter_test()<define_variables>
def feat_stats(train, test):
features_g = g_features
features_c = c_features
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in features_c:
df[f'{feature}_cubed'] = df[feature] ** 3
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_cubed'] = df[feature] ** 3
return train, test
train_features2,test_features2=feat_stats(train_features2,test_features2 )
count_dict = {}
correct_dict = {}
time_dict = {}
last_n_dict = {}<drop_column>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
train = train[['user_id', 'content_id', 'timestamp', 'part', 'task_container_id', 'question_hardness', TARGET]]<categorify>
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )
%%time
prev_test_df = None
for(test_df, sample_prediction_df)in iter_test:
if prev_test_df is not None:
prev_test_df[TARGET] = eval(test_df["prior_group_answers_correct"].iloc[0])
 for row_id, prev_row in enumerate(prev_test_df[['user_id', 'content_id', 'timestamp', 'part', 'task_container_id', 'question_hardness', TARGET, 'content_type_id']].values):
if prev_row[7]==0:
count_dict, correct_dict, time_dict, last_n_dict = update_dicts(prev_row, count_dict, correct_dict, time_dict, last_n_dict)
 test_df = pd.merge(test_df, meta_features, on=['content_type_id', 'content_id'], how='left')
 test_df['content_id_target_mean'] = merge_feat_val(test_df['content_id'], content_dict)
 test_df = get_q_hardness_bins(test_df)
 prev_test_df = test_df.copy()
test_df = test_df.sort_values(['user_id','timestamp'], ascending=False)
 test_df = replace_bools(test_df)
 test_df = test_df[test_df['content_type_id'] == 0].reset_index(drop=True)
 user_feats = np.full(( len(test_df), 23), dtype=np.float32, fill_value=np.nan)
 answer_feats = np.zeros(( len(test_df), 4), dtype=np.float32)
 for row_id, curr_row in enumerate(test_df[['user_id', 'content_id', 'timestamp', 'part', 'task_container_id', 'question_hardness']].values):
user = curr_row[0]
if user not in count_dict:
if user in user_idx:
for train_row_id, train_curr_row in enumerate(train[train.user_id==user].values):
count_dict, correct_dict, time_dict, last_n_dict = update_dicts(train_curr_row, count_dict, correct_dict, time_dict, last_n_dict)
 user_feats[row_id] = get_row_values(curr_row, count_dict, correct_dict, time_dict, last_n_dict)
 answer_feats[row_id] = get_answer_feats(curr_row, answer_dist_dict)
 test_df = calc_feats_from_stats(test_df, user_feats)
 test_df = add_answer_feats(test_df, answer_feats)
 test_preds = model.predict(test_df[feats])
 test_df[TARGET] = test_preds
set_predict(test_df[['row_id', TARGET]] )<load_pretrained>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
r<set_options>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))
sns.set()
DEFAULT_FIG_WIDTH = 20
sns.set_context("paper", font_scale = 1.2)
 warnings.filterwarnings('ignore' )<import_modules>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
print('Python : ' + sys.version.split('
')[0])
 print('Numpy : ' + np.__version__)
 print('Pandas : ' + pd.__version__)
 print('PyTorch : ' + torch.__version__ )<train_model>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
 print('Running on device: {}'.format(DEVICE))<set_options>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def seed_everything(s):
random.seed(s)
 os.environ['PYTHONHASHSEED'] = str(s)
 np.random.seed(s)
 torch.manual_seed(s)
 torch.cuda.manual_seed(s)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
if torch.cuda.is_available() :
torch.cuda.manual_seed_all(s )<define_variables>
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
HOME = "./"
DATA_HOME = r'.. /input/riiid-test-answer-prediction'
MODEL_NAME = "SAKT-v1"
MODEL_PATH = HOME + MODEL_NAME
STAGE = "stage1"
MODEL_BEST = 'model_best.pt'
FOLD = 1
MAX_SEQ = 100
CONTENT_TYPE_ID = "content_type_id"
CONTENT_ID = "content_id"
TARGET = "answered_correctly"
USER_ID = "user_id"
TASK_CONTAINER_ID = "task_container_id"
TIMESTAMP = "timestamp"
PART = "part"
ELAPSE = "prior_question_elapsed_time"
DIFF = 'content_difficulty'
CORR = "user_correctness"<load_from_csv>
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
USE_PICKLE = True
dtype = {
USER_ID: 'int32',
CONTENT_ID: 'int16',
CONTENT_TYPE_ID: 'bool',
TARGET:'int8',
}
train_df = dt.fread(os.path.join(DATA_HOME, 'train.csv'), columns = set(dtype.keys())).to_pandas()
train_df = train_df[train_df[CONTENT_TYPE_ID] == False].reset_index(drop = True)
 train_df.head()
user_agg = train_df.groupby(USER_ID)[TARGET].agg(['sum', 'count'] ).astype('int16')
 content_agg = train_df.groupby(CONTENT_ID)[TARGET].agg(['sum', 'count'])
 del train_df; gc.collect()
if not USE_PICKLE:
train_df[DIFF] = train_df[CONTENT_ID].map(content_agg['sum'] / content_agg['count'])
 train_df.fillna(0, inplace = True)
 skills = train_df[CONTENT_ID].unique()
n_skill = 13523
print("Number of skills", n_skill)
 elapse = train_df[ELAPSE].unique()
n_elapse = 300
print("Number of skills", n_elapse)
 batch_size = 100_000
batches = []
for i in range(int(train_df[USER_ID].nunique() //batch_size)+1):
batch = [i * batch_size, min(( i + 1)* batch_size, train_df[USER_ID].nunique())]
batches.append(batch)
 picked_user = []
for i, batch in enumerate(batches):
u_s = train_df[USER_ID].unique() [batch[0]]
u_e = train_df[USER_ID].unique() [batch[1]-1]
picked_user.append([u_s, u_e])
 indx = []
for i, batch in enumerate(picked_user):
idx_s = list(train_df[train_df[USER_ID] == batch[0]].index)[0]
idx_e = list(train_df[train_df[USER_ID] == batch[1]].index)[-1]
indx.append([idx_s, idx_e])
 for i, batch in enumerate(indx):
sub_df = train_df.loc[batch[0]:batch[1]][[USER_ID, CONTENT_ID,
TASK_CONTAINER_ID,
PART, DIFF, ELAPSE,
CORR, TARGET]].groupby(USER_ID ).apply(lambda r:(r[CONTENT_ID].values,
r[TASK_CONTAINER_ID].values,
r[PART].values,
r[DIFF].values,
r[ELAPSE].values,
r[CORR].values,
r[TARGET].values))
sub_df.to_pickle(f'train_group_{i}.pkl')
 del sub_df
del train_df; gc.collect()<load_pretrained>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
train_group = []
for i in range(4):
train_group.append(pd.read_pickle(os.path.join(r'.. /input/riiid-chunking', f'train_group_{i}.pkl')))
 train_group = pd.concat(train_group )<load_from_csv>
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
questions_df = pd.read_csv(
 '.. /input/riiid-test-answer-prediction/questions.csv',
usecols = [0, 3],
dtype = {'question_id': 'int16', 'part': 'int8'}
)<define_variables>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
PAD_TOKEN = -1
START_TOKEN = -2
MASK_TOKEN = -3
special_token = [PAD_TOKEN, START_TOKEN, MASK_TOKEN]
PAD_ID = 0
START_ID = 1
MASK_ID = 2
RESPONSE_FALSE_ID = 3
RESPONSE_TRUE_ID = 4<categorify>
model = Model(num_features, num_all_targets)
 model
responses_total_tokens = np.concatenate(( special_token, [0, 1]))
questions_encoding_mapper = dict(zip(responses_total_tokens, np.arange(len(responses_total_tokens))))<load_pretrained>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()
with open(os.path.join(r'.. /input/r3id-info-public', 'train_valid_split_indices_fold_1.json')) as json_file:
splitting_index = json.load(json_file )<concatenate>
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
def add_start_token(x):
return(*x, np.append([START_TOKEN], x[-1][:-1]))
train_group = train_group.apply(add_start_token)
 train_group<feature_engineering>
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
def train_extraction(user):
if str(user)not in splitting_index.keys() :
return train_group[user]
else:
if splitting_index[str(user)] == 0:
return np.nan
else:
return tuple([seq[:splitting_index[str(user)]] for seq in train_group[user]])
 sub_train_ = train_group.copy()
for i in tqdm(train_group.index):
sub_train_[i] = train_extraction(i)
 sub_train_.dropna(inplace = True )<feature_engineering>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
def valid_extraction(user):
if str(user)not in splitting_index.keys() :
return np.nan
else:
if splitting_index[str(user)] == 0:
return train_group[user]
else:
return tuple([seq[splitting_index[str(user)]:] for seq in train_group[user]])
 valid_df = train_group.copy()
for i in tqdm(train_group.index):
valid_df[i] = valid_extraction(i)
 valid_df.dropna(inplace = True )<concatenate>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )
<define_variables><EOS> 
submission = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 submission.to_csv('submission.csv', index=False)