Kaggle competition: Mechanisms of Action (MoA) Prediction
def get_features_questions_df() :
features_questions_df = pd.DataFrame(index=get_index_np())
 cols = [
'content_id',
'part',
'tag_1',
'tag_2',
'content_count',
'mean_content_id_pos',
'bundle_id',
]
for feature in tqdm(cols):
features_questions_df[feature] = load_feature(feature)
 features_questions_df.drop_duplicates(subset='content_id', inplace=True)
 features_questions_df.sort_values('content_id', inplace=True)
 features_questions_df.reset_index(drop=True, inplace=True)
 return features_questions_df
features_questions_df = get_features_questions_df()
print(f'features_questions_df, rows: {features_questions_df.shape[0]}')
 display(features_questions_df.head() )<create_dataframe>
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
import tensorflow_addons as tfa
from sklearn.model_selection import KFold
from sklearn.metrics import log_loss
from tqdm.notebook import tqdm
def get_state() :
features_df = pd.DataFrame(index=get_index_np())
 cols = [
'user_id', 'content_id', 'answered_correctly','lectures_seen',
'mean_user_accuracy', 'mean_content_accuracy',
'last_correct_time_elapsed', 'last_incorrect_time_elapsed', 'part', 'user_part_answered_consecutive',
'timestamp', 'last_interaction_elapsed_time_l1', 'last_interaction_elapsed_time_l2', 'last_interaction_elapsed_time_l3',
]
for f in tqdm(cols):
features_df[f] = load_feature(f)
 last_features = features_df.groupby('user_id')[[
'timestamp', 'lectures_seen', 'mean_content_accuracy', 'part', 'user_part_answered_consecutive',
'last_correct_time_elapsed', 'last_incorrect_time_elapsed',
'last_interaction_elapsed_time_l1', 'last_interaction_elapsed_time_l2', 'last_interaction_elapsed_time_l3',
]].last()
last_correct_features = features_df.groupby(['user_id', 'answered_correctly'])['timestamp'].last()
features_df.drop([
'timestamp', 'last_interaction_elapsed_time_l1', 'last_interaction_elapsed_time_l2', 'last_interaction_elapsed_time_l3',
'last_correct_time_elapsed', 'last_incorrect_time_elapsed', 'user_part_answered_consecutive',
], axis=1, inplace=True)
 features_df_grouped_by_user = features_df[['user_id', 'answered_correctly']].groupby('user_id')['answered_correctly']
mean_user_accuracy = features_df_grouped_by_user.mean().values.astype(np.float32)
 answered_correctly_user = features_df_grouped_by_user.sum().values.astype(np.uint16)
 answered_user = features_df_grouped_by_user.count().values.astype(np.uint16)
 last_questions_answered_100 = features_df.groupby('user_id')[['user_id', 'answered_correctly']].tail(100 ).groupby('user_id')['answered_correctly'].apply(np.array ).apply(np.flip)
 mean_content_accuracy_sum = features_df.groupby('user_id')['mean_content_accuracy'].sum().values
mean_accuracy_diff_sum =(features_df.groupby('user_id')['mean_user_accuracy'].sum() - features_df.groupby('user_id')['mean_content_accuracy'].sum() ).values
user_part_features = features_df.groupby(['user_id', 'part'])['answered_correctly'].agg(['count', 'sum'] ).reset_index().astype({'user_id': int, 'part': int})
 del features_df_grouped_by_user, features_df
gc.collect()
with open('/kaggle/input/riiid-answer-correctness-prediction-features/state.pkl', 'rb')as state_pickle_file:
state = pickle.load(state_pickle_file)
 for idx, user_id in tqdm(enumerate(state.keys()), total=len(state)) :
state[user_id]['mean_user_accuracy'] = mean_user_accuracy[idx]
state[user_id]['answered_correctly_user'] = answered_correctly_user[idx]
state[user_id]['answered_user'] = answered_user[idx]
state[user_id]['mean_content_accuracy_sum'] = mean_content_accuracy_sum[idx]
state[user_id]['mean_accuracy_diff_sum'] = mean_accuracy_diff_sum[idx]
state[user_id]['lectures_seen'] = last_features.loc[user_id, 'lectures_seen']
state[user_id]['timestamp'] = last_features.loc[user_id, 'timestamp']
state[user_id]['last_mean_content_accuracy'] = last_features.loc[user_id, 'mean_content_accuracy']
state[user_id]['last_part'] = last_features.loc[user_id, 'part']
state[user_id]['user_part_answered_consecutive'] = last_features.loc[user_id, 'user_part_answered_consecutive']
state[user_id]['last_correct_timestamp'] = last_correct_features.loc[user_id, True] if(user_id, True)in last_correct_features else np.nan
state[user_id]['last_incorrect_timestamp'] = last_correct_features.loc[user_id, False] if(user_id, False)in last_correct_features else np.nan
state[user_id]['last_interaction_elapsed_time_l1'] = last_features.loc[user_id, 'last_interaction_elapsed_time_l1']
state[user_id]['last_interaction_elapsed_time_l2'] = last_features.loc[user_id, 'last_interaction_elapsed_time_l2']
state[user_id]['last_interaction_elapsed_time_l3'] = last_features.loc[user_id, 'last_interaction_elapsed_time_l3']
state[user_id]['mean_user_accuracy_r10'] = deque(last_questions_answered_100[user_id][:10], maxlen=10)
 state[user_id]['mean_user_accuracy_r25'] = deque(last_questions_answered_100[user_id][:25], maxlen=25)
 state[user_id]['mean_user_accuracy_r100'] = deque(last_questions_answered_100[user_id], maxlen=100)
 for idx,(user_id, part, part_count, part_sum)in tqdm(user_part_features.iterrows() , total=user_part_features.index.size):
state[user_id][f'part_{part}_count'] = part_count
state[user_id][f'part_{part}_sum'] = part_sum
return state
state = get_state()
gc.collect()<define_variables>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
def get_new_user(row):
return {
'mean_user_accuracy': 0.680,
'answered_correctly_user': 0,
'answered_user': 0,
'user_content_attempts': dict() ,
'lectures_seen': 0,
'timestamp': row['timestamp'],
'last_mean_content_accuracy': 0,
'last_part': int(row['part']),
'user_part_answered_consecutive': 0,
'last_correct_timestamp': np.nan,
'last_incorrect_timestamp': np.nan,
'last_interaction_elapsed_time_l1': 0,
'last_interaction_elapsed_time_l2': 0,
'last_interaction_elapsed_time_l3': 0,
'mean_user_accuracy_r10': deque([], maxlen=10),
'mean_user_accuracy_r25': deque([], maxlen=25),
'mean_user_accuracy_r100': deque([], maxlen=100),
'mean_accuracy_diff_sum': 0,
'mean_content_accuracy_sum': 0,
}
def get_user_data_dict() :
return {
'mean_user_accuracy': [],
'answered_correctly_user': [],
'answered_user': [],
'lectures_seen': [],
'last_interaction_elapsed_time_l1': [],
'last_interaction_elapsed_time_l2': [],
'last_interaction_elapsed_time_l3': [],
'mean_user_accuracy_r10': [],
'mean_user_accuracy_r25': [],
'mean_user_accuracy_r100': [],
'mean_user_part_accuracy': [],
'answered_part_user': [],
'mean_accuracy_diff': [],
'mean_user_content_accuracy': [],
'last_question_mean_acc_diff': [],
'user_part_answered_consecutive': [],
'mean_user_mean_content_acc_diff': [],
'last_correct_time_elapsed': [],
'last_incorrect_time_elapsed': [],
}<load_from_csv>
def preprocess(df):
df = df.copy()
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
train_targets = train_targets.loc[train['cp_type']==0].reset_index(drop=True)
 train = train.loc[train['cp_type']==0].reset_index(drop=True )
lectures = pd.read_csv('/kaggle/input/riiid-test-answer-prediction/lectures.csv', dtype={ 'tag': np.uint8 })[['lecture_id', 'tag']]
content_id2tag = pd.Series(lectures['tag'].values, index=lectures['lecture_id'].values)
 def get_user_data(state, test_df):
user_data = get_user_data_dict()
part_first_question_mean_accuracy_dict = {1: 0.75, 2: 0.60, 3: 0.49, 4: 0.41, 5: 0.52, 6: 0.51, 7: 0.47}
cols = ['user_id', 'content_id', 'content_type_id', 'timestamp', 'part', 'mean_content_accuracy']
for idx,(user_id, content_id, is_lecture, timestamp, part, mean_content_accuracy)in test_df[cols].iterrows() :
if is_lecture:
state[user_id]['lectures_seen'] += 1
for key in user_data.keys() :
user_data[key].append(0)
 else:
part = int(part)
 state[user_id]['last_interaction_elapsed_time_l3'] = state[user_id]['last_interaction_elapsed_time_l2']
state[user_id]['last_interaction_elapsed_time_l2'] = state[user_id]['last_interaction_elapsed_time_l1']
if timestamp != state[user_id]['timestamp']:
state[user_id]['last_interaction_elapsed_time_l1'] = timestamp - state[user_id]['timestamp']
state[user_id]['timestamp'] = timestamp
cols = [
'mean_user_accuracy', 'answered_correctly_user', 'answered_user', 'lectures_seen',
'last_interaction_elapsed_time_l1', 'last_interaction_elapsed_time_l2', 'last_interaction_elapsed_time_l3',
]
for feature in cols:
user_data[feature].append(state[user_id][feature])
 if state[user_id]['answered_user'] > 0:
user_data['mean_accuracy_diff'].append(state[user_id]['mean_accuracy_diff_sum'] / state[user_id]['answered_user'])
 user_data['mean_user_content_accuracy'].append(state[user_id]['mean_content_accuracy_sum'] / state[user_id]['answered_user'])
 user_data['mean_user_accuracy_r10'].append(np.array(state[user_id]['mean_user_accuracy_r10'] ).mean())
 user_data['mean_user_accuracy_r25'].append(np.array(state[user_id]['mean_user_accuracy_r25'] ).mean())
 user_data['mean_user_accuracy_r100'].append(np.array(state[user_id]['mean_user_accuracy_r100'] ).mean())
 else:
user_data['mean_accuracy_diff'].append(0)
 user_data['mean_user_content_accuracy'].append(0)
 user_data['mean_user_accuracy_r10'].append(0.68)
 user_data['mean_user_accuracy_r25'].append(0.68)
 user_data['mean_user_accuracy_r100'].append(0.68)
 if state[user_id]['last_mean_content_accuracy'] != 0:
user_data['last_question_mean_acc_diff'].append(mean_content_accuracy - state[user_id]['last_mean_content_accuracy'])
 else:
user_data['last_question_mean_acc_diff'].append(0)
 state[user_id]['last_mean_content_accuracy'] = mean_content_accuracy
if f'part_{part}_count' in state[user_id]:
if state[user_id][f'part_{part}_sum'] == 0:
user_data['mean_user_part_accuracy'].append(0)
 else:
user_data['mean_user_part_accuracy'].append(state[user_id][f'part_{part}_sum'] / state[user_id][f'part_{part}_count'])
 user_data['answered_part_user'].append(state[user_id][f'part_{part}_count'])
 else:
state[user_id][f'part_{part}_sum'] = 0
state[user_id][f'part_{part}_count'] = 0
user_data['mean_user_part_accuracy'].append(part_first_question_mean_accuracy_dict[part])
 user_data['answered_part_user'].append(0)
 if part == state[user_id]['last_part']:
state[user_id]['user_part_answered_consecutive'] += 1
else:
state[user_id]['last_part'] = part
state[user_id]['user_part_answered_consecutive'] = 0
user_data['user_part_answered_consecutive'].append(state[user_id]['user_part_answered_consecutive'])
 user_data['mean_user_mean_content_acc_diff'].append(mean_content_accuracy - state[user_id]['mean_user_accuracy'])
 user_data['last_correct_time_elapsed'].append(timestamp - state[user_id]['last_correct_timestamp'])
 user_data['last_incorrect_time_elapsed'].append(timestamp - state[user_id]['last_incorrect_timestamp'])
 return user_data<feature_engineering>
def nn_model(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Dense(1024),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.41),
tf.keras.layers.PReLU() ,
tf.keras.layers.Dense(512),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.38),
tf.keras.layers.PReLU() ,
tf.keras.layers.Dense(256),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.35),
tf.keras.layers.PReLU() ,
tf.keras.layers.Dense(206, activation="sigmoid")
 ])
 model.compile(tfa.optimizers.AdamW(lr = 1e-3, weight_decay = 1e-5, clipvalue = 900),
loss='binary_crossentropy', 
)
 return model
def add_new_users(test_df):
for idx, row in test_df.iterrows() :
if not row['user_id'] in state:
state[row['user_id']] = get_new_user(row )<feature_engineering>
def add_attempt_retry(test_df):
attempt = []
for user_id, content_id in test_df[['user_id', 'content_id']].itertuples(name=None, index=False):
if content_id in state[user_id]['user_content_attempts']:
state[user_id]['user_content_attempts'][content_id] += 1
else:
state[user_id]['user_content_attempts'][content_id] = 0
attempt.append(state[user_id]['user_content_attempts'][content_id])
 test_df['attempt'] = attempt
test_df['retry'] = test_df['attempt'] > 0<load_pretrained>
top_feats = [ 0, 1, 2, 3, 5, 6, 8, 9, 10, 11, 12, 14, 15,
16, 18, 19, 20, 21, 23, 24, 25, 27, 28, 29, 30, 31,
32, 33, 34, 35, 36, 37, 39, 40, 41, 42, 44, 45, 46,
48, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
63, 64, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76,
78, 79, 80, 81, 82, 83, 84, 86, 87, 88, 89, 90, 92,
93, 94, 95, 96, 97, 99, 100, 101, 103, 104, 105, 106, 107,
108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120,
121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 132, 133, 134,
135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147,
149, 150, 151, 152, 153, 154, 155, 157, 159, 160, 161, 163, 164,
165, 166, 167, 168, 169, 170, 172, 173, 175, 176, 177, 178, 180,
181, 182, 183, 184, 186, 187, 188, 189, 190, 191, 192, 193, 195,
197, 198, 199, 202, 203, 205, 206, 208, 209, 210, 211, 212, 213,
214, 215, 218, 219, 220, 221, 222, 224, 225, 227, 228, 229, 230,
231, 232, 233, 234, 236, 238, 239, 240, 241, 242, 243, 244, 245,
246, 248, 249, 250, 251, 253, 254, 255, 256, 257, 258, 259, 260,
261, 263, 265, 266, 268, 270, 271, 272, 273, 275, 276, 277, 279,
282, 283, 286, 287, 288, 289, 290, 294, 295, 296, 297, 299, 300,
301, 302, 303, 304, 305, 306, 308, 309, 310, 311, 312, 313, 315,
316, 317, 320, 321, 322, 324, 325, 326, 327, 328, 329, 330, 331,
332, 333, 334, 335, 338, 339, 340, 341, 343, 344, 345, 346, 347,
349, 350, 351, 352, 353, 355, 356, 357, 358, 359, 360, 361, 362,
363, 364, 365, 366, 368, 369, 370, 371, 372, 374, 375, 376, 377,
378, 379, 380, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391,
392, 393, 394, 395, 397, 398, 399, 400, 401, 403, 405, 406, 407,
408, 410, 411, 412, 413, 414, 415, 417, 418, 419, 420, 421, 422,
423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435,
436, 437, 438, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450,
452, 453, 454, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465,
466, 468, 469, 471, 472, 473, 474, 475, 476, 477, 478, 479, 482,
483, 485, 486, 487, 488, 489, 491, 492, 494, 495, 496, 500, 501,
502, 503, 505, 506, 507, 509, 510, 511, 512, 513, 514, 516, 517,
518, 519, 521, 523, 525, 526, 527, 528, 529, 530, 531, 532, 533,
534, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547,
549, 550, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563,
564, 565, 566, 567, 569, 570, 571, 572, 573, 574, 575, 577, 580,
581, 582, 583, 586, 587, 590, 591, 592, 593, 595, 596, 597, 598,
599, 600, 601, 602, 603, 605, 607, 608, 609, 611, 612, 613, 614,
615, 616, 617, 619, 622, 623, 625, 627, 630, 631, 632, 633, 634,
635, 637, 638, 639, 642, 643, 644, 645, 646, 647, 649, 650, 651,
652, 654, 655, 658, 659, 660, 661, 662, 663, 664, 666, 667, 668,
669, 670, 672, 674, 675, 676, 677, 678, 680, 681, 682, 684, 685,
686, 687, 688, 689, 691, 692, 694, 695, 696, 697, 699, 700, 701,
702, 703, 704, 705, 707, 708, 709, 711, 712, 713, 714, 715, 716,
717, 723, 725, 727, 728, 729, 730, 731, 732, 734, 736, 737, 738,
739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751,
752, 753, 754, 755, 756, 758, 759, 760, 761, 762, 763, 764, 765,
766, 767, 769, 770, 771, 772, 774, 775, 780, 781, 782, 783, 784,
785, 787, 788, 790, 793, 795, 797, 799, 800, 801, 805, 808, 809,
811, 812, 813, 816, 819, 820, 821, 822, 823, 825, 826, 827, 829,
831, 832, 833, 834, 835, 837, 838, 839, 840, 841, 842, 844, 845,
846, 847, 848, 850, 851, 852, 854, 855, 856, 858, 860, 861, 862,
864, 867, 868, 870, 871, 873, 874]
print(len(top_feats))
with open('/kaggle/input/riiid-answer-correctness-prediction-features/mean_content_accuracy_cases_dict.pickle', 'rb')as f:
mean_content_accuracy_cases_dict = pickle.load(f)
 def add_mean_content_accuracy(test_df):
mean_content_accuracy = []
for key in test_df[['content_id', 'prior_question_had_explanation', 'retry']].itertuples(name=None, index=False):
if key in mean_content_accuracy_cases_dict:
mean_content_accuracy.append(mean_content_accuracy_cases_dict[key])
 else:
mean_content_accuracy.append(0)
 test_df['mean_content_accuracy'] = mean_content_accuracy<define_variables>
def metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels=[0,1]))
return np.mean(metrics )
for(content_id, prior_question_had_explanation, retry), mean_content_accuracy in mean_content_accuracy_cases_dict.items() :
if content_id == 6116:
print(f'content_id {content_id}, prior_question_had_explanation: {prior_question_had_explanation}, retry: {retry}, mean_content_accuracy: {mean_content_accuracy}' )<feature_engineering>
print(f'OOF Metric: {metric(train_targets, res)}' )
def update_user_data(state, prev_test_df):
for idx, row in prev_test_df.iterrows() :
if not row['content_type_id']:
answered_correctly = row['answered_correctly']
user_id = row['user_id']
part = int(row['part'])
 state[user_id]['answered_correctly_user'] += answered_correctly
state[user_id]['answered_user'] += 1
state[user_id]['mean_user_accuracy'] = state[user_id]['answered_correctly_user'] / state[user_id]['answered_user']
state[user_id]['mean_user_accuracy_r10'].appendleft(answered_correctly)
 state[user_id]['mean_user_accuracy_r25'].appendleft(answered_correctly)
 state[user_id]['mean_user_accuracy_r100'].appendleft(answered_correctly)
 state[user_id][f'part_{part}_sum'] += answered_correctly
state[user_id][f'part_{part}_count'] += 1
state[user_id]['mean_accuracy_diff_sum'] += row['mean_user_accuracy'] - row['mean_content_accuracy']
state[user_id]['mean_content_accuracy_sum'] += row['mean_content_accuracy']
if answered_correctly:
state[user_id]['last_correct_timestamp'] = row['timestamp']
else:
state[user_id]['last_incorrect_timestamp'] = row['timestamp']<split>
ss.loc[test['cp_type']==1, train_targets.columns] = 0
env = riiideducation.make_env()
iter_test = env.iter_test()<feature_engineering>
ss.to_csv('submission.csv', index=False )