Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<drop_column>
device_lib.list_local_devices()
combdata = combdata.drop(labels='Cabin', axis=1 )<drop_column>
lectures_df = pd.read_csv('.. /input/nadare-processed-2/lectures.csv')
 questions_df = pd.read_csv('.. /input/nadare-processed-2/questions.csv')
 choices, timestamps, elapsed_times, orders, explanations, user_id_map = pd.read_pickle(".. /input/nadare-processed-2/raw_items.pickle")
 timestamp_bins, elapsedtime_bins = pd.read_pickle(".. /input/nadare-processed-2/bins.pickle" )
combdata = combdata.drop(labels='Embarked', axis=1 )<drop_column>
timestamp_bins = tf.constant(timestamp_bins, dtype="float32")
 elapsedtime_bins = tf.constant(elapsedtime_bins, dtype="float32" )
combdata =combdata.drop(labels=['Age', 'Fare', 'Name'],axis = 1 )<import_modules>
delta_bins =pd.read_pickle(".. /input/nadare-processed-2/delta_bins.pickle")
 delta_bins[-2:] = 1e18
from sklearn.svm import SVC
from collections import Counter
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold, learning_curve
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier, VotingClassifier<drop_column>
qp_original_sum = pd.read_pickle(".. /input/nadare-processed-2/question_part_agg.pickle" ).sum()
part_mean = tf.constant([qp_original_sum[f"part{i}_sum"] / qp_original_sum[f"part{i}_count"] for i in range(7)], dtype="float32")
 part_mean = tf.concat([part_mean, [qp_original_sum[f"question_sum"] / qp_original_sum[f"question_count"]]], axis=0)
 part_rate = tf.constant([qp_original_sum[f"part{i}_count"] / qp_original_sum["question_count"] for i in range(7)], dtype="float32" )
train = combdata.loc[combdata['source']=="train"]
test = combdata.loc[combdata['source']=="test"]
test.drop(labels=["Survived"],axis = 1,inplace=True)
 train.drop(labels=["source"],axis = 1,inplace=True)
 test.drop(labels=["source"],axis = 1,inplace=True)
 test.shape<prepare_x_and_y>
qp_agg_df = pd.read_pickle(".. /input/nadare-processed-2/question_part_agg.pickle")
 qp_agg_df = qp_agg_df.set_index("user_ix" ).astype(np.float32 )
train["Survived"] = train["Survived"].astype(int)
 Y_train = train["Survived"]
X_train = train.drop(labels = ["Survived"],axis = 1)
 X_train.shape<choose_model_class>
env = riiideducation.make_env()
iter_test = env.iter_test()
kfold = StratifiedKFold(n_splits=10 )<choose_model_class>
lectures_df["type_of"] = lectures_df["type_of"].map({k: v for v, k in enumerate(lectures_df["type_of"].unique())} )
random_state = 2
classifiers = []
classifiers.append(KNeighborsClassifier())
 classifiers.append(LinearDiscriminantAnalysis())
 classifiers.append(SVC(random_state=random_state))
classifiers.append(MLPClassifier(random_state=random_state))
classifiers.append(ExtraTreesClassifier(random_state=random_state))
classifiers.append(LogisticRegression(random_state = random_state))
classifiers.append(DecisionTreeClassifier(random_state=random_state))
classifiers.append(RandomForestClassifier(random_state=random_state))
classifiers.append(GradientBoostingClassifier(random_state=random_state))
classifiers.append(AdaBoostClassifier(DecisionTreeClassifier(random_state=random_state),random_state=random_state,learning_rate=0.1))
cv_results = []
for classifier in classifiers :
cv_results.append(cross_val_score(classifier, X_train, y = Y_train, scoring = "accuracy", cv = kfold, n_jobs=4))
cv_means = []
cv_std = []
for cv_result in cv_results:
cv_means.append(cv_result.mean())
 cv_std.append(cv_result.std())
 cv_res = pd.DataFrame({"CrossValMeans":cv_means,"CrossValerrors": cv_std,
"Algorithm":["SVC",
"AdaBoost",
"ExtraTrees",
"KNeighboors",
"DecisionTree",
"RandomForest",
"GradientBoosting",
"LogisticRegression",
"MultipleLayerPerceptron",
"LinearDiscriminantAnalysis"]})
 g = sns.barplot("CrossValMeans","Algorithm",data = cv_res, palette="Set3",orient = "h",**{'xerr':cv_std})
 g.set_xlabel("Mean Accuracy")
 g = g.set_title("Cross validation scores" )<choose_model_class>
choice_is_correct = [False] *(( lectures_df["content_id"].max() +1)* 4)
 choice_is_wrong = [False] *(( lectures_df["content_id"].max() +1)* 4)
 choice_is_question = [False] *(( lectures_df["content_id"].max() +1)* 4)
 answer_accuracy_mean = [questions_df["content_4_accuracy"].mean() ] *(( lectures_df["content_id"].max() +1)* 4)
 answer_accuracy2_mean = [questions_df["content_2_accuracy"].mean() ] *(( lectures_df["content_id"].max() +1)* 4)
 answer_elapsed_time_mean = [questions_df["content_elapsed_time_mean"].mean() ] *(( lectures_df["content_id"].max() +1)* 4)
 answer_correct_elapssed_time_mean = [questions_df["correct_elapsed_time_mean"].mean() ] *(( lectures_df["content_id"].max() +1)* 4)
 answer_correct_et_rate = [1.] *(( lectures_df["content_id"].max() +1)* 4)
 answer_elapsed_time_std = [questions_df["content_elapsed_time_std"].mean() ] *(( lectures_df["content_id"].max() +1)* 4)
 choice_rate = [[0]*4 for _ in range(( lectures_df["content_id"].max() +1)* 4)]
correct_choice = [-1] *(( lectures_df["content_id"].max() +1)* 4)
 choice_parts = [-1] *(( lectures_df["content_id"].max() +1)* 4)
 choice_typeof = [-1] *(( lectures_df["content_id"].max() +1)* 4)
 choice_tags = [[] for _ in range(( lectures_df["content_id"].max() +1)* 4)]
for i, row in questions_df.iterrows() :
tags = [] if pd.isna(row["tags"])else list(map(int, row["tags"].split()))
for i in range(4):
choice_is_correct[i + row["content_id"]*4] =(i == row["correct_answer"])
 choice_is_wrong[i + row["content_id"]*4] =(i != row["correct_answer"])
 choice_is_question[i + row["content_id"]*4] = True
answer_accuracy_mean[i + row["content_id"]*4] = row["content_4_accuracy"]
answer_accuracy2_mean[i + row["content_id"]*4] = row["content_2_accuracy"]
answer_elapsed_time_mean[i + row["content_id"]*4] = row["content_elapsed_time_mean"]
answer_correct_elapssed_time_mean[i + row["content_id"]*4] = row["correct_elapsed_time_mean"]
answer_correct_et_rate[i + row["content_id"]*4] = row["correct_elapsed_time_mean"] / row["content_elapsed_time_mean"]
answer_elapsed_time_std[i + row["content_id"]*4] = row["content_elapsed_time_std"]
for j in range(4):
choice_rate[i + row["content_id"]*4][j] = row[f"choice_rate_{j}"]
correct_choice[i + row["content_id"]*4] = row["correct_choice"]
choice_tags[i + row["content_id"]*4] = [t for t in tags]
choice_parts[i + row["content_id"]*4] = int(row["part"])
 for i, row in lectures_df.iterrows() :
tags = [row["tag"]]
for i in range(4):
cid = int(row["content_id"])
 choice_tags[i + cid*4] = [t for t in tags]
choice_parts[i + cid*4] = int(row["part"])
 choice_typeof[i + cid*4] = int(row["type_of"])
 choice_is_correct = tf.constant(choice_is_correct)
 choice_is_wrong = tf.constant(choice_is_wrong)
 choice_is_question = tf.constant(choice_is_question)
 answer_accuracy_mean = tf.constant(answer_accuracy_mean)
 answer_accuracy2_mean = tf.constant(answer_accuracy2_mean)
 answer_elapsed_time_mean = tf.constant(answer_elapsed_time_mean)
 answer_elapsed_time_std = tf.constant(answer_elapsed_time_std)
 correct_choice = tf.constant(correct_choice, dtype="int32")
 choice_rate = tf.constant(choice_rate, dtype="float32")
 choice_parts = tf.constant(choice_parts, dtype="int32")
 choice_typeof = tf.constant(choice_typeof, dtype="int32")
 choice_tags = tf.keras.preprocessing.sequence.pad_sequences(choice_tags, dtype="int16", value=-1, padding="post")+ 1
DTC = DecisionTreeClassifier()
adaDTC = AdaBoostClassifier(DTC, random_state=7)
 ada_param_grid = {"base_estimator__criterion" : ["gini", "entropy"],
"base_estimator__splitter" : ["best", "random"],
"algorithm" : ["SAMME","SAMME.R"],
"n_estimators" :[1,2],
"learning_rate": [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3,1.5]}
gsadaDTC = GridSearchCV(adaDTC,param_grid = ada_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsadaDTC.fit(X_train,Y_train)
 ada_best = gsadaDTC.best_estimator_
gsadaDTC.best_score_<train_on_grid>
question_id_map = {id_: i+1 for i, id_ in enumerate(questions_df["question_id"])}
lecture_id_map = {id_: i+questions_df.shape[0]+1 for i, id_ in enumerate(lectures_df["lecture_id"])}
ExtC = ExtraTreesClassifier()
ex_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsExtC = GridSearchCV(ExtC,param_grid = ex_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsExtC.fit(X_train,Y_train)
 ExtC_best = gsExtC.best_estimator_
gsExtC.best_score_<train_on_grid>
class BaseModel(tf.keras.Model):
def __init__(self, user_in_train,
content_embedding_size, choice_embedding_size,
choice_is_correct, choice_is_wrong, choice_is_question,
choice_tags, choice_parts, choice_typeof,
answer_accuracy_mean, answer_accuracy2_mean, answer_elapsed_time_mean, answer_elapsed_time_std, answer_correct_elapssed_time_mean,
correct_choice, choice_rate,
sample_window=500):
super(BaseModel, self ).__init__()
self.content_embedding = tf.keras.layers.Embedding(content_embedding_size+2, 64, mask_zero=True)
 self.choice_embedding = tf.keras.layers.Embedding(choice_embedding_size+2, 64, mask_zero=True)
 self.choice_tags_vec = tf.keras.layers.Embedding(choice_tags.shape[0],
choice_tags.shape[1],
weights=[choice_tags],
embeddings_initializer = None,
dtype="int16",
trainable=False)
 self.tag_embedding = tf.keras.layers.Embedding(choice_embedding_size+2, 16, mask_zero=True)
 self.timestamp_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)
 self.lag_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)
 self.delta_timestamp_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)
 self.cooltime_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)
 self.elapsed_embedding = tf.keras.layers.Embedding(100, 32, mask_zero=True)
 self.positional_embedding = tf.keras.layers.Embedding(500, 32, mask_zero=True)
 self.choice_is_correct = choice_is_correct
self.choice_is_wrong = choice_is_wrong
self.choice_is_question = choice_is_question
self.choice_tags = choice_tags
self.choice_parts = choice_parts
self.choice_typeof = choice_typeof
self.answer_accuracy_mean = answer_accuracy_mean
self.answer_accuracy2_mean = answer_accuracy2_mean
self.answer_elapsed_time_mean = answer_elapsed_time_mean
self.answer_elapsed_time_std = answer_elapsed_time_std
self.answer_correct_elapssed_time_mean = answer_correct_elapssed_time_mean
self.correct_choice = correct_choice
self.choice_rate = choice_rate
_ = self.content_embedding(0)
 _ = self.choice_embedding(0)
 self.order_in = tf.keras.layers.InputLayer(input_shape=(1,))
self.feat_in = tf.keras.layers.InputLayer(input_shape=(76,))
self.oh_in = tf.keras.layers.InputLayer(input_shape=(7,))
self.table_bn_clipper_qc = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.bn_clipper_history = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.bn_clipper_0 = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.bn_clipper_1 = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.bn_clipper_2 = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.dense_qc_u0 = tf.keras.layers.Dense(64)
 self.dense_qc_v0 = tf.keras.layers.Dense(305)
 self.bn_qc0 = tf.keras.layers.BatchNormalization()
self.dense_qc_u1 = tf.keras.layers.Dense(64)
 self.dense_qc_v1 = tf.keras.layers.Dense(305)
 self.bn_qc1 = tf.keras.layers.BatchNormalization()
self.dense_kv_u0 = tf.keras.layers.Dense(64)
 self.dense_kv_v0 = tf.keras.layers.Dense(312)
 self.bn_kv0 = tf.keras.layers.BatchNormalization()
self.dense_kv_u1 = tf.keras.layers.Dense(64)
 self.dense_kv_v1 = tf.keras.layers.Dense(312)
 self.bn_kv1 = tf.keras.layers.BatchNormalization()
self.zero_embedding = tf.keras.layers.Embedding(1, 312)
 self.encoder_masks = []
for i in range(5):
row = tf.reshape(tf.repeat(tf.range(100*(i+1)) , 100),(100*(i+1), 100))
col = tf.reshape(tf.tile(tf.range(100)+100*i, [100*(i+1)]),(100*(i+1), 100))
self.encoder_masks.append(tf.cast(tf.expand_dims(tf.transpose(tf.where(row > col, tf.float32.min/100, 0.)), axis=0), "float32"))
self.dense_sa_k0 = tf.keras.layers.Dense(64)
 self.bn_sa_k0 = tf.keras.layers.BatchNormalization()
self.do_sa_k0 = tf.keras.layers.Dropout(0.1)
 self.dense_ts_k0 = tf.keras.layers.Dense(64)
 self.bn_ts_k0 = tf.keras.layers.BatchNormalization()
self.do_ts_k0 = tf.keras.layers.Dropout(0.1)
 self.do_ts_k1 = tf.keras.layers.Dropout(0.1)
 self.history_key_conv0 = tf.keras.layers.Conv1D(32, 1, 1, activation="selu")
 self.history_key_conv1 = tf.keras.layers.Conv1D(32, 1, 1)
 self.history_correct_key_conv = tf.keras.layers.Conv1D(32, 1, 1, activation="relu")
 self.history_choice_conv0 = tf.keras.layers.Conv1D(32, 1, 1, activation="selu")
 self.history_choice_conv1 = tf.keras.layers.Conv1D(32, 1, 1)
 self.history_wrong_choice_conv = tf.keras.layers.Conv1D(32, 1, 1, activation="relu")
 self.history_val_conv0 = tf.keras.layers.Conv1D(32, 1, 1, activation="selu")
 self.history_val_conv1 = tf.keras.layers.Conv1D(32, 1, 1)
 self.gloval_avg_pool_key_drop = tf.keras.layers.Dropout(0.1)
 self.gloval_avg_pool_choice_drop = tf.keras.layers.Dropout(0.1)
 self.gloval_avg_pool_val_drop = tf.keras.layers.Dropout(0.1)
 self.gloval_max_pool_key = tf.keras.layers.GlobalMaxPool1D()
self.gloval_avg_pool_key = tf.keras.layers.GlobalAvgPool1D()
self.gloval_max_pool_correct_key = tf.keras.layers.GlobalMaxPool1D()
self.gloval_max_pool_choice = tf.keras.layers.GlobalMaxPool1D()
self.gloval_avg_pool_choice = tf.keras.layers.GlobalAvgPool1D()
self.gloval_max_pool_wrong_choice = tf.keras.layers.GlobalMaxPool1D()
self.gloval_max_pool_val = tf.keras.layers.GlobalMaxPool1D()
self.gloval_avg_pool_val = tf.keras.layers.GlobalAvgPool1D()
self.dense_u0 = tf.keras.layers.Dense(256)
 self.dense_v0 = tf.keras.layers.Dense(1214)
 self.dense_u1 = tf.keras.layers.Dense(256)
 self.dense_v1 = tf.keras.layers.Dense(1214)
 self.dense_u2 = tf.keras.layers.Dense(256)
 self.dense_v2 = tf.keras.layers.Dense(1214)
 self.bnc0 = tf.keras.layers.BatchNormalization()
self.bnc1 = tf.keras.layers.BatchNormalization()
self.bnc2 = tf.keras.layers.BatchNormalization()
self.dense1 = tf.keras.layers.Dense(512, activation='relu')
 self.dense2 = tf.keras.layers.Dense(512, activation='relu')
 self.dense3 = tf.keras.layers.Dense(512, activation='relu')
 self.bn0 = tf.keras.layers.BatchNormalization()
self.bn1 = tf.keras.layers.BatchNormalization()
self.bn2 = tf.keras.layers.BatchNormalization()
self.bn3 = tf.keras.layers.BatchNormalization()
self.do0 = tf.keras.layers.Dropout(0)
 self.do1 = tf.keras.layers.Dropout(3/10)
 self.do2 = tf.keras.layers.Dropout(3/10)
 self.do3 = tf.keras.layers.Dropout(0)
 self.out = tf.keras.layers.Dense(1, activation='sigmoid')
 self.sample_window = sample_window
@tf.function(input_signature=[
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int16),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int64),
tf.TensorSpec(shape=(None, 500), dtype=tf.int16),
tf.TensorSpec(shape=(None, 500), dtype=tf.float32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int16),
tf.TensorSpec(shape=(None, 500), dtype=tf.float32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=(None, 76), dtype=tf.float32),
tf.TensorSpec(shape=[], dtype=tf.bool),
] 
)
 def call(self, user_ix, content_id, timestamp_cat, lag_time, order,
choice_history, timestamp_history, delta_timestamp_history,
raw_elapsed_time_history, elapsed_time_history,
explanation_history, cooltime_history,
part, feat,
training=None):
if training:
user_ix = tf.where(tf.random.uniform([len(user_ix)], maxval=1)<.01, 0, user_ix)
 content_emb = self.content_embedding(content_id)
 timestamp_emb = self.timestamp_embedding(tf.reshape(timestamp_cat,(-1,)))
 choice_emb = tf.reduce_mean(tf.concat([tf.expand_dims(self.choice_embedding(content_id*4 + i), axis=1)for i in range(4)], axis=1), axis=1)
 lag_emb = self.lag_embedding(tf.reshape(lag_time,(-1,)))
 order_in = self.order_in(tf.expand_dims(tf.cast(order, "float32"), axis=-1))
order_in = tf.math.log1p(order_in)
 feat_in = self.feat_in(feat)
 part_oh = tf.reshape(self.oh_in(tf.one_hot(tf.cast(part, "int32"), 7)) ,(-1, 7))
content_tags_emb = self.tag_embedding(self.choice_tags_vec(tf.reshape(content_id*4,(-1,))))
emb_max = tf.keras.backend.max(content_tags_emb, axis=1)
 emb_min = tf.keras.backend.min(content_tags_emb, axis=1)
 content_tags_emb = tf.where(emb_max > -emb_min, emb_max, emb_min)
 correct_choice = tf.reshape(tf.one_hot(tf.gather(self.correct_choice, content_id*4), 4),(-1, 4))
choice_rate = tf.reshape(tf.gather(self.choice_rate, content_id*4),(-1, 4))
content_accuracy = tf.reshape(tf.gather(self.answer_accuracy_mean, content_id*4),(-1, 1))
content_norm = tf.stop_gradient(tf.math.log1p(tf.norm(content_emb, axis=1, keepdims=True)))
 timestamp_norm = tf.stop_gradient(tf.math.log1p(tf.norm(timestamp_emb, axis=1, keepdims=True)))
 choice_norm = tf.stop_gradient(tf.math.log1p(tf.norm(choice_emb, axis=1, keepdims=True)))
 lag_norm = tf.stop_gradient(tf.math.log1p(tf.norm(lag_emb, axis=1, keepdims=True)))
 tag_norm = tf.stop_gradient(tf.math.log1p(tf.norm(content_tags_emb, axis=1, keepdims=True)))
 table_features = tf.reshape(tf.concat([order_in, feat_in, part_oh, content_norm, timestamp_norm, choice_norm, lag_norm, tag_norm, correct_choice, choice_rate], axis=-1),(-1, 97))
table_features = self.table_bn_clipper_qc(table_features)
 query_context_features_0 = tf.concat([content_emb, choice_emb, timestamp_emb, lag_emb, content_tags_emb, table_features], axis=1)
 query_context_features_0 = tf.reshape(query_context_features_0,(-1, 305))
query_context_features = query_context_features_0 * self.dense_qc_v0(self.dense_qc_u0(self.bn_qc0(query_context_features_0)))+ query_context_features_0
query_context_features = query_context_features_0 * self.dense_qc_v1(self.dense_qc_u1(self.bn_qc1(query_context_features)))+ query_context_features
content_mask = choice_history > 0
idx = tf.cast(tf.where(content_mask), "int32")
 content_history = choice_history // 4
correct_history = tf.expand_dims(tf.gather(tf.where(self.choice_is_correct, 1., 0.) , choice_history), axis=-1)
 wrong_history = tf.expand_dims(tf.gather(tf.where(self.choice_is_wrong, 1., 0.) , choice_history), axis=-1)
 question_history = tf.expand_dims(tf.gather(tf.where(self.choice_is_question, 1., 0.) , choice_history), axis=-1)
 accuracy_history = tf.expand_dims(tf.where(tf.reshape(question_history,(-1, 500)) ==1., tf.gather(self.answer_accuracy_mean, choice_history), 0), axis=-1)
 history_choice_emb = self.choice_embedding(tf.boolean_mask(choice_history, content_mask))
history_delta_timestamp_emb = self.delta_timestamp_embedding(tf.boolean_mask(delta_timestamp_history, content_mask))
history_elapsed_time_emb = self.elapsed_embedding(tf.boolean_mask(elapsed_time_history, content_mask))
history_cooltime_emb = self.cooltime_embedding(tf.boolean_mask(cooltime_history, content_mask))
history_explanation = tf.expand_dims(tf.cast(tf.boolean_mask(explanation_history, content_mask), "float32"), -1)
 history_position_emb = self.positional_embedding(tf.boolean_mask(tf.reshape(tf.tile(tf.range(self.sample_window), [len(user_ix)]),(len(user_ix), 500)) , content_mask))
masked_choice = tf.boolean_mask(choice_history, content_mask)
 history_choice = tf.reshape(tf.one_hot(tf.where(tf.gather(self.choice_is_question, masked_choice), masked_choice%4, -1), 4),(-1, 4))
history_correct_choice = tf.reshape(tf.one_hot(tf.gather(self.correct_choice, masked_choice), 4),(-1, 4))
history_choice_rate = tf.reshape(tf.gather(self.choice_rate, masked_choice),(-1, 4))
history_is_correct = tf.expand_dims(tf.gather(tf.where(self.choice_is_correct, 1., 0.) , tf.boolean_mask(choice_history, content_mask)) , -1)
 history_is_wrong = tf.expand_dims(tf.gather(tf.where(self.choice_is_wrong, 1., 0.) , tf.boolean_mask(choice_history, content_mask)) , -1)
 history_is_question = tf.expand_dims(tf.gather(tf.where(self.choice_is_question, 1., 0.) , tf.boolean_mask(choice_history, content_mask)) , -1)
 history_answer_mean = tf.expand_dims(tf.gather(self.answer_accuracy_mean, tf.boolean_mask(choice_history, content_mask)) , -1)
 history_answer2_mean = tf.expand_dims(tf.gather(self.answer_accuracy2_mean, tf.boolean_mask(choice_history, content_mask)) , -1)
 history_elapsed_time_mean = tf.expand_dims(tf.gather(self.answer_elapsed_time_mean, tf.boolean_mask(choice_history, content_mask)) , -1)
 history_correct_elapsed_time_mean = tf.expand_dims(tf.gather(self.answer_correct_elapssed_time_mean, tf.boolean_mask(choice_history, content_mask)) , -1)
 history_raw_elapsed_time = tf.expand_dims(tf.boolean_mask(raw_elapsed_time_history, content_mask), -1)
 history_ac_et_rate = history_correct_elapsed_time_mean / history_elapsed_time_mean
history_elapsed_time_rate = tf.math.log1p(tf.where(history_is_question == 1., history_raw_elapsed_time / history_elapsed_time_mean, 1))
history_correct_elapsed_time_rate = tf.math.log1p(tf.where(history_is_question == 1., history_raw_elapsed_time / history_correct_elapsed_time_mean, 1))
history_elapsed_time_mean = tf.math.log1p(history_elapsed_time_mean)
 history_correct_elapsed_time_mean = tf.math.log1p(history_correct_elapsed_time_mean)
 history_content_emb = self.content_embedding(tf.boolean_mask(choice_history//4, content_mask))
history_tag_emb = self.tag_embedding(self.choice_tags_vec(tf.boolean_mask(choice_history, content_mask)))
 history_tag_emb_max = tf.keras.backend.max(history_tag_emb, axis=1)
 history_tag_emb_min = tf.keras.backend.min(history_tag_emb, axis=1)
 history_tag_emb_cd = tf.cast(history_tag_emb_max > history_tag_emb_min, "float32")
 history_tag_emb = history_tag_emb_max * history_tag_emb_cd + history_tag_emb_min *(1 - history_tag_emb_cd)
 history_part_oh = tf.one_hot(tf.gather(self.choice_parts, tf.boolean_mask(choice_history, content_mask)) , 7)
 history_typeof_oh = tf.one_hot(tf.gather(self.choice_typeof, tf.boolean_mask(choice_history, content_mask)) , 4)
 history_choice_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_choice_emb ,axis=1, keepdims=True)))
 history_delta_timestamp_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_delta_timestamp_emb ,axis=1, keepdims=True)))
 history_elapsed_time_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_elapsed_time_emb ,axis=1, keepdims=True)))
 history_cooltime_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_cooltime_emb ,axis=1, keepdims=True)))
 history_position_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_position_emb ,axis=1, keepdims=True)))
 history_tag_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_tag_emb ,axis=1, keepdims=True)))
 history_norms = self.bn_clipper_history(tf.concat([history_choice_norm, history_delta_timestamp_norm, history_elapsed_time_norm,
history_cooltime_norm, history_position_norm, history_tag_norm], axis=1))
history_keyvalue_0 = tf.concat([history_content_emb,
history_choice_emb,
history_delta_timestamp_emb,
history_elapsed_time_emb,
history_cooltime_emb,
history_position_emb,
history_norms,
history_explanation,
history_choice,
history_correct_choice,
history_choice_rate,
history_is_correct,
history_is_wrong,
history_is_question,
history_answer_mean,
history_answer2_mean,
history_elapsed_time_mean,
history_correct_elapsed_time_mean,
history_ac_et_rate,
history_elapsed_time_rate,
history_correct_elapsed_time_rate,
history_tag_emb,
history_part_oh,
history_typeof_oh],
axis = -1)
 zero_embedding = self.zero_embedding(0)
 history_keyvalue = history_keyvalue_0 * self.dense_kv_v0(self.dense_kv_u0(self.bn_kv0(history_keyvalue_0)))+ history_keyvalue_0
history_keyvalue = history_keyvalue_0 * self.dense_kv_v1(self.dense_kv_u1(self.bn_kv1(history_keyvalue)))+ history_keyvalue
history_keyvalue = tf.scatter_nd(idx, history_keyvalue,(len(user_ix), self.sample_window, 312))
history_keyvalue += tf.expand_dims(tf.cast(~content_mask, "float32"), -1)* tf.expand_dims(zero_embedding, axis=0)
 query, context_features = tf.split(query_context_features, [64, -1], axis=1)
 query = tf.expand_dims(query, axis=1)
 history_key, history_choice, history_value = tf.split(history_keyvalue, [64, 64, -1], axis=2)
 query0 = tf.identity(query)
 history_key0 = tf.identity(history_key)
 queries = []
key_attentions = []
choice_attentions = []
value_attentions = []
add_features = []
queries.append(query)
 logit = tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(history_key, [0, 0, 0], [-1, -1, 64]), transpose_b=True)/ 8
weight = self.do_ts_k0(tf.math.softmax(logit))
key_attention = tf.matmul(weight, history_key)
 choice_attention = tf.matmul(weight, history_choice)
 sc_logit = tf.clip_by_value(( tf.math.sign(logit)* tf.math.log1p(tf.abs(logit))), -4, 4)
 logit_attention = tf.reduce_sum(weight * sc_logit, axis=2)
 cossim = tf.reshape(tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(key_attention, [0, 0, 0], [-1, -1, 64]), transpose_b=True)/(tf.norm(query, axis=2, keepdims=True)*tf.norm(key_attention, axis=2, keepdims=True)+1e-9),(-1, 1))
ac_div = tf.matmul(weight, question_history)
 ac_num = tf.matmul(weight, correct_history)
 eac_num = tf.matmul(weight, accuracy_history)
 eac_rate = tf.reshape(tf.where(ac_div > 0, eac_num/(ac_div+1e-11), 0.657),(-1, 1))
eac_rate = tf.math.maximum(0.01, eac_rate)
 ac_rate = tf.reshape(tf.where(ac_div > 0, ac_num/(ac_div+1e-11), 0.657),(-1, 1))
ac_rate =(ac_rate * tf.cast(tf.expand_dims(order, axis=1), "float32")+ eac_rate * 10.) /(tf.cast(tf.expand_dims(order, axis=1), "float32")+ 10.)
ac_eac_rate = ac_rate / eac_rate
h_ac_rate = 2 /(1/tf.maximum(0.01, ac_rate)+ 1/tf.maximum(0.01, content_accuracy))
cossim= tf.clip_by_value(cossim, -1, 1)
 logit_attention = tf.clip_by_value(logit_attention, -5, 5)
 ac_rate = tf.clip_by_value(ac_rate, 0, 1)
 eac_rate = tf.clip_by_value(eac_rate, 0, 1)
 ac_eac_rate = tf.clip_by_value(tf.math.log1p(tf.abs(ac_eac_rate)) , -1, 5)
 calc_feature = tf.stop_gradient(tf.concat([cossim, logit_attention, ac_rate, eac_rate, ac_eac_rate, h_ac_rate], axis=1))
norm_feature = tf.stop_gradient(tf.math.log1p(tf.concat([tf.norm(query, axis=2), tf.norm(key_attention, axis=2), tf.norm(choice_attention, axis=2)], axis=1)))
 add_feature = self.bn_clipper_0(tf.concat([tf.reshape(calc_feature,(-1, 1, 6)) , tf.reshape(norm_feature,(-1, 1, 3)) ], axis=2))
key_attentions.append(key_attention)
 choice_attentions.append(choice_attention)
 add_features.append(tf.reshape(add_feature,(-1, 9)))
 query = query0 * self.dense_ts_k0(self.bn_ts_k0(tf.concat([key_attention, choice_attention, add_feature], axis=2)))+ query
history_keys = tf.split(history_key, [100]*5, axis=1)
 history_keys0 = tf.split(history_key0, [100]*5, axis=1)
 history_timestamps = tf.split(timestamp_history, [100]*5, axis=1)
 zero_emb_key, zero_emb_choice, _ = tf.split(model.zero_embedding(0), [64, 64, -1], axis=0)
 zero_emb_key = tf.reshape(tf.tile(zero_emb_key, [len(user_ix)]),(-1, 1, 64))
zero_emb_choice = tf.reshape(tf.tile(zero_emb_choice, [len(user_ix)]),(-1, 1, 64))
history_new_keys = []
attentions = []
history_attentions = []
for i in range(5):
key_slice = tf.slice(history_key, [0, 0, 0], [-1, 100*(i+1), -1])
 key_slice = tf.concat([zero_emb_key, key_slice], axis=1)
 choice_slice = tf.slice(history_choice, [0, 0, 0], [-1, 100*(i+1), -1])
 choice_slice = tf.concat([zero_emb_choice, choice_slice], axis=1)
 timestamp_slice = tf.slice(timestamp_history, [0, 0], [-1, 100*(i+1)])
 timestamp_slice = tf.concat([tf.ones(( len(user_ix), 1), dtype="int64")*-1, timestamp_slice], axis=1)
 query_timestamp = tf.maximum(tf.constant(0, dtype="int64"), tf.reshape(tf.repeat(history_timestamps[i], 100*(i+1)+1, axis=1),(-1, 100, 100*(i+1)+1)))
 key_timestamp = tf.reshape(tf.tile(timestamp_slice, [1, 100]),(-1, 100, 100*(i+1)+1))
encoder_mask = tf.where(key_timestamp < query_timestamp, 0., tf.float32.min/100)
 logit = tf.matmul(tf.slice(history_keys[i], [0, 0, 0], [-1, -1, 64]), tf.slice(key_slice, [0, 0, 0], [-1, -1, 64]), transpose_b=True)+ encoder_mask
weight = self.do_sa_k0(tf.math.softmax(logit/8))
key_attention = tf.matmul(weight, key_slice)
 choice_attention = tf.matmul(weight, choice_slice)
 history_exist = tf.where(history_timestamps[0] >= 0, 1., 0.)
sc_logit = tf.clip_by_value(tf.math.sign(logit)* tf.math.log1p(tf.abs(logit)) , -4, 4)
 history_logit = tf.reduce_sum(sc_logit * weight, axis=2)* history_exist
history_cossim = tf.reduce_sum(key_attention * history_keys[i], axis=2)/(tf.norm(key_attention, axis=2)* tf.norm(history_keys[i], axis=2)+ 1e-9)
 history_cossim *= history_exist
history_cossim = tf.clip_by_value(history_cossim, -1, 1)
 history_logit = tf.stop_gradient(tf.expand_dims(history_logit, axis=2))
history_exist = tf.stop_gradient(tf.expand_dims(history_exist, axis=2))
history_cossim = tf.stop_gradient(tf.expand_dims(history_cossim, axis=2))
history_add = self.bn_clipper_1(tf.concat([history_logit, history_logit, history_cossim], axis=2))
attention = tf.concat([key_attention, choice_attention, history_add], axis=2)
 history_new_keys.append(history_keys0[i] * self.dense_sa_k0(self.bn_sa_k0(attention)) + history_keys[i])
 history_attentions.append(attention)
 history_key = tf.concat(history_new_keys, axis=1)
 history_value = tf.concat([history_value, tf.concat(history_attentions, axis=1)], axis=2)
 queries.append(query)
 logit = tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(history_key, [0, 0, 0], [-1, -1, 64]), transpose_b=True)/ 8
weight = self.do_ts_k1(tf.math.softmax(logit))
key_attention = tf.matmul(weight, history_key)
 choice_attention = tf.matmul(weight, history_choice)
 value_attention = tf.matmul(weight, history_value)
 sc_logit = tf.clip_by_value(( tf.math.sign(logit)* tf.math.log1p(tf.abs(logit))), -4, 4)
 logit_attention = tf.reduce_sum(weight * sc_logit, axis=2)
 cossim = tf.reshape(tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(key_attention, [0, 0, 0], [-1, -1, 64]), transpose_b=True)/(tf.norm(query, axis=2, keepdims=True)*tf.norm(key_attention, axis=2, keepdims=True)+1e-9),(-1, 1))
ac_div = tf.matmul(weight, question_history)
 ac_num = tf.matmul(weight, correct_history)
 eac_num = tf.matmul(weight, accuracy_history)
 eac_rate = tf.reshape(tf.where(ac_div > 0, eac_num/(ac_div+1e-11), 0.657),(-1, 1))
eac_rate = tf.math.maximum(0.01, eac_rate)
 ac_rate = tf.reshape(tf.where(ac_div > 0, ac_num/(ac_div+1e-11), 0.657),(-1, 1))
ac_rate =(ac_rate * tf.cast(tf.expand_dims(order, axis=1), "float32")+ eac_rate * 10.) /(tf.cast(tf.expand_dims(order, axis=1), "float32")+ 10.)
h_ac_rate = 2 /(1/tf.maximum(0.01, ac_rate)+ 1/tf.maximum(0.01, content_accuracy))
ac_eac_rate = ac_rate / eac_rate
cossim= tf.clip_by_value(cossim, -1, 1)
 logit_attention = tf.clip_by_value(logit_attention, -5, 5)
 ac_rate = tf.clip_by_value(ac_rate, 0, 1)
 eac_rate = tf.clip_by_value(eac_rate, 0, 1)
 ac_eac_rate = tf.clip_by_value(tf.math.log1p(tf.abs(ac_eac_rate)) , -1, 5)
 calc_feature = tf.stop_gradient(tf.concat([cossim, logit_attention, ac_rate, eac_rate, ac_eac_rate, h_ac_rate], axis=1))
norm_feature = tf.stop_gradient(tf.math.log1p(tf.concat([tf.norm(query, axis=2), tf.norm(key_attention, axis=2), tf.norm(choice_attention, axis=2)], axis=1)))
 add_feature = self.bn_clipper_2(tf.concat([tf.reshape(calc_feature,(-1, 1, 6)) , tf.reshape(norm_feature,(-1, 1, 3)) ], axis=2))
key_attentions.append(key_attention)
 choice_attentions.append(choice_attention)
 value_attentions.append(value_attention)
 add_features.append(tf.reshape(add_feature,(-1, 9)))
 queries_vec = tf.reshape(tf.concat(queries, axis=2),(-1, 128))
key_attentions = tf.reshape(tf.concat(key_attentions, axis=2),(-1, 128))
choice_attentions = tf.reshape(tf.concat(choice_attentions, axis=2),(-1, 128))
value_attentions = tf.reduce_mean(tf.concat(value_attentions, axis=1), axis=1)
 feature_attentions = tf.concat(add_features, axis=1)
 filter_res_key_max = self.gloval_max_pool_key(self.history_key_conv0(history_key))
filter_res_key_avg = self.gloval_avg_pool_key(self.gloval_avg_pool_key_drop(self.history_key_conv1(history_key)))
 filter_res_correct_key_max = self.gloval_max_pool_correct_key(self.history_correct_key_conv(history_key)* correct_history)
 filter_res_choice_max = self.gloval_max_pool_choice(self.history_choice_conv0(history_choice))
filter_res_choice_avg = self.gloval_avg_pool_choice(self.gloval_avg_pool_choice_drop(self.history_choice_conv1(history_choice)))
 filter_res_wrong_choice_max = self.gloval_max_pool_wrong_choice(self.history_wrong_choice_conv(history_choice)* wrong_history)
 filter_res_val_max = self.gloval_max_pool_val(self.history_val_conv0(history_value))
filter_res_val_avg = self.gloval_avg_pool_val(self.gloval_avg_pool_val_drop(self.history_val_conv1(history_value)))
 filter_res = tf.concat([filter_res_key_max, filter_res_key_avg, filter_res_correct_key_max,
filter_res_choice_max, filter_res_choice_avg,filter_res_wrong_choice_max,
filter_res_val_max, filter_res_val_avg], axis=1)
 table_features = tf.concat([table_features, feature_attentions], axis=1)
 X0 = tf.keras.layers.Concatenate()([context_features,
queries_vec, key_attentions, choice_attentions, value_attentions,
feature_attentions, filter_res])
 prod_out0 = self.dense_v0(self.dense_u0(self.bnc0(X0)))
 X = X0 * prod_out0 + X0
prod_out1 = self.dense_v1(self.dense_u1(self.bnc1(X)))
 X = X0 * prod_out1 + X
prod_out2 = self.dense_v2(self.dense_u2(self.bnc2(X)))
 X_feat = tf.concat([X0 * prod_out2 + X, table_features], axis=1)
 X = self.bn0(X_feat)
 X = self.do0(X)
 X = self.dense1(X)
 X = self.bn1(X)
 X = self.do1(X)
 X = self.dense2(X)
 X = self.bn2(X)
 X = self.do2(X)
 X = self.dense3(X)
 X = self.bn3(X)
 X = self.do3(X)
 return self.out(X )
RFC = RandomForestClassifier()
rf_param_grid = {"max_depth": [None],
"max_features": [1, 3, 10],
"min_samples_split": [2, 3, 10],
"min_samples_leaf": [1, 3, 10],
"bootstrap": [False],
"n_estimators" :[100,300],
"criterion": ["gini"]}
gsRFC = GridSearchCV(RFC,param_grid = rf_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsRFC.fit(X_train,Y_train)
 RFC_best = gsRFC.best_estimator_
gsRFC.best_score_<train_on_grid>
feat_in_col =(
 ['content_2_accuracy', 'content_3_accuracy', 'content_4_accuracy', "idf"]
+ ["correct_streak", "wrong_streak"]
+ [f"user_choice_{i}_rate_rate" for i in range(4)]
+ [f"user_wrong_choice_{i}_rate_rate" for i in range(4)]
+ [f"user_choice_rate_rate_correct"]
+ [f"user_wrong_choice_rate_rate_correct"]
+ ["harmonic_mean", "part_harmonic_mean", "frequency", "content_seen_sum", "acc_mean", "acc_per_elapse", "elp_mean", "scaled_elapsed_time_mean", "exp_mean"]
+ ['part0_sum', 'part0_count', 'part1_sum', 'part1_count', 'part2_sum', 'part2_count', 'part3_sum', 'part3_count', 'part4_sum', 'part4_count', 'part5_sum', 'part5_count', 'part6_sum', 'part6_count', 'question_sum', 'question_count']
+ [f"part{i}_accuracy_rate" for _ in range(7)] + ["total_accuracy_rate"]
+ [f"part{i}_count_rate" for _ in range(7)]
+ ["questions_one_day", "questions_one_hour", "questions_one_week"]
+ ["day_angle_cos_mean", "day_angle_sin_mean", "day_angle_norm", "day_angle_normed_cos", "day_angle_normed_sin", "week_angle_cos_mean", "week_angle_sin_mean", "week_angle_norm", "week_angle_normed_cos", "week_angle_normed_sin"]
+ ["last_correct_ts", "last_wrong_ts", "last_correct_ts_rate", "last_wrong_ts_rate"]
+ ["delta_last_2", "delta_last_1", "delta_last_0"] 
)
 context_features =(
 [f"context_choice_emb_{i}" for i in range(64)]
+ [f"context_timestamp_emb_{i}" for i in range(32)]
+ [f"context_lag_emb_{i}" for i in range(32)]
+ [f"content_tags_emb_{i}" for i in range(16)]
+ ["context_order"]
+ ["context_" + feat for feat in feat_in_col]
+ [f"context_part_oh_{i}" for i in range(7)]
+ ["context_content_norm", "context_timestamp_norm", "context_choice_norm", "context_lag_norm", "context_tag_norm"]
+ [f"context_correct_choice_oh_{i}" for i in range(4)]
+ [f"context_content_choice_rate_{i}" for i in range(4)] 
)
 history_value_features =(
 [f"history_delta_timestamp_emb_{i}" for i in range(32)]
+ [f"history_elapsed_time_emb_{i}" for i in range(32)]
+ [f"history_cooltime_emb_{i}" for i in range(32)]
+ [f"history_position_emb_{i}" for i in range(32)]
+ ["history_choice_norm", "history_delta_timestamp_norm", "history_elapsed_time_norm", "history_cooltime_norm", "history_position_norm", "history_tag_norm"]
+ ["history_explanation"]
+ [f"history_choice{i}" for i in range(4)]
+ [f"history_correct_choice{i}" for i in range(4)]
+ [f"history_choice_rate{i}" for i in range(4)]
+ ["history_is_correct", "history_is_wrong", "history_is_question"]
+ ["history_answer_mean", "history_answer2_mean"]
+ ["history_elapsed_time_mean", "history_correct_elapsed_time_mean", "history_ac_et_rate"]
+ ["history_elapsed_time_rate", "history_correct_elapsed_time_rate"]
+ [f"history_tag_{i}" for i in range(16)]
+ [f"history_part_oh_{i}" for i in range(7)]
+ [f"history_typeof_oh_{i}" for i in range(4)]
+ [f"history_encoder_key_attention_{i}" for i in range(64)]
+ [f"history_encoder_choice_attention_{i}" for i in range(64)]
+ [f"history_encoder_logit", "history_encoder_exist", "history_encoder_cossim"] 
)
 filter_cols =(
 [f"filter_res_key_max_{i}" for i in range(32)]
+ [f"filter_res_key_avg_{i}" for i in range(32)]
+ [f"filter_res_correct_key_max_{i}" for i in range(32)]
+ [f"filter_res_choice_max_{i}" for i in range(32)]
+ [f"filter_res_choice_avg_{i}" for i in range(32)]
+ [f"filter_res_wrong_choice_max_{i}" for i in range(32)]
+ [f"filter_res_value_max_{i}" for i in range(32)]
+ [f"filter_res_value_avg_{i}" for i in range(32)] 
)
 X0_cols =(
 context_features
+ [f"query_emb_level0_{i}" for i in range(64)] + [f"query_emb_level1_{i}" for i in range(64)]
+ [f"key_attention_emb_level0_{i}" for i in range(64)] + [f"key_attention_emb_level1_{i}" for i in range(64)]
+ [f"choice_attention_emb_level0_{i}" for i in range(64)] + [f"choice_attention_emb_level1_{i}" for i in range(64)]
+ history_value_features
+ [f"history_level0_{name}" for name in ["cossim", "logit_attention", "ac_rate", "eac_rate", "ac_eac_rate"," h_ac_rate"]]
+ [f"history_level0_{name}_norm" for name in ["query_attention", "key_attention", "choice_attention"]]
+ [f"history_level1_{name}" for name in ["cossim", "logit_attention", "ac_rate", "eac_rate", "ac_eac_rate"," h_ac_rate"]]
+ [f"history_level1_{name}_norm" for name in ["query_attention", "key_attention", "choice_attention"]]
+ filter_cols 
)
 table_features =(
 ["raw_order"]
+ ["raw_" + feat for feat in feat_in_col]
+ [f"raw_part_oh_{i}" for i in range(7)]
+ ["raw_content_norm", "raw_timestamp_norm", "raw_choice_norm", "raw_lag_norm", "raw_tag_norm"]
+ [f"raw_correct_choice_oh_{i}" for i in range(4)]
+ [f"raw_content_choice_rate_{i}" for i in range(4)]
+ [f"raw_history_level0_{name}" for name in ["cossim", "logit_attention", "ac_rate", "eac_rate", "ac_eac_rate"," h_ac_rate"]]
+ [f"raw_history_level0_{name}_norm" for name in ["query_attention", "key_attention", "choice_attention"]]
+ [f"raw_history_level1_{name}" for name in ["cossim", "logit_attention", "ac_rate", "eac_rate", "ac_eac_rate"," h_ac_rate"]]
+ [f"raw_history_level1_{name}_norm" for name in ["query_attention", "key_attention", "choice_attention"]]
)
GBC = GradientBoostingClassifier()
gb_param_grid = {'loss' : ["deviance"],
'n_estimators' : [100,200,300],
'learning_rate': [0.1, 0.05, 0.01],
'max_depth': [4, 8],
'min_samples_leaf': [100,150],
'max_features': [0.3, 0.1]
}
gsGBC = GridSearchCV(GBC,param_grid = gb_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsGBC.fit(X_train,Y_train)
 GBC_best = gsGBC.best_estimator_
gsGBC.best_score_<train_on_grid>
model = BaseModel(max(user_id_map.values()), lectures_df["content_id"].max() , lectures_df["content_id"].max() *4 + 3,
choice_is_correct, choice_is_wrong, choice_is_question,
choice_tags, choice_parts, choice_typeof,
answer_accuracy_mean, answer_accuracy2_mean, answer_elapsed_time_mean, answer_elapsed_time_std, answer_correct_elapssed_time_mean,
correct_choice, choice_rate)
 model.load_weights(".. /input/nadare-processed-2/model8" )
SVMC = SVC(probability=True)
 svc_param_grid = {'kernel': ['rbf'],
'gamma': [ 0.001, 0.01, 0.1, 1],
'C': [1, 10, 50, 100,200,300, 1000]}
gsSVMC = GridSearchCV(SVMC,param_grid = svc_param_grid, cv=kfold, scoring="accuracy", n_jobs= 4, verbose = 1)
 gsSVMC.fit(X_train,Y_train)
 SVMC_best = gsSVMC.best_estimator_
gsSVMC.best_score_<train_model>
class Predictor(tf.Module):
def __init__(self,
timestamp_bins,
delta_bins,
elapsedtime_bins,
choice_is_correct,
choice_is_wrong,
choice_is_question,
answer_accuracy_mean,
answer_elapsed_time_mean,
answer_elapsed_time_std,
correct_choice,
choice_rate,
part_mean,
part_rate):
self.sample_window = 500
self.timestamp_bins = tf.constant(timestamp_bins, dtype="float32")
 self.delta_bins = tf.constant(delta_bins, dtype="float32")
 self.elapsedtime_bins = tf.constant(elapsedtime_bins, dtype="float32")
 self.choice_is_correct = choice_is_correct
self.choice_is_wrong = choice_is_wrong
self.choice_is_question = choice_is_question
self.answer_accuracy_mean = answer_accuracy_mean
self.answer_elapsed_time_mean = answer_elapsed_time_mean
self.answer_elapsed_time_std = answer_elapsed_time_std
self.correct_choice = correct_choice
self.choice_rate = choice_rate
self.part_mean = part_mean
self.part_rate = part_rate
@tf.function(input_signature=[
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int64),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int64),
tf.TensorSpec(shape=(None, 500), dtype=tf.float32),
tf.TensorSpec(shape=(None, 500), dtype=tf.float32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=(None, 4), dtype=tf.float32),
tf.TensorSpec(shape=(None, 16), dtype=tf.float32)
 ] 
)
 def __call__(self, user_ix, content_id, timestamp, order,
choice_history, raw_timestamp_history, raw_elapsed_time_history, explanation_history,
part, feat, raw_feat):
N = len(user_ix)
 timestamp_cat = tf.reshape(tf.cast(tf.searchsorted(self.timestamp_bins, tf.cast(timestamp, "float32")) , dtype="int16"),(-1,))
timestamp_history = tf.identity(raw_timestamp_history)
 delta_timestamp_history = tf.expand_dims(timestamp, axis=-1)- raw_timestamp_history
delta_timestamp_history = tf.cast(tf.searchsorted(self.delta_bins, tf.cast(tf.keras.backend.flatten(delta_timestamp_history), "float32"), side="right"), dtype="int16")
 delta_timestamp_history = tf.reshape(delta_timestamp_history, [-1, self.sample_window])
 elapsed_time_history = tf.cast(tf.searchsorted(self.elapsedtime_bins, tf.cast(tf.keras.backend.flatten(raw_elapsed_time_history), "float32"), side="right"), dtype="int16")
 elapsed_time_history = tf.reshape(elapsed_time_history, [-1, self.sample_window])
 raw_elapsed_time_history = tf.cast(tf.reshape(raw_elapsed_time_history, [-1, self.sample_window]), "float32")
 timestamp_histories = tf.split(tf.cast(timestamp_history, "float32"), 500, axis=1)
 elapsed_time_histories = tf.split(raw_elapsed_time_history, 500, axis=1)
 lag_time = tf.where(tf.reshape(timestamp_histories[-1],(-1,)) < 0, 0, tf.searchsorted(self.delta_bins, tf.cast(timestamp, "float32")- tf.reshape(timestamp_histories[-1],(-1,))))
ts = tf.reshape(tf.cast(timestamp, "float32"),(-1, 1))
ept = tf.ones(( N, 1), dtype="float32")* tf.float32.max/100
nept = tf.zeros(( N, 1), dtype="float32")
 cooltimes = []
for i in range(499, -1, -1):
tsn = timestamp_histories[i]
if i < 499:
ept = tf.where(timestamp_histories[i]==timestamp_histories[i+1], ept, nept)
 cooltimes.append(ts - tsn - ept)
 ts = tf.where(elapsed_time_histories[i] <= 0, ts, tsn)
 if i < 499:
nept = tf.where(timestamp_histories[i]==timestamp_histories[i+1], elapsed_time_histories[i] + nept, elapsed_time_histories[i])
 nept = tf.where(elapsed_time_histories[i] <= 0, ept, nept)
 else:
nept = tf.where(elapsed_time_histories[i] <= 0, ept, elapsed_time_histories[i])
 cooltimes = tf.concat(cooltimes[::-1], axis=1)
 cooltimes = tf.searchsorted(self.delta_bins, tf.reshape(cooltimes,(-1,)))
 cooltime_history = tf.reshape(cooltimes, [N, self.sample_window])
 cooltime_history = tf.where(choice_history > 0, tf.minimum(9999, cooltime_history+1), 0)
 correct_answer_sum = tf.reduce_sum(tf.where(tf.gather(self.choice_is_correct, choice_history), 1., 0.) , axis=1, keepdims=True)
 history_is_question = tf.gather(self.choice_is_question, choice_history)
 history_is_correct = tf.gather(self.choice_is_correct, choice_history)
 history_is_wrong = tf.gather(self.choice_is_wrong, choice_history)
 explanation_sum = tf.reduce_sum(tf.where(history_is_question, explanation_history, 0), axis=1, keepdims=True)
 question_sum = tf.reduce_sum(tf.where(history_is_question, 1., 0.) , axis=1, keepdims=True)
 wrong_sum = tf.reduce_sum(tf.where(history_is_wrong, 1., 0.) , axis=1, keepdims=True)
 elapsed_acc_sum = tf.reduce_sum(tf.where(history_is_question, tf.gather(self.answer_accuracy_mean, choice_history), 0), axis=1, keepdims=True)
 elapsed_time_content_mean = tf.gather(self.answer_elapsed_time_mean, choice_history)
 elapsed_time_content_std = tf.gather(self.answer_elapsed_time_std, choice_history)
 scaled_elapsed_time = tf.where(history_is_question, tf.math.log1p(raw_elapsed_time_history), 0.)
scaled_elapsed_time_sum = tf.reduce_sum(scaled_elapsed_time, axis=1, keepdims=True)
 correct_streak = tf.zeros(( N, 1))
correct_continue = tf.ones(( N, 1))
wrong_streak = tf.zeros(( N, 1))
wrong_continue = tf.ones(( N, 1))
for i in range(500):
isc = tf.where(tf.slice(history_is_correct, [0, 500-i-1], [-1, 1]), 1., 0.)
isw = tf.where(tf.slice(history_is_wrong, [0, 500-i-1], [-1, 1]), 1., 0.)
isq = tf.where(tf.slice(history_is_question, [0, 500-i-1], [-1, 1]), 0., 1.)
cc = tf.maximum(isq, isc)
 ww = tf.maximum(isq, isw)
 correct_continue *= cc
wrong_continue *= ww
correct_streak += correct_continue * cc
wrong_streak += wrong_continue * ww
correct_streak = tf.math.log1p(correct_streak)
 wrong_streak = tf.math.log1p(wrong_streak)
 user_real_choice_rate = tf.reduce_sum(tf.one_hot(tf.where(history_is_question, choice_history%4, -1), 4), axis=1)
 user_elapsed_choice_rate = tf.reduce_sum(tf.gather(choice_rate, choice_history), axis=1)
 user_choice_rate_rate = user_real_choice_rate /(user_elapsed_choice_rate + 1e-9)
 user_choice_rate_rate =(user_choice_rate_rate * question_sum + 10)/(question_sum + 10)
 wrong_user_real_choice_rate = tf.reduce_sum(tf.one_hot(tf.where(history_is_wrong, choice_history%4, -1), 4), axis=1)
 wrong_user_elapsed_choice_rate = tf.reduce_sum(tf.gather(choice_rate, choice_history)* tf.expand_dims(tf.where(history_is_wrong, 1., 0.) , axis=-1), axis=1)
 wrong_user_choice_rate_rate = wrong_user_real_choice_rate /(wrong_user_elapsed_choice_rate + 1e-9)
 wrong_user_choice_rate_rate =(wrong_user_choice_rate_rate * wrong_sum + 10)/(wrong_sum + 10)
 user_choice_rate_rate_correct = tf.reduce_sum(user_choice_rate_rate * tf.one_hot(tf.gather(self.correct_choice, content_id*4), 4), axis=1, keepdims=True)
 wrong_user_choice_rate_rate_correct = tf.reduce_sum(wrong_user_choice_rate_rate * tf.one_hot(tf.gather(self.correct_choice, content_id*4), 4), axis=1, keepdims=True)
 rate_feat = tf.concat([user_choice_rate_rate, wrong_user_choice_rate_rate, user_choice_rate_rate_correct, wrong_user_choice_rate_rate_correct], axis=1)
 frequency = tf.math.log1p(tf.cast(timestamp, "float32")/(tf.cast(order, "float32")+1e-9))
frequency = tf.expand_dims(tf.where(order > 0, frequency, 15), axis=-1)
 content_seen_sum = tf.math.log1p(tf.reduce_sum(tf.where(tf.logical_and(tf.equal(choice_history//4, tf.expand_dims(content_id, axis=1)) , choice_history > 0), 1., 0.) , axis=1, keepdims=True))
elp_mean = tf.where(question_sum > 0,(elapsed_acc_sum)/(question_sum + 1e-9), 0.657)
 acc_mean = tf.where(question_sum > 0,(correct_answer_sum)/(question_sum + 1e-9), 0.657)
 acc_mean =(acc_mean * question_sum + elp_mean * 10)/(question_sum + 10)
 exp_mean = tf.where(question_sum > 0,(explanation_sum + 5e-10)/(question_sum + 1e-9),.5)
 acc_per_elapse = tf.math.log1p(( acc_mean + 1e-9)/ tf.math.maximum(elp_mean, 0.01))
scaled_elapsed_time_mean = tf.where(question_sum>0,(scaled_elapsed_time_sum)/(question_sum + 1e-9), 0)
 choice_sum = tf.reduce_sum(tf.where(choice_history > 0, 1., 0.) , axis=1)
 day_msecs =(1000 * 60 * 60 * 24)
 day_angle =(tf.cast(timestamp_history, "float32")% day_msecs)/ day_msecs * 2 * np.pi
day_angle_cos_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.cos(day_angle), 0), axis=1)/ tf.math.maximum(choice_sum, 1e-9)
 day_angle_sin_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.sin(day_angle), 0), axis=1)/ tf.math.maximum(choice_sum, 1e-9)
 day_angle_norm = tf.math.sqrt(tf.math.square(day_angle_cos_mean)+ tf.math.square(day_angle_sin_mean))
day_angle_normed_cos = day_angle_cos_mean / tf.math.maximum(day_angle_norm, 1e-9)
 day_angle_normed_sin = day_angle_sin_mean / tf.math.maximum(day_angle_norm, 1e-9)
 week_msecs =(1000 * 60 * 60 * 24 * 7)
 week_angle =(tf.cast(timestamp_history, "float32")% week_msecs)/ week_msecs * 2 * np.pi
week_angle_cos_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.cos(week_angle), 0), axis=1)/ tf.math.maximum(choice_sum, 1e-9)
 week_angle_sin_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.sin(week_angle), 0), axis=1)/ tf.math.maximum(choice_sum, 1e-9)
 week_angle_norm = tf.math.sqrt(tf.math.square(week_angle_cos_mean)+ tf.math.square(week_angle_sin_mean))
week_angle_normed_cos = week_angle_cos_mean / tf.math.maximum(week_angle_norm, 1e-9)
 week_angle_normed_sin = week_angle_sin_mean / tf.math.maximum(week_angle_norm, 1e-9)
 last3_timestamp = tf.where(tf.slice(choice_history, [0, 497], [-1, 3])> 0, tf.slice(timestamp_history, [0, 497], [-1, 3]), [[132324, 75973, 32387]])
 delta_last3 = tf.math.log1p(tf.cast(tf.reshape(timestamp,(-1, 1)) - last3_timestamp, "float32"))
last_correct_ts = tf.math.log1p(tf.cast(timestamp - tf.reduce_max(tf.where(tf.gather(self.choice_is_correct, choice_history), timestamp_history, 0), axis=1), "float32"))
last_wrong_ts = tf.math.log1p(tf.cast(timestamp - tf.reduce_max(tf.where(tf.gather(self.choice_is_wrong, choice_history), timestamp_history, 0), axis=1), "float32"))
last_correct_ts_rate = tf.where(timestamp > 0, last_correct_ts / tf.math.log1p(tf.cast(timestamp, "float32")) , 1)
 last_wrong_ts_rate = tf.where(timestamp > 0, last_wrong_ts / tf.math.log1p(tf.cast(timestamp, "float32")) , 1)
 new_feat = [day_angle_cos_mean, day_angle_sin_mean, day_angle_norm, day_angle_normed_cos, day_angle_normed_sin,
week_angle_cos_mean, week_angle_sin_mean, week_angle_norm, week_angle_normed_cos, week_angle_normed_sin,
last_correct_ts, last_wrong_ts, last_correct_ts_rate, last_wrong_ts_rate]
new_feat = tf.concat([tf.expand_dims(f, axis=-1)for f in new_feat] + [delta_last3], axis=1)
 questions_one_hour = tf.math.log1p(tf.cast(500 - tf.searchsorted(timestamp_history, tf.maximum(tf.expand_dims(timestamp-3600*1000, axis=1), 0)) , "float32"))
questions_one_day = tf.math.log1p(tf.cast(500 - tf.searchsorted(timestamp_history, tf.maximum(tf.expand_dims(timestamp-24*3600*1000, axis=1), 0)) , "float32"))
questions_one_week = tf.math.log1p(tf.cast(500 - tf.searchsorted(timestamp_history, tf.maximum(tf.expand_dims(timestamp-7*24*3600*1000, axis=1), 0)) , "float32"))
num_col = tf.constant([0, 2, 4, 6, 8, 10, 12, 14])
 div_col = tf.constant([1, 3, 5, 7, 9, 11, 13, 15])
 row_ixs = tf.repeat(tf.range(N), len(num_col))
num_col = tf.tile(num_col, [N])
 div_col = tf.tile(div_col, [N])
 num_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, num_col], axis=1)) ,(N, -1))
div_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, div_col], axis=1)) ,(N, -1))
mr_mean_feat = num_feat /(div_feat + 1e-9)
 mr_mean_feat =(mr_mean_feat * div_feat + self.part_mean * 10)/(div_feat + 10)
 num_col = tf.constant([1, 3, 5, 7, 9, 11, 13])
 div_col = tf.constant([15, 15, 15, 15, 15, 15, 15])
 row_ixs = tf.repeat(tf.range(N), len(num_col))
num_col = tf.tile(num_col, [N])
 div_col = tf.tile(div_col, [N])
 num_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, num_col], axis=1)) ,(N, -1))
div_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, div_col], axis=1)) ,(N, -1))
mr_rate_feat = num_feat /(div_feat + 1e-9)
 mr_rate_feat =(mr_rate_feat * tf.expand_dims(tf.cast(order, "float32"), axis=1)+ self.part_rate * 10)/(tf.expand_dims(tf.cast(order, "float32"), axis=1)+ 10)
 mr_feat = tf.concat([mr_mean_feat, mr_rate_feat], axis=1)
 count_feat = tf.math.log1p(raw_feat)
 content_accuracy = tf.reshape(tf.gather(self.answer_accuracy_mean, content_id*4),(-1, 1))
harmonic_mean = 2/(1/tf.math.maximum(1e-9, tf.reshape(tf.gather_nd(mr_mean_feat, tf.stack([tf.range(N), tf.ones(N, dtype="int32")*7], axis=1)) ,(-1, 1)))+ 1/tf.math.maximum(1e-9, content_accuracy))
part_harmonic_mean = 2/(1/tf.math.maximum(1e-9, tf.reshape(tf.gather_nd(mr_mean_feat, tf.stack([tf.range(N), part], axis=1)) ,(-1, 1)))+ 1/tf.math.maximum(1e-9, content_accuracy))
new_feats = tf.concat([correct_streak, wrong_streak, rate_feat, harmonic_mean, part_harmonic_mean, frequency, content_seen_sum, acc_mean, acc_per_elapse, elp_mean, scaled_elapsed_time_mean, exp_mean, count_feat, mr_feat, questions_one_day, questions_one_hour, questions_one_week, new_feat], axis=1)
 feat = tf.concat([feat, new_feats], axis=1)
 return(user_ix, content_id, timestamp_cat, lag_time, order, choice_history, timestamp_history, delta_timestamp_history,
raw_elapsed_time_history, elapsed_time_history, explanation_history, cooltime_history, part, feat )
votingC = VotingClassifier(estimators=[('rfc', RFC_best),('extc', ExtC_best), 
('svc', SVMC_best),('adac',ada_best),('gbc',GBC_best)], voting='soft', n_jobs=4)
 votingC = votingC.fit(X_train, Y_train )<save_to_csv>
predictor = Predictor(timestamp_bins,
delta_bins,
elapsedtime_bins,
choice_is_correct,
choice_is_wrong,
choice_is_question,
answer_accuracy_mean,
answer_elapsed_time_mean,
answer_elapsed_time_std,
correct_choice,
choice_rate,
part_mean,
part_rate )
test_Survived = pd.Series(votingC.predict(test), name="Survived")
 results = pd.concat([IDtest,test_Survived],axis=1)
 results.to_csv("Final Submission File.csv",index=False )<import_modules>
questions_df = questions_df.set_index("content_id" )
<import_modules><EOS> 
warnings.simplefilter('ignore')
 feat_col = ['content_2_accuracy', 'content_3_accuracy', 'content_4_accuracy', "idf"]
raw_feat_col = ['part0_sum', 'part0_count', 'part1_sum',
'part1_count', 'part2_sum', 'part2_count', 'part3_sum', 'part3_count',
'part4_sum', 'part4_count', 'part5_sum', 'part5_count', 'part6_sum',
'part6_count', 'question_sum', 'question_count']
user_ix_last = max(user_id_map.values())+ 1
user_ix_max = tf.constant(max(user_id_map.values()), dtype="int32")
 past_df = None
i = 0
collect()
mapping_question = np.vectorize(lambda x: question_id_map[x], otypes=["int64"])
 mapping_lecture = np.vectorize(lambda x: lecture_id_map[x], otypes=["int64"])
 for(test_raw_df, sample_prediction_df)in tqdm(iter_test):
test_raw_df.loc[test_raw_df["content_type_id"] == False, "content_id"] = mapping_question(test_raw_df.loc[test_raw_df["content_type_id"] == False, "content_id"].values)
 test_raw_df.loc[test_raw_df["content_type_id"] == True, "content_id"] = mapping_lecture(test_raw_df.loc[test_raw_df["content_type_id"] == True, "content_id"].values)
 if past_df is not None:
past_df["user_answer"] = eval(test_raw_df["prior_group_responses"].values[0])
 past_df["choice_id"] = past_df["content_id"]*4 + past_df["user_answer"] *(past_df["user_answer"] >= 0)
 for user_ix, choice_id, timestamp, cid, part in zip(past_df["user_ix"], past_df["choice_id"], past_df["timestamp"], past_df["content_id"], past_df["part"]):
choices[user_ix].append(choice_id)
 timestamps[user_ix].append(timestamp)
 elapsed_times[user_ix].append(0 if cid > 13523 else -1)
 explanations[user_ix].append(0 if cid > 13523 else -1)
 orders[user_ix] += 1
if cid <= 13523:
part = int(part)
 qp_agg_df.at[user_ix, f"part{part}_count"] += 1
qp_agg_df.at[user_ix, f"part{part}_sum"] += int(choice_is_correct[choice_id])
 qp_agg_df.at[user_ix, f"question_count"] += 1
qp_agg_df.at[user_ix, f"question_sum"] += int(choice_is_correct[choice_id])
 for user_id in test_raw_df["user_id"].values:
if not user_id in user_id_map.keys() :
user_id_map[user_id] = user_ix_last
choices[user_ix_last] = []
timestamps[user_ix_last] = []
elapsed_times[user_ix_last] = []
explanations[user_ix_last] = []
orders[user_ix_last] = 0
qp_agg_df.loc[user_ix_last, :] = 0
user_ix_last += 1
get = np.vectorize(lambda x: user_id_map[x])
 test_raw_df["user_ix"] = get(test_raw_df["user_id"])
 test_users = test_raw_df["user_ix"].unique()
test_questions = test_raw_df[test_raw_df["content_type_id"] == False]["content_id"].unique()
test_raw_df = test_raw_df.merge(questions_df.loc[test_questions, ["part", 'content_2_accuracy', 'content_3_accuracy', 'content_4_accuracy', "idf"]],
on="content_id",
how="left")
 test_df = test_raw_df[test_raw_df["content_type_id"] == 0]
test_df["part"] = test_df["part"].astype(np.int32)
 test_df = test_df.merge(qp_agg_df.loc[test_users, :],
on="user_ix",
how="left")
 for user_ix, time, expl in zip(test_df["user_ix"], test_df["prior_question_elapsed_time"], test_df["prior_question_had_explanation"]):
if pd.isna(time):
continue
elapsed_times[user_ix] = [time if(t == -1 and c <= 54095)else t for t, c in zip(elapsed_times.get(user_ix, []), choices.get(user_ix, [])) ]
explanations[user_ix] = [expl if(t == -1 and c <= 54095)else t for t, c in zip(explanations.get(user_ix, []), choices.get(user_ix, [])) ]
order = []
choice_history = []
delta_timestamp_history = []
elapsed_time_history = []
explanation_history = []
for user_ix in test_df["user_ix"]:
order.append(orders.get(user_ix, 0))
choice_history.append(choices.get(user_ix, []))
delta_timestamp_history.append(timestamps.get(user_ix, []))
elapsed_time_history.append(elapsed_times.get(user_ix, []))
explanation_history.append(explanations.get(user_ix, []))
choice_history = tf.keras.preprocessing.sequence.pad_sequences(choice_history, maxlen=500, dtype="int32")
 raw_timestamp_history= tf.keras.preprocessing.sequence.pad_sequences(delta_timestamp_history, maxlen=500, value=-1, dtype="int64")
 raw_elapsed_time_history = tf.keras.preprocessing.sequence.pad_sequences(elapsed_time_history, maxlen=500, value=-1, dtype="float32")
 explanation_history = tf.keras.preprocessing.sequence.pad_sequences(explanation_history, maxlen=500, value=-1, dtype="float32")
 choice_history = tf.constant(choice_history, dtype="int32")
 raw_timestamp_history = tf.constant(raw_timestamp_history, dtype="int64")
 raw_elapsed_time_history = tf.constant(raw_elapsed_time_history, dtype="float32")
 explanation_history = tf.constant(explanation_history, dtype="float32")
 user_ix = tf.constant(test_df["user_ix"], dtype="int32")
 content_id = tf.constant(test_df["content_id"], dtype="int32")
 timestamp = tf.constant(test_df["timestamp"], dtype="int64")
 part = tf.constant(test_df["part"], dtype="int32")
 order = tf.constant(order, dtype="int32")
 feat = tf.constant(test_df[feat_col].values, dtype="float32")
 raw_feat = tf.constant(test_df[raw_feat_col].values, dtype="float32")
 start = timer()
input_ = predictor(user_ix, content_id, timestamp, order, choice_history, raw_timestamp_history, raw_elapsed_time_history, explanation_history, part, feat, raw_feat)
 print(timer() - start)
 start = timer()
nn_pred = model(*input_, training=False)
 print(timer() - start)
 test_df["answered_correctly"] =(np.squeeze(nn_pred.numpy()))
past_df = test_raw_df
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])