Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<define_variables>
import numpy as np
import pandas as pd
from datetime import datetime
question_id_map = {id_: i+1 for i, id_ in enumerate(questions_df["question_id"])}
lecture_id_map = {id_: i+questions_df.shape[0]+1 for i, id_ in enumerate(lectures_df["lecture_id"])}<import_modules>
init_time = datetime.now()
class BaseModel(tf.keras.Model):
def __init__(self, user_in_train,
content_embedding_size, choice_embedding_size,
choice_is_correct, choice_is_wrong, choice_is_question,
choice_tags, choice_parts, choice_typeof,
answer_accuracy_mean, answer_accuracy2_mean, answer_elapsed_time_mean, answer_elapsed_time_std, answer_correct_elapssed_time_mean,
correct_choice, choice_rate,
sample_window=500):
super(BaseModel, self ).__init__()
self.content_embedding = tf.keras.layers.Embedding(content_embedding_size+2, 64, mask_zero=True)
 self.choice_embedding = tf.keras.layers.Embedding(choice_embedding_size+2, 64, mask_zero=True)
 self.choice_tags_vec = tf.keras.layers.Embedding(choice_tags.shape[0],
choice_tags.shape[1],
weights=[choice_tags],
embeddings_initializer = None,
dtype="int16",
trainable=False)
 self.tag_embedding = tf.keras.layers.Embedding(choice_embedding_size+2, 16, mask_zero=True)
 self.timestamp_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)
 self.lag_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)
 self.delta_timestamp_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)
 self.cooltime_embedding = tf.keras.layers.Embedding(10000, 32, mask_zero=True)
 self.elapsed_embedding = tf.keras.layers.Embedding(100, 32, mask_zero=True)
 self.positional_embedding = tf.keras.layers.Embedding(500, 32, mask_zero=True)
 self.choice_is_correct = choice_is_correct
self.choice_is_wrong = choice_is_wrong
self.choice_is_question = choice_is_question
self.choice_tags = choice_tags
self.choice_parts = choice_parts
self.choice_typeof = choice_typeof
self.answer_accuracy_mean = answer_accuracy_mean
self.answer_accuracy2_mean = answer_accuracy2_mean
self.answer_elapsed_time_mean = answer_elapsed_time_mean
self.answer_elapsed_time_std = answer_elapsed_time_std
self.answer_correct_elapssed_time_mean = answer_correct_elapssed_time_mean
self.correct_choice = correct_choice
self.choice_rate = choice_rate
_ = self.content_embedding(0)
 _ = self.choice_embedding(0)
 self.order_in = tf.keras.layers.InputLayer(input_shape=(1,))
self.feat_in = tf.keras.layers.InputLayer(input_shape=(76,))
self.oh_in = tf.keras.layers.InputLayer(input_shape=(7,))
self.table_bn_clipper_qc = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.bn_clipper_history = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.bn_clipper_0 = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.bn_clipper_1 = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.bn_clipper_2 = tf.keras.Sequential([tf.keras.layers.BatchNormalization() , tf.keras.layers.Lambda(lambda x: tf.clip_by_value(x, -5, 5)) ])
 self.dense_qc_u0 = tf.keras.layers.Dense(64)
 self.dense_qc_v0 = tf.keras.layers.Dense(305)
 self.bn_qc0 = tf.keras.layers.BatchNormalization()
self.dense_qc_u1 = tf.keras.layers.Dense(64)
 self.dense_qc_v1 = tf.keras.layers.Dense(305)
 self.bn_qc1 = tf.keras.layers.BatchNormalization()
self.dense_kv_u0 = tf.keras.layers.Dense(64)
 self.dense_kv_v0 = tf.keras.layers.Dense(312)
 self.bn_kv0 = tf.keras.layers.BatchNormalization()
self.dense_kv_u1 = tf.keras.layers.Dense(64)
 self.dense_kv_v1 = tf.keras.layers.Dense(312)
 self.bn_kv1 = tf.keras.layers.BatchNormalization()
self.zero_embedding = tf.keras.layers.Embedding(1, 312)
 self.encoder_masks = []
for i in range(5):
row = tf.reshape(tf.repeat(tf.range(100*(i+1)) , 100),(100*(i+1), 100))
col = tf.reshape(tf.tile(tf.range(100)+100*i, [100*(i+1)]),(100*(i+1), 100))
self.encoder_masks.append(tf.cast(tf.expand_dims(tf.transpose(tf.where(row > col, tf.float32.min/100, 0.)), axis=0), "float32"))
self.dense_sa_k0 = tf.keras.layers.Dense(64)
 self.bn_sa_k0 = tf.keras.layers.BatchNormalization()
self.do_sa_k0 = tf.keras.layers.Dropout(0.1)
 self.dense_ts_k0 = tf.keras.layers.Dense(64)
 self.bn_ts_k0 = tf.keras.layers.BatchNormalization()
self.do_ts_k0 = tf.keras.layers.Dropout(0.1)
 self.do_ts_k1 = tf.keras.layers.Dropout(0.1)
 self.history_key_conv0 = tf.keras.layers.Conv1D(32, 1, 1, activation="selu")
 self.history_key_conv1 = tf.keras.layers.Conv1D(32, 1, 1)
 self.history_correct_key_conv = tf.keras.layers.Conv1D(32, 1, 1, activation="relu")
 self.history_choice_conv0 = tf.keras.layers.Conv1D(32, 1, 1, activation="selu")
 self.history_choice_conv1 = tf.keras.layers.Conv1D(32, 1, 1)
 self.history_wrong_choice_conv = tf.keras.layers.Conv1D(32, 1, 1, activation="relu")
 self.history_val_conv0 = tf.keras.layers.Conv1D(32, 1, 1, activation="selu")
 self.history_val_conv1 = tf.keras.layers.Conv1D(32, 1, 1)
 self.gloval_avg_pool_key_drop = tf.keras.layers.Dropout(0.1)
 self.gloval_avg_pool_choice_drop = tf.keras.layers.Dropout(0.1)
 self.gloval_avg_pool_val_drop = tf.keras.layers.Dropout(0.1)
 self.gloval_max_pool_key = tf.keras.layers.GlobalMaxPool1D()
self.gloval_avg_pool_key = tf.keras.layers.GlobalAvgPool1D()
self.gloval_max_pool_correct_key = tf.keras.layers.GlobalMaxPool1D()
self.gloval_max_pool_choice = tf.keras.layers.GlobalMaxPool1D()
self.gloval_avg_pool_choice = tf.keras.layers.GlobalAvgPool1D()
self.gloval_max_pool_wrong_choice = tf.keras.layers.GlobalMaxPool1D()
self.gloval_max_pool_val = tf.keras.layers.GlobalMaxPool1D()
self.gloval_avg_pool_val = tf.keras.layers.GlobalAvgPool1D()
self.dense_u0 = tf.keras.layers.Dense(256)
 self.dense_v0 = tf.keras.layers.Dense(1214)
 self.dense_u1 = tf.keras.layers.Dense(256)
 self.dense_v1 = tf.keras.layers.Dense(1214)
 self.dense_u2 = tf.keras.layers.Dense(256)
 self.dense_v2 = tf.keras.layers.Dense(1214)
 self.bnc0 = tf.keras.layers.BatchNormalization()
self.bnc1 = tf.keras.layers.BatchNormalization()
self.bnc2 = tf.keras.layers.BatchNormalization()
self.dense1 = tf.keras.layers.Dense(512, activation='relu')
 self.dense2 = tf.keras.layers.Dense(512, activation='relu')
 self.dense3 = tf.keras.layers.Dense(512, activation='relu')
 self.bn0 = tf.keras.layers.BatchNormalization()
self.bn1 = tf.keras.layers.BatchNormalization()
self.bn2 = tf.keras.layers.BatchNormalization()
self.bn3 = tf.keras.layers.BatchNormalization()
self.do0 = tf.keras.layers.Dropout(0)
 self.do1 = tf.keras.layers.Dropout(3/10)
 self.do2 = tf.keras.layers.Dropout(3/10)
 self.do3 = tf.keras.layers.Dropout(0)
 self.out = tf.keras.layers.Dense(1, activation='sigmoid')
 self.sample_window = sample_window
@tf.function(input_signature=[
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int16),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int64),
tf.TensorSpec(shape=(None, 500), dtype=tf.int16),
tf.TensorSpec(shape=(None, 500), dtype=tf.float32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int16),
tf.TensorSpec(shape=(None, 500), dtype=tf.float32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=(None, 76), dtype=tf.float32),
tf.TensorSpec(shape=[], dtype=tf.bool),
] 
)
 def call(self, user_ix, content_id, timestamp_cat, lag_time, order,
choice_history, timestamp_history, delta_timestamp_history,
raw_elapsed_time_history, elapsed_time_history,
explanation_history, cooltime_history,
part, feat,
training=None):
if training:
user_ix = tf.where(tf.random.uniform([len(user_ix)], maxval=1)<.01, 0, user_ix)
 content_emb = self.content_embedding(content_id)
 timestamp_emb = self.timestamp_embedding(tf.reshape(timestamp_cat,(-1,)))
 choice_emb = tf.reduce_mean(tf.concat([tf.expand_dims(self.choice_embedding(content_id*4 + i), axis=1)for i in range(4)], axis=1), axis=1)
 lag_emb = self.lag_embedding(tf.reshape(lag_time,(-1,)))
 order_in = self.order_in(tf.expand_dims(tf.cast(order, "float32"), axis=-1))
order_in = tf.math.log1p(order_in)
 feat_in = self.feat_in(feat)
 part_oh = tf.reshape(self.oh_in(tf.one_hot(tf.cast(part, "int32"), 7)) ,(-1, 7))
content_tags_emb = self.tag_embedding(self.choice_tags_vec(tf.reshape(content_id*4,(-1,))))
emb_max = tf.keras.backend.max(content_tags_emb, axis=1)
 emb_min = tf.keras.backend.min(content_tags_emb, axis=1)
 content_tags_emb = tf.where(emb_max > -emb_min, emb_max, emb_min)
 correct_choice = tf.reshape(tf.one_hot(tf.gather(self.correct_choice, content_id*4), 4),(-1, 4))
choice_rate = tf.reshape(tf.gather(self.choice_rate, content_id*4),(-1, 4))
content_accuracy = tf.reshape(tf.gather(self.answer_accuracy_mean, content_id*4),(-1, 1))
content_norm = tf.stop_gradient(tf.math.log1p(tf.norm(content_emb, axis=1, keepdims=True)))
 timestamp_norm = tf.stop_gradient(tf.math.log1p(tf.norm(timestamp_emb, axis=1, keepdims=True)))
 choice_norm = tf.stop_gradient(tf.math.log1p(tf.norm(choice_emb, axis=1, keepdims=True)))
 lag_norm = tf.stop_gradient(tf.math.log1p(tf.norm(lag_emb, axis=1, keepdims=True)))
 tag_norm = tf.stop_gradient(tf.math.log1p(tf.norm(content_tags_emb, axis=1, keepdims=True)))
 table_features = tf.reshape(tf.concat([order_in, feat_in, part_oh, content_norm, timestamp_norm, choice_norm, lag_norm, tag_norm, correct_choice, choice_rate], axis=-1),(-1, 97))
table_features = self.table_bn_clipper_qc(table_features)
 query_context_features_0 = tf.concat([content_emb, choice_emb, timestamp_emb, lag_emb, content_tags_emb, table_features], axis=1)
 query_context_features_0 = tf.reshape(query_context_features_0,(-1, 305))
query_context_features = query_context_features_0 * self.dense_qc_v0(self.dense_qc_u0(self.bn_qc0(query_context_features_0)))+ query_context_features_0
query_context_features = query_context_features_0 * self.dense_qc_v1(self.dense_qc_u1(self.bn_qc1(query_context_features)))+ query_context_features
content_mask = choice_history > 0
idx = tf.cast(tf.where(content_mask), "int32")
 content_history = choice_history // 4
correct_history = tf.expand_dims(tf.gather(tf.where(self.choice_is_correct, 1., 0.) , choice_history), axis=-1)
 wrong_history = tf.expand_dims(tf.gather(tf.where(self.choice_is_wrong, 1., 0.) , choice_history), axis=-1)
 question_history = tf.expand_dims(tf.gather(tf.where(self.choice_is_question, 1., 0.) , choice_history), axis=-1)
 accuracy_history = tf.expand_dims(tf.where(tf.reshape(question_history,(-1, 500)) ==1., tf.gather(self.answer_accuracy_mean, choice_history), 0), axis=-1)
 history_choice_emb = self.choice_embedding(tf.boolean_mask(choice_history, content_mask))
history_delta_timestamp_emb = self.delta_timestamp_embedding(tf.boolean_mask(delta_timestamp_history, content_mask))
history_elapsed_time_emb = self.elapsed_embedding(tf.boolean_mask(elapsed_time_history, content_mask))
history_cooltime_emb = self.cooltime_embedding(tf.boolean_mask(cooltime_history, content_mask))
history_explanation = tf.expand_dims(tf.cast(tf.boolean_mask(explanation_history, content_mask), "float32"), -1)
 history_position_emb = self.positional_embedding(tf.boolean_mask(tf.reshape(tf.tile(tf.range(self.sample_window), [len(user_ix)]),(len(user_ix), 500)) , content_mask))
masked_choice = tf.boolean_mask(choice_history, content_mask)
 history_choice = tf.reshape(tf.one_hot(tf.where(tf.gather(self.choice_is_question, masked_choice), masked_choice%4, -1), 4),(-1, 4))
history_correct_choice = tf.reshape(tf.one_hot(tf.gather(self.correct_choice, masked_choice), 4),(-1, 4))
history_choice_rate = tf.reshape(tf.gather(self.choice_rate, masked_choice),(-1, 4))
history_is_correct = tf.expand_dims(tf.gather(tf.where(self.choice_is_correct, 1., 0.) , tf.boolean_mask(choice_history, content_mask)) , -1)
 history_is_wrong = tf.expand_dims(tf.gather(tf.where(self.choice_is_wrong, 1., 0.) , tf.boolean_mask(choice_history, content_mask)) , -1)
 history_is_question = tf.expand_dims(tf.gather(tf.where(self.choice_is_question, 1., 0.) , tf.boolean_mask(choice_history, content_mask)) , -1)
 history_answer_mean = tf.expand_dims(tf.gather(self.answer_accuracy_mean, tf.boolean_mask(choice_history, content_mask)) , -1)
 history_answer2_mean = tf.expand_dims(tf.gather(self.answer_accuracy2_mean, tf.boolean_mask(choice_history, content_mask)) , -1)
 history_elapsed_time_mean = tf.expand_dims(tf.gather(self.answer_elapsed_time_mean, tf.boolean_mask(choice_history, content_mask)) , -1)
 history_correct_elapsed_time_mean = tf.expand_dims(tf.gather(self.answer_correct_elapssed_time_mean, tf.boolean_mask(choice_history, content_mask)) , -1)
 history_raw_elapsed_time = tf.expand_dims(tf.boolean_mask(raw_elapsed_time_history, content_mask), -1)
 history_ac_et_rate = history_correct_elapsed_time_mean / history_elapsed_time_mean
history_elapsed_time_rate = tf.math.log1p(tf.where(history_is_question == 1., history_raw_elapsed_time / history_elapsed_time_mean, 1))
history_correct_elapsed_time_rate = tf.math.log1p(tf.where(history_is_question == 1., history_raw_elapsed_time / history_correct_elapsed_time_mean, 1))
history_elapsed_time_mean = tf.math.log1p(history_elapsed_time_mean)
 history_correct_elapsed_time_mean = tf.math.log1p(history_correct_elapsed_time_mean)
 history_content_emb = self.content_embedding(tf.boolean_mask(choice_history//4, content_mask))
history_tag_emb = self.tag_embedding(self.choice_tags_vec(tf.boolean_mask(choice_history, content_mask)))
 history_tag_emb_max = tf.keras.backend.max(history_tag_emb, axis=1)
 history_tag_emb_min = tf.keras.backend.min(history_tag_emb, axis=1)
 history_tag_emb_cd = tf.cast(history_tag_emb_max > history_tag_emb_min, "float32")
 history_tag_emb = history_tag_emb_max * history_tag_emb_cd + history_tag_emb_min *(1 - history_tag_emb_cd)
 history_part_oh = tf.one_hot(tf.gather(self.choice_parts, tf.boolean_mask(choice_history, content_mask)) , 7)
 history_typeof_oh = tf.one_hot(tf.gather(self.choice_typeof, tf.boolean_mask(choice_history, content_mask)) , 4)
 history_choice_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_choice_emb ,axis=1, keepdims=True)))
 history_delta_timestamp_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_delta_timestamp_emb ,axis=1, keepdims=True)))
 history_elapsed_time_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_elapsed_time_emb ,axis=1, keepdims=True)))
 history_cooltime_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_cooltime_emb ,axis=1, keepdims=True)))
 history_position_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_position_emb ,axis=1, keepdims=True)))
 history_tag_norm = tf.stop_gradient(tf.math.log1p(tf.norm(history_tag_emb ,axis=1, keepdims=True)))
 history_norms = self.bn_clipper_history(tf.concat([history_choice_norm, history_delta_timestamp_norm, history_elapsed_time_norm,
history_cooltime_norm, history_position_norm, history_tag_norm], axis=1))
history_keyvalue_0 = tf.concat([history_content_emb,
history_choice_emb,
history_delta_timestamp_emb,
history_elapsed_time_emb,
history_cooltime_emb,
history_position_emb,
history_norms,
history_explanation,
history_choice,
history_correct_choice,
history_choice_rate,
history_is_correct,
history_is_wrong,
history_is_question,
history_answer_mean,
history_answer2_mean,
history_elapsed_time_mean,
history_correct_elapsed_time_mean,
history_ac_et_rate,
history_elapsed_time_rate,
history_correct_elapsed_time_rate,
history_tag_emb,
history_part_oh,
history_typeof_oh],
axis = -1)
 zero_embedding = self.zero_embedding(0)
 history_keyvalue = history_keyvalue_0 * self.dense_kv_v0(self.dense_kv_u0(self.bn_kv0(history_keyvalue_0)))+ history_keyvalue_0
history_keyvalue = history_keyvalue_0 * self.dense_kv_v1(self.dense_kv_u1(self.bn_kv1(history_keyvalue)))+ history_keyvalue
history_keyvalue = tf.scatter_nd(idx, history_keyvalue,(len(user_ix), self.sample_window, 312))
history_keyvalue += tf.expand_dims(tf.cast(~content_mask, "float32"), -1)* tf.expand_dims(zero_embedding, axis=0)
 query, context_features = tf.split(query_context_features, [64, -1], axis=1)
 query = tf.expand_dims(query, axis=1)
 history_key, history_choice, history_value = tf.split(history_keyvalue, [64, 64, -1], axis=2)
 query0 = tf.identity(query)
 history_key0 = tf.identity(history_key)
 queries = []
key_attentions = []
choice_attentions = []
value_attentions = []
add_features = []
queries.append(query)
 logit = tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(history_key, [0, 0, 0], [-1, -1, 64]), transpose_b=True)/ 8
weight = self.do_ts_k0(tf.math.softmax(logit))
key_attention = tf.matmul(weight, history_key)
 choice_attention = tf.matmul(weight, history_choice)
 sc_logit = tf.clip_by_value(( tf.math.sign(logit)* tf.math.log1p(tf.abs(logit))), -4, 4)
 logit_attention = tf.reduce_sum(weight * sc_logit, axis=2)
 cossim = tf.reshape(tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(key_attention, [0, 0, 0], [-1, -1, 64]), transpose_b=True)/(tf.norm(query, axis=2, keepdims=True)*tf.norm(key_attention, axis=2, keepdims=True)+1e-9),(-1, 1))
ac_div = tf.matmul(weight, question_history)
 ac_num = tf.matmul(weight, correct_history)
 eac_num = tf.matmul(weight, accuracy_history)
 eac_rate = tf.reshape(tf.where(ac_div > 0, eac_num/(ac_div+1e-11), 0.657),(-1, 1))
eac_rate = tf.math.maximum(0.01, eac_rate)
 ac_rate = tf.reshape(tf.where(ac_div > 0, ac_num/(ac_div+1e-11), 0.657),(-1, 1))
ac_rate =(ac_rate * tf.cast(tf.expand_dims(order, axis=1), "float32")+ eac_rate * 10.) /(tf.cast(tf.expand_dims(order, axis=1), "float32")+ 10.)
ac_eac_rate = ac_rate / eac_rate
h_ac_rate = 2 /(1/tf.maximum(0.01, ac_rate)+ 1/tf.maximum(0.01, content_accuracy))
cossim= tf.clip_by_value(cossim, -1, 1)
 logit_attention = tf.clip_by_value(logit_attention, -5, 5)
 ac_rate = tf.clip_by_value(ac_rate, 0, 1)
 eac_rate = tf.clip_by_value(eac_rate, 0, 1)
 ac_eac_rate = tf.clip_by_value(tf.math.log1p(tf.abs(ac_eac_rate)) , -1, 5)
 calc_feature = tf.stop_gradient(tf.concat([cossim, logit_attention, ac_rate, eac_rate, ac_eac_rate, h_ac_rate], axis=1))
norm_feature = tf.stop_gradient(tf.math.log1p(tf.concat([tf.norm(query, axis=2), tf.norm(key_attention, axis=2), tf.norm(choice_attention, axis=2)], axis=1)))
 add_feature = self.bn_clipper_0(tf.concat([tf.reshape(calc_feature,(-1, 1, 6)) , tf.reshape(norm_feature,(-1, 1, 3)) ], axis=2))
key_attentions.append(key_attention)
 choice_attentions.append(choice_attention)
 add_features.append(tf.reshape(add_feature,(-1, 9)))
 query = query0 * self.dense_ts_k0(self.bn_ts_k0(tf.concat([key_attention, choice_attention, add_feature], axis=2)))+ query
history_keys = tf.split(history_key, [100]*5, axis=1)
 history_keys0 = tf.split(history_key0, [100]*5, axis=1)
 history_timestamps = tf.split(timestamp_history, [100]*5, axis=1)
 zero_emb_key, zero_emb_choice, _ = tf.split(model.zero_embedding(0), [64, 64, -1], axis=0)
 zero_emb_key = tf.reshape(tf.tile(zero_emb_key, [len(user_ix)]),(-1, 1, 64))
zero_emb_choice = tf.reshape(tf.tile(zero_emb_choice, [len(user_ix)]),(-1, 1, 64))
history_new_keys = []
attentions = []
history_attentions = []
for i in range(5):
key_slice = tf.slice(history_key, [0, 0, 0], [-1, 100*(i+1), -1])
 key_slice = tf.concat([zero_emb_key, key_slice], axis=1)
 choice_slice = tf.slice(history_choice, [0, 0, 0], [-1, 100*(i+1), -1])
 choice_slice = tf.concat([zero_emb_choice, choice_slice], axis=1)
 timestamp_slice = tf.slice(timestamp_history, [0, 0], [-1, 100*(i+1)])
 timestamp_slice = tf.concat([tf.ones(( len(user_ix), 1), dtype="int64")*-1, timestamp_slice], axis=1)
 query_timestamp = tf.maximum(tf.constant(0, dtype="int64"), tf.reshape(tf.repeat(history_timestamps[i], 100*(i+1)+1, axis=1),(-1, 100, 100*(i+1)+1)))
 key_timestamp = tf.reshape(tf.tile(timestamp_slice, [1, 100]),(-1, 100, 100*(i+1)+1))
encoder_mask = tf.where(key_timestamp < query_timestamp, 0., tf.float32.min/100)
 logit = tf.matmul(tf.slice(history_keys[i], [0, 0, 0], [-1, -1, 64]), tf.slice(key_slice, [0, 0, 0], [-1, -1, 64]), transpose_b=True)+ encoder_mask
weight = self.do_sa_k0(tf.math.softmax(logit/8))
key_attention = tf.matmul(weight, key_slice)
 choice_attention = tf.matmul(weight, choice_slice)
 history_exist = tf.where(history_timestamps[0] >= 0, 1., 0.)
sc_logit = tf.clip_by_value(tf.math.sign(logit)* tf.math.log1p(tf.abs(logit)) , -4, 4)
 history_logit = tf.reduce_sum(sc_logit * weight, axis=2)* history_exist
history_cossim = tf.reduce_sum(key_attention * history_keys[i], axis=2)/(tf.norm(key_attention, axis=2)* tf.norm(history_keys[i], axis=2)+ 1e-9)
 history_cossim *= history_exist
history_cossim = tf.clip_by_value(history_cossim, -1, 1)
 history_logit = tf.stop_gradient(tf.expand_dims(history_logit, axis=2))
history_exist = tf.stop_gradient(tf.expand_dims(history_exist, axis=2))
history_cossim = tf.stop_gradient(tf.expand_dims(history_cossim, axis=2))
history_add = self.bn_clipper_1(tf.concat([history_logit, history_logit, history_cossim], axis=2))
attention = tf.concat([key_attention, choice_attention, history_add], axis=2)
 history_new_keys.append(history_keys0[i] * self.dense_sa_k0(self.bn_sa_k0(attention)) + history_keys[i])
 history_attentions.append(attention)
 history_key = tf.concat(history_new_keys, axis=1)
 history_value = tf.concat([history_value, tf.concat(history_attentions, axis=1)], axis=2)
 queries.append(query)
 logit = tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(history_key, [0, 0, 0], [-1, -1, 64]), transpose_b=True)/ 8
weight = self.do_ts_k1(tf.math.softmax(logit))
key_attention = tf.matmul(weight, history_key)
 choice_attention = tf.matmul(weight, history_choice)
 value_attention = tf.matmul(weight, history_value)
 sc_logit = tf.clip_by_value(( tf.math.sign(logit)* tf.math.log1p(tf.abs(logit))), -4, 4)
 logit_attention = tf.reduce_sum(weight * sc_logit, axis=2)
 cossim = tf.reshape(tf.matmul(tf.slice(query, [0, 0, 0], [-1, -1, 64]), tf.slice(key_attention, [0, 0, 0], [-1, -1, 64]), transpose_b=True)/(tf.norm(query, axis=2, keepdims=True)*tf.norm(key_attention, axis=2, keepdims=True)+1e-9),(-1, 1))
ac_div = tf.matmul(weight, question_history)
 ac_num = tf.matmul(weight, correct_history)
 eac_num = tf.matmul(weight, accuracy_history)
 eac_rate = tf.reshape(tf.where(ac_div > 0, eac_num/(ac_div+1e-11), 0.657),(-1, 1))
eac_rate = tf.math.maximum(0.01, eac_rate)
 ac_rate = tf.reshape(tf.where(ac_div > 0, ac_num/(ac_div+1e-11), 0.657),(-1, 1))
ac_rate =(ac_rate * tf.cast(tf.expand_dims(order, axis=1), "float32")+ eac_rate * 10.) /(tf.cast(tf.expand_dims(order, axis=1), "float32")+ 10.)
h_ac_rate = 2 /(1/tf.maximum(0.01, ac_rate)+ 1/tf.maximum(0.01, content_accuracy))
ac_eac_rate = ac_rate / eac_rate
cossim= tf.clip_by_value(cossim, -1, 1)
 logit_attention = tf.clip_by_value(logit_attention, -5, 5)
 ac_rate = tf.clip_by_value(ac_rate, 0, 1)
 eac_rate = tf.clip_by_value(eac_rate, 0, 1)
 ac_eac_rate = tf.clip_by_value(tf.math.log1p(tf.abs(ac_eac_rate)) , -1, 5)
 calc_feature = tf.stop_gradient(tf.concat([cossim, logit_attention, ac_rate, eac_rate, ac_eac_rate, h_ac_rate], axis=1))
norm_feature = tf.stop_gradient(tf.math.log1p(tf.concat([tf.norm(query, axis=2), tf.norm(key_attention, axis=2), tf.norm(choice_attention, axis=2)], axis=1)))
 add_feature = self.bn_clipper_2(tf.concat([tf.reshape(calc_feature,(-1, 1, 6)) , tf.reshape(norm_feature,(-1, 1, 3)) ], axis=2))
key_attentions.append(key_attention)
 choice_attentions.append(choice_attention)
 value_attentions.append(value_attention)
 add_features.append(tf.reshape(add_feature,(-1, 9)))
 queries_vec = tf.reshape(tf.concat(queries, axis=2),(-1, 128))
key_attentions = tf.reshape(tf.concat(key_attentions, axis=2),(-1, 128))
choice_attentions = tf.reshape(tf.concat(choice_attentions, axis=2),(-1, 128))
value_attentions = tf.reduce_mean(tf.concat(value_attentions, axis=1), axis=1)
 feature_attentions = tf.concat(add_features, axis=1)
 filter_res_key_max = self.gloval_max_pool_key(self.history_key_conv0(history_key))
filter_res_key_avg = self.gloval_avg_pool_key(self.gloval_avg_pool_key_drop(self.history_key_conv1(history_key)))
 filter_res_correct_key_max = self.gloval_max_pool_correct_key(self.history_correct_key_conv(history_key)* correct_history)
 filter_res_choice_max = self.gloval_max_pool_choice(self.history_choice_conv0(history_choice))
filter_res_choice_avg = self.gloval_avg_pool_choice(self.gloval_avg_pool_choice_drop(self.history_choice_conv1(history_choice)))
 filter_res_wrong_choice_max = self.gloval_max_pool_wrong_choice(self.history_wrong_choice_conv(history_choice)* wrong_history)
 filter_res_val_max = self.gloval_max_pool_val(self.history_val_conv0(history_value))
filter_res_val_avg = self.gloval_avg_pool_val(self.gloval_avg_pool_val_drop(self.history_val_conv1(history_value)))
 filter_res = tf.concat([filter_res_key_max, filter_res_key_avg, filter_res_correct_key_max,
filter_res_choice_max, filter_res_choice_avg,filter_res_wrong_choice_max,
filter_res_val_max, filter_res_val_avg], axis=1)
 table_features = tf.concat([table_features, feature_attentions], axis=1)
 X0 = tf.keras.layers.Concatenate()([context_features,
queries_vec, key_attentions, choice_attentions, value_attentions,
feature_attentions, filter_res])
 prod_out0 = self.dense_v0(self.dense_u0(self.bnc0(X0)))
 X = X0 * prod_out0 + X0
prod_out1 = self.dense_v1(self.dense_u1(self.bnc1(X)))
 X = X0 * prod_out1 + X
prod_out2 = self.dense_v2(self.dense_u2(self.bnc2(X)))
 X_feat = tf.concat([X0 * prod_out2 + X, table_features], axis=1)
 X = self.bn0(X_feat)
 X = self.do0(X)
 X = self.dense1(X)
 X = self.bn1(X)
 X = self.do1(X)
 X = self.dense2(X)
 X = self.bn2(X)
 X = self.do2(X)
 X = self.dense3(X)
 X = self.bn3(X)
 X = self.do3(X)
 return self.out(X )<define_variables>
IS_DEV = True
TREAT_OUTLIERS = False
if(IS_DEV == True):
TRAIN_FILE = '.. /input/train.csv'
TEST_FILE = '.. /input/test.csv'
COMPRESSION = None
SKIP_ROWS = range(1, 170000000)
 else:
TRAIN_FILE = '.. /input/train.csv.zip'
TEST_FILE = '.. /input/test.csv.zip'
COMPRESSION = 'zip'
TRAIN_SAMPLE_FILE = '.. /input/' + 'train_sample.csv'
LEARNING_RATE_INIT = 0.001
LEARNING_RATE_END = 0.0001
BATCH_SIZE = 20000
EPOCHS = 2
EMBEDDING_N = 50
DENSE_N = 1024
SPATIAL_DROPOUT_1D = 0.2
DROPOUT_1 = 0.2
DROPOUT_2 = 0.2
feat_in_col =(
 ['content_2_accuracy', 'content_3_accuracy', 'content_4_accuracy', "idf"]
+ ["correct_streak", "wrong_streak"]
+ [f"user_choice_{i}_rate_rate" for i in range(4)]
+ [f"user_wrong_choice_{i}_rate_rate" for i in range(4)]
+ [f"user_choice_rate_rate_correct"]
+ [f"user_wrong_choice_rate_rate_correct"]
+ ["harmonic_mean", "part_harmonic_mean", "frequency", "content_seen_sum", "acc_mean", "acc_per_elapse", "elp_mean", "scaled_elapsed_time_mean", "exp_mean"]
+ ['part0_sum', 'part0_count', 'part1_sum', 'part1_count', 'part2_sum', 'part2_count', 'part3_sum', 'part3_count', 'part4_sum', 'part4_count', 'part5_sum', 'part5_count', 'part6_sum', 'part6_count', 'question_sum', 'question_count']
+ [f"part{i}_accuracy_rate" for _ in range(7)] + ["total_accuracy_rate"]
+ [f"part{i}_count_rate" for _ in range(7)]
+ ["questions_one_day", "questions_one_hour", "questions_one_week"]
+ ["day_angle_cos_mean", "day_angle_sin_mean", "day_angle_norm", "day_angle_normed_cos", "day_angle_normed_sin", "week_angle_cos_mean", "week_angle_sin_mean", "week_angle_norm", "week_angle_normed_cos", "week_angle_normed_sin"]
+ ["last_correct_ts", "last_wrong_ts", "last_correct_ts_rate", "last_wrong_ts_rate"]
+ ["delta_last_2", "delta_last_1", "delta_last_0"] 
)
 context_features =(
 [f"context_choice_emb_{i}" for i in range(64)]
+ [f"context_timestamp_emb_{i}" for i in range(32)]
+ [f"context_lag_emb_{i}" for i in range(32)]
+ [f"content_tags_emb_{i}" for i in range(16)]
+ ["context_order"]
+ ["context_" + feat for feat in feat_in_col]
+ [f"context_part_oh_{i}" for i in range(7)]
+ ["context_content_norm", "context_timestamp_norm", "context_choice_norm", "context_lag_norm", "context_tag_norm"]
+ [f"context_correct_choice_oh_{i}" for i in range(4)]
+ [f"context_content_choice_rate_{i}" for i in range(4)] 
)
 history_value_features =(
 [f"history_delta_timestamp_emb_{i}" for i in range(32)]
+ [f"history_elapsed_time_emb_{i}" for i in range(32)]
+ [f"history_cooltime_emb_{i}" for i in range(32)]
+ [f"history_position_emb_{i}" for i in range(32)]
+ ["history_choice_norm", "history_delta_timestamp_norm", "history_elapsed_time_norm", "history_cooltime_norm", "history_position_norm", "history_tag_norm"]
+ ["history_explanation"]
+ [f"history_choice{i}" for i in range(4)]
+ [f"history_correct_choice{i}" for i in range(4)]
+ [f"history_choice_rate{i}" for i in range(4)]
+ ["history_is_correct", "history_is_wrong", "history_is_question"]
+ ["history_answer_mean", "history_answer2_mean"]
+ ["history_elapsed_time_mean", "history_correct_elapsed_time_mean", "history_ac_et_rate"]
+ ["history_elapsed_time_rate", "history_correct_elapsed_time_rate"]
+ [f"history_tag_{i}" for i in range(16)]
+ [f"history_part_oh_{i}" for i in range(7)]
+ [f"history_typeof_oh_{i}" for i in range(4)]
+ [f"history_encoder_key_attention_{i}" for i in range(64)]
+ [f"history_encoder_choice_attention_{i}" for i in range(64)]
+ [f"history_encoder_logit", "history_encoder_exist", "history_encoder_cossim"] 
)
 filter_cols =(
 [f"filter_res_key_max_{i}" for i in range(32)]
+ [f"filter_res_key_avg_{i}" for i in range(32)]
+ [f"filter_res_correct_key_max_{i}" for i in range(32)]
+ [f"filter_res_choice_max_{i}" for i in range(32)]
+ [f"filter_res_choice_avg_{i}" for i in range(32)]
+ [f"filter_res_wrong_choice_max_{i}" for i in range(32)]
+ [f"filter_res_value_max_{i}" for i in range(32)]
+ [f"filter_res_value_avg_{i}" for i in range(32)] 
)
 X0_cols =(
 context_features
+ [f"query_emb_level0_{i}" for i in range(64)] + [f"query_emb_level1_{i}" for i in range(64)]
+ [f"key_attention_emb_level0_{i}" for i in range(64)] + [f"key_attention_emb_level1_{i}" for i in range(64)]
+ [f"choice_attention_emb_level0_{i}" for i in range(64)] + [f"choice_attention_emb_level1_{i}" for i in range(64)]
+ history_value_features
+ [f"history_level0_{name}" for name in ["cossim", "logit_attention", "ac_rate", "eac_rate", "ac_eac_rate"," h_ac_rate"]]
+ [f"history_level0_{name}_norm" for name in ["query_attention", "key_attention", "choice_attention"]]
+ [f"history_level1_{name}" for name in ["cossim", "logit_attention", "ac_rate", "eac_rate", "ac_eac_rate"," h_ac_rate"]]
+ [f"history_level1_{name}_norm" for name in ["query_attention", "key_attention", "choice_attention"]]
+ filter_cols 
)
 table_features =(
 ["raw_order"]
+ ["raw_" + feat for feat in feat_in_col]
+ [f"raw_part_oh_{i}" for i in range(7)]
+ ["raw_content_norm", "raw_timestamp_norm", "raw_choice_norm", "raw_lag_norm", "raw_tag_norm"]
+ [f"raw_correct_choice_oh_{i}" for i in range(4)]
+ [f"raw_content_choice_rate_{i}" for i in range(4)]
+ [f"raw_history_level0_{name}" for name in ["cossim", "logit_attention", "ac_rate", "eac_rate", "ac_eac_rate"," h_ac_rate"]]
+ [f"raw_history_level0_{name}_norm" for name in ["query_attention", "key_attention", "choice_attention"]]
+ [f"raw_history_level1_{name}" for name in ["cossim", "logit_attention", "ac_rate", "eac_rate", "ac_eac_rate"," h_ac_rate"]]
+ [f"raw_history_level1_{name}_norm" for name in ["query_attention", "key_attention", "choice_attention"]]
)<load_pretrained>
dtypes = {
'ip': 'uint32',
'app': 'uint16',
'device': 'uint16',
'os': 'uint16',
'channel': 'uint16',
'is_attributed': 'uint8',
'click_id': 'uint32'
}
model = BaseModel(max(user_id_map.values()), lectures_df["content_id"].max() , lectures_df["content_id"].max() *4 + 3,
choice_is_correct, choice_is_wrong, choice_is_question,
choice_tags, choice_parts, choice_typeof,
answer_accuracy_mean, answer_accuracy2_mean, answer_elapsed_time_mean, answer_elapsed_time_std, answer_correct_elapssed_time_mean,
correct_choice, choice_rate)
 model.load_weights(".. /input/nadare-processed-2/model8" )<import_modules>
train_sample = pd.read_csv(
 TRAIN_SAMPLE_FILE,
dtype=dtypes,
usecols=['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed'],
compression = None,
header=0,
engine='c' 
)
 train_sample.head()
class Predictor(tf.Module):
def __init__(self,
timestamp_bins,
delta_bins,
elapsedtime_bins,
choice_is_correct,
choice_is_wrong,
choice_is_question,
answer_accuracy_mean,
answer_elapsed_time_mean,
answer_elapsed_time_std,
correct_choice,
choice_rate,
part_mean,
part_rate):
self.sample_window = 500
self.timestamp_bins = tf.constant(timestamp_bins, dtype="float32")
 self.delta_bins = tf.constant(delta_bins, dtype="float32")
 self.elapsedtime_bins = tf.constant(elapsedtime_bins, dtype="float32")
 self.choice_is_correct = choice_is_correct
self.choice_is_wrong = choice_is_wrong
self.choice_is_question = choice_is_question
self.answer_accuracy_mean = answer_accuracy_mean
self.answer_elapsed_time_mean = answer_elapsed_time_mean
self.answer_elapsed_time_std = answer_elapsed_time_std
self.correct_choice = correct_choice
self.choice_rate = choice_rate
self.part_mean = part_mean
self.part_rate = part_rate
@tf.function(input_signature=[
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=None, dtype=tf.int64),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int32),
tf.TensorSpec(shape=(None, 500), dtype=tf.int64),
tf.TensorSpec(shape=(None, 500), dtype=tf.float32),
tf.TensorSpec(shape=(None, 500), dtype=tf.float32),
tf.TensorSpec(shape=None, dtype=tf.int32),
tf.TensorSpec(shape=(None, 4), dtype=tf.float32),
tf.TensorSpec(shape=(None, 16), dtype=tf.float32)
 ] 
)
 def __call__(self, user_ix, content_id, timestamp, order,
choice_history, raw_timestamp_history, raw_elapsed_time_history, explanation_history,
part, feat, raw_feat):
N = len(user_ix)
 timestamp_cat = tf.reshape(tf.cast(tf.searchsorted(self.timestamp_bins, tf.cast(timestamp, "float32")) , dtype="int16"),(-1,))
timestamp_history = tf.identity(raw_timestamp_history)
 delta_timestamp_history = tf.expand_dims(timestamp, axis=-1)- raw_timestamp_history
delta_timestamp_history = tf.cast(tf.searchsorted(self.delta_bins, tf.cast(tf.keras.backend.flatten(delta_timestamp_history), "float32"), side="right"), dtype="int16")
 delta_timestamp_history = tf.reshape(delta_timestamp_history, [-1, self.sample_window])
 elapsed_time_history = tf.cast(tf.searchsorted(self.elapsedtime_bins, tf.cast(tf.keras.backend.flatten(raw_elapsed_time_history), "float32"), side="right"), dtype="int16")
 elapsed_time_history = tf.reshape(elapsed_time_history, [-1, self.sample_window])
 raw_elapsed_time_history = tf.cast(tf.reshape(raw_elapsed_time_history, [-1, self.sample_window]), "float32")
 timestamp_histories = tf.split(tf.cast(timestamp_history, "float32"), 500, axis=1)
 elapsed_time_histories = tf.split(raw_elapsed_time_history, 500, axis=1)
 lag_time = tf.where(tf.reshape(timestamp_histories[-1],(-1,)) < 0, 0, tf.searchsorted(self.delta_bins, tf.cast(timestamp, "float32")- tf.reshape(timestamp_histories[-1],(-1,))))
ts = tf.reshape(tf.cast(timestamp, "float32"),(-1, 1))
ept = tf.ones(( N, 1), dtype="float32")* tf.float32.max/100
nept = tf.zeros(( N, 1), dtype="float32")
 cooltimes = []
for i in range(499, -1, -1):
tsn = timestamp_histories[i]
if i < 499:
ept = tf.where(timestamp_histories[i]==timestamp_histories[i+1], ept, nept)
 cooltimes.append(ts - tsn - ept)
 ts = tf.where(elapsed_time_histories[i] <= 0, ts, tsn)
 if i < 499:
nept = tf.where(timestamp_histories[i]==timestamp_histories[i+1], elapsed_time_histories[i] + nept, elapsed_time_histories[i])
 nept = tf.where(elapsed_time_histories[i] <= 0, ept, nept)
 else:
nept = tf.where(elapsed_time_histories[i] <= 0, ept, elapsed_time_histories[i])
 cooltimes = tf.concat(cooltimes[::-1], axis=1)
 cooltimes = tf.searchsorted(self.delta_bins, tf.reshape(cooltimes,(-1,)))
 cooltime_history = tf.reshape(cooltimes, [N, self.sample_window])
 cooltime_history = tf.where(choice_history > 0, tf.minimum(9999, cooltime_history+1), 0)
 correct_answer_sum = tf.reduce_sum(tf.where(tf.gather(self.choice_is_correct, choice_history), 1., 0.) , axis=1, keepdims=True)
 history_is_question = tf.gather(self.choice_is_question, choice_history)
 history_is_correct = tf.gather(self.choice_is_correct, choice_history)
 history_is_wrong = tf.gather(self.choice_is_wrong, choice_history)
 explanation_sum = tf.reduce_sum(tf.where(history_is_question, explanation_history, 0), axis=1, keepdims=True)
 question_sum = tf.reduce_sum(tf.where(history_is_question, 1., 0.) , axis=1, keepdims=True)
 wrong_sum = tf.reduce_sum(tf.where(history_is_wrong, 1., 0.) , axis=1, keepdims=True)
 elapsed_acc_sum = tf.reduce_sum(tf.where(history_is_question, tf.gather(self.answer_accuracy_mean, choice_history), 0), axis=1, keepdims=True)
 elapsed_time_content_mean = tf.gather(self.answer_elapsed_time_mean, choice_history)
 elapsed_time_content_std = tf.gather(self.answer_elapsed_time_std, choice_history)
 scaled_elapsed_time = tf.where(history_is_question, tf.math.log1p(raw_elapsed_time_history), 0.)
scaled_elapsed_time_sum = tf.reduce_sum(scaled_elapsed_time, axis=1, keepdims=True)
 correct_streak = tf.zeros(( N, 1))
correct_continue = tf.ones(( N, 1))
wrong_streak = tf.zeros(( N, 1))
wrong_continue = tf.ones(( N, 1))
for i in range(500):
isc = tf.where(tf.slice(history_is_correct, [0, 500-i-1], [-1, 1]), 1., 0.)
isw = tf.where(tf.slice(history_is_wrong, [0, 500-i-1], [-1, 1]), 1., 0.)
isq = tf.where(tf.slice(history_is_question, [0, 500-i-1], [-1, 1]), 0., 1.)
cc = tf.maximum(isq, isc)
 ww = tf.maximum(isq, isw)
 correct_continue *= cc
wrong_continue *= ww
correct_streak += correct_continue * cc
wrong_streak += wrong_continue * ww
correct_streak = tf.math.log1p(correct_streak)
 wrong_streak = tf.math.log1p(wrong_streak)
 user_real_choice_rate = tf.reduce_sum(tf.one_hot(tf.where(history_is_question, choice_history%4, -1), 4), axis=1)
 user_elapsed_choice_rate = tf.reduce_sum(tf.gather(choice_rate, choice_history), axis=1)
 user_choice_rate_rate = user_real_choice_rate /(user_elapsed_choice_rate + 1e-9)
 user_choice_rate_rate =(user_choice_rate_rate * question_sum + 10)/(question_sum + 10)
 wrong_user_real_choice_rate = tf.reduce_sum(tf.one_hot(tf.where(history_is_wrong, choice_history%4, -1), 4), axis=1)
 wrong_user_elapsed_choice_rate = tf.reduce_sum(tf.gather(choice_rate, choice_history)* tf.expand_dims(tf.where(history_is_wrong, 1., 0.) , axis=-1), axis=1)
 wrong_user_choice_rate_rate = wrong_user_real_choice_rate /(wrong_user_elapsed_choice_rate + 1e-9)
 wrong_user_choice_rate_rate =(wrong_user_choice_rate_rate * wrong_sum + 10)/(wrong_sum + 10)
 user_choice_rate_rate_correct = tf.reduce_sum(user_choice_rate_rate * tf.one_hot(tf.gather(self.correct_choice, content_id*4), 4), axis=1, keepdims=True)
 wrong_user_choice_rate_rate_correct = tf.reduce_sum(wrong_user_choice_rate_rate * tf.one_hot(tf.gather(self.correct_choice, content_id*4), 4), axis=1, keepdims=True)
 rate_feat = tf.concat([user_choice_rate_rate, wrong_user_choice_rate_rate, user_choice_rate_rate_correct, wrong_user_choice_rate_rate_correct], axis=1)
 frequency = tf.math.log1p(tf.cast(timestamp, "float32")/(tf.cast(order, "float32")+1e-9))
frequency = tf.expand_dims(tf.where(order > 0, frequency, 15), axis=-1)
 content_seen_sum = tf.math.log1p(tf.reduce_sum(tf.where(tf.logical_and(tf.equal(choice_history//4, tf.expand_dims(content_id, axis=1)) , choice_history > 0), 1., 0.) , axis=1, keepdims=True))
elp_mean = tf.where(question_sum > 0,(elapsed_acc_sum)/(question_sum + 1e-9), 0.657)
 acc_mean = tf.where(question_sum > 0,(correct_answer_sum)/(question_sum + 1e-9), 0.657)
 acc_mean =(acc_mean * question_sum + elp_mean * 10)/(question_sum + 10)
 exp_mean = tf.where(question_sum > 0,(explanation_sum + 5e-10)/(question_sum + 1e-9),.5)
 acc_per_elapse = tf.math.log1p(( acc_mean + 1e-9)/ tf.math.maximum(elp_mean, 0.01))
scaled_elapsed_time_mean = tf.where(question_sum>0,(scaled_elapsed_time_sum)/(question_sum + 1e-9), 0)
 choice_sum = tf.reduce_sum(tf.where(choice_history > 0, 1., 0.) , axis=1)
 day_msecs =(1000 * 60 * 60 * 24)
 day_angle =(tf.cast(timestamp_history, "float32")% day_msecs)/ day_msecs * 2 * np.pi
day_angle_cos_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.cos(day_angle), 0), axis=1)/ tf.math.maximum(choice_sum, 1e-9)
 day_angle_sin_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.sin(day_angle), 0), axis=1)/ tf.math.maximum(choice_sum, 1e-9)
 day_angle_norm = tf.math.sqrt(tf.math.square(day_angle_cos_mean)+ tf.math.square(day_angle_sin_mean))
day_angle_normed_cos = day_angle_cos_mean / tf.math.maximum(day_angle_norm, 1e-9)
 day_angle_normed_sin = day_angle_sin_mean / tf.math.maximum(day_angle_norm, 1e-9)
 week_msecs =(1000 * 60 * 60 * 24 * 7)
 week_angle =(tf.cast(timestamp_history, "float32")% week_msecs)/ week_msecs * 2 * np.pi
week_angle_cos_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.cos(week_angle), 0), axis=1)/ tf.math.maximum(choice_sum, 1e-9)
 week_angle_sin_mean = tf.reduce_sum(tf.where(choice_history > 0, tf.math.sin(week_angle), 0), axis=1)/ tf.math.maximum(choice_sum, 1e-9)
 week_angle_norm = tf.math.sqrt(tf.math.square(week_angle_cos_mean)+ tf.math.square(week_angle_sin_mean))
week_angle_normed_cos = week_angle_cos_mean / tf.math.maximum(week_angle_norm, 1e-9)
 week_angle_normed_sin = week_angle_sin_mean / tf.math.maximum(week_angle_norm, 1e-9)
 last3_timestamp = tf.where(tf.slice(choice_history, [0, 497], [-1, 3])> 0, tf.slice(timestamp_history, [0, 497], [-1, 3]), [[132324, 75973, 32387]])
 delta_last3 = tf.math.log1p(tf.cast(tf.reshape(timestamp,(-1, 1)) - last3_timestamp, "float32"))
last_correct_ts = tf.math.log1p(tf.cast(timestamp - tf.reduce_max(tf.where(tf.gather(self.choice_is_correct, choice_history), timestamp_history, 0), axis=1), "float32"))
last_wrong_ts = tf.math.log1p(tf.cast(timestamp - tf.reduce_max(tf.where(tf.gather(self.choice_is_wrong, choice_history), timestamp_history, 0), axis=1), "float32"))
last_correct_ts_rate = tf.where(timestamp > 0, last_correct_ts / tf.math.log1p(tf.cast(timestamp, "float32")) , 1)
 last_wrong_ts_rate = tf.where(timestamp > 0, last_wrong_ts / tf.math.log1p(tf.cast(timestamp, "float32")) , 1)
 new_feat = [day_angle_cos_mean, day_angle_sin_mean, day_angle_norm, day_angle_normed_cos, day_angle_normed_sin,
week_angle_cos_mean, week_angle_sin_mean, week_angle_norm, week_angle_normed_cos, week_angle_normed_sin,
last_correct_ts, last_wrong_ts, last_correct_ts_rate, last_wrong_ts_rate]
new_feat = tf.concat([tf.expand_dims(f, axis=-1)for f in new_feat] + [delta_last3], axis=1)
 questions_one_hour = tf.math.log1p(tf.cast(500 - tf.searchsorted(timestamp_history, tf.maximum(tf.expand_dims(timestamp-3600*1000, axis=1), 0)) , "float32"))
questions_one_day = tf.math.log1p(tf.cast(500 - tf.searchsorted(timestamp_history, tf.maximum(tf.expand_dims(timestamp-24*3600*1000, axis=1), 0)) , "float32"))
questions_one_week = tf.math.log1p(tf.cast(500 - tf.searchsorted(timestamp_history, tf.maximum(tf.expand_dims(timestamp-7*24*3600*1000, axis=1), 0)) , "float32"))
num_col = tf.constant([0, 2, 4, 6, 8, 10, 12, 14])
 div_col = tf.constant([1, 3, 5, 7, 9, 11, 13, 15])
 row_ixs = tf.repeat(tf.range(N), len(num_col))
num_col = tf.tile(num_col, [N])
 div_col = tf.tile(div_col, [N])
 num_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, num_col], axis=1)) ,(N, -1))
div_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, div_col], axis=1)) ,(N, -1))
mr_mean_feat = num_feat /(div_feat + 1e-9)
 mr_mean_feat =(mr_mean_feat * div_feat + self.part_mean * 10)/(div_feat + 10)
 num_col = tf.constant([1, 3, 5, 7, 9, 11, 13])
 div_col = tf.constant([15, 15, 15, 15, 15, 15, 15])
 row_ixs = tf.repeat(tf.range(N), len(num_col))
num_col = tf.tile(num_col, [N])
 div_col = tf.tile(div_col, [N])
 num_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, num_col], axis=1)) ,(N, -1))
div_feat = tf.reshape(tf.gather_nd(raw_feat, tf.stack([row_ixs, div_col], axis=1)) ,(N, -1))
mr_rate_feat = num_feat /(div_feat + 1e-9)
 mr_rate_feat =(mr_rate_feat * tf.expand_dims(tf.cast(order, "float32"), axis=1)+ self.part_rate * 10)/(tf.expand_dims(tf.cast(order, "float32"), axis=1)+ 10)
 mr_feat = tf.concat([mr_mean_feat, mr_rate_feat], axis=1)
 count_feat = tf.math.log1p(raw_feat)
 content_accuracy = tf.reshape(tf.gather(self.answer_accuracy_mean, content_id*4),(-1, 1))
harmonic_mean = 2/(1/tf.math.maximum(1e-9, tf.reshape(tf.gather_nd(mr_mean_feat, tf.stack([tf.range(N), tf.ones(N, dtype="int32")*7], axis=1)) ,(-1, 1)))+ 1/tf.math.maximum(1e-9, content_accuracy))
part_harmonic_mean = 2/(1/tf.math.maximum(1e-9, tf.reshape(tf.gather_nd(mr_mean_feat, tf.stack([tf.range(N), part], axis=1)) ,(-1, 1)))+ 1/tf.math.maximum(1e-9, content_accuracy))
new_feats = tf.concat([correct_streak, wrong_streak, rate_feat, harmonic_mean, part_harmonic_mean, frequency, content_seen_sum, acc_mean, acc_per_elapse, elp_mean, scaled_elapsed_time_mean, exp_mean, count_feat, mr_feat, questions_one_day, questions_one_hour, questions_one_week, new_feat], axis=1)
 feat = tf.concat([feat, new_feats], axis=1)
 return(user_ix, content_id, timestamp_cat, lag_time, order, choice_history, timestamp_history, delta_timestamp_history,
raw_elapsed_time_history, elapsed_time_history, explanation_history, cooltime_history, part, feat )<choose_model_class>
print('Loading data...')
 start_time = datetime.now()
train = pd.read_csv(
 TRAIN_FILE,
dtype=dtypes,
usecols=['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed'],
compression = COMPRESSION,
header=0,
skiprows=SKIP_ROWS,
engine='c' 
)
 test = pd.read_csv(
 TEST_FILE,
dtype=dtypes,
usecols=['ip', 'app', 'device', 'os', 'channel', 'click_time', 'click_id'],
compression = COMPRESSION,
header=0,
engine='c' 
)
 print('End of Loading Data: {time_taken}'.format(time_taken=datetime.now() - start_time))
predictor = Predictor(timestamp_bins,
delta_bins,
elapsedtime_bins,
choice_is_correct,
choice_is_wrong,
choice_is_question,
answer_accuracy_mean,
answer_elapsed_time_mean,
answer_elapsed_time_std,
correct_choice,
choice_rate,
part_mean,
part_rate )<drop_column>
print(len(train))
class Outliers() :
data = None
def __init__(self, data):

 self.data = data
def treat_outliers_by_iqr(self, colName, what_sd=1.5):
mean_value = np.mean(self.data[colName].values)
 sd_value = np.std(self.data[colName].values)
 max_value = mean_value + what_sd * sd_value
min_value = mean_value - what_sd * sd_value
new_data = self.data[(self.data[colName] >= min_value)&(self.data[colName] <= max_value)]
outliers = self.data[(self.data[colName] > max_value)|(self.data[colName] < min_value)]
print('%Outliers: {outlier_percentage:<8} Mean: {mean_value:<8} SD: {sd_value:<8} IQR Min: {iqr_min:<8} IQR Max: {iqr_max:<8} Min: {mini:<8} Max: {maxi:<8} Outliers: {outliers:<8}
outlier_percentage=np.around(( len(outliers)/len(self.data)) *100, decimals=2),
mean_value=np.around(mean_value, decimals=2), sd_value=np.around(sd_value, decimals=2),
iqr_min=np.around(min_value, decimals=2),
iqr_max=np.around(max_value, decimals=2),
mini=np.around(min(self.data[colName].values), decimals=2),
maxi=np.around(max(self.data[colName].values), decimals=2),
outliers=len(outliers),
app_count=len(new_data.app.unique()),
size=len(new_data)
))
return outliers, new_data
if(TREAT_OUTLIERS == True):
print('Remove Outliers...')
 start_time = datetime.now()
outs = Outliers(train)
 outliers, train = outs.treat_outliers_by_iqr('is_attributed')
 print('End of Outlier Removal: {time_taken}'.format(time_taken=datetime.now() - start_time))
questions_df = questions_df.set_index("content_id" )<define_variables>
train_size = len(train)
 train = train.append(test)
 del test
gc.collect()
warnings.simplefilter('ignore')
 feat_col = ['content_2_accuracy', 'content_3_accuracy', 'content_4_accuracy', "idf"]
raw_feat_col = ['part0_sum', 'part0_count', 'part1_sum',
'part1_count', 'part2_sum', 'part2_count', 'part3_sum', 'part3_count',
'part4_sum', 'part4_count', 'part5_sum', 'part5_count', 'part6_sum',
'part6_count', 'question_sum', 'question_count']
user_ix_last = max(user_id_map.values())+ 1
user_ix_max = tf.constant(max(user_id_map.values()), dtype="int32")
 past_df = None
i = 0
collect()
mapping_question = np.vectorize(lambda x: question_id_map[x], otypes=["int64"])
 mapping_lecture = np.vectorize(lambda x: lecture_id_map[x], otypes=["int64"])
 for(test_raw_df, sample_prediction_df)in tqdm(iter_test):
test_raw_df.loc[test_raw_df["content_type_id"] == False, "content_id"] = mapping_question(test_raw_df.loc[test_raw_df["content_type_id"] == False, "content_id"].values)
 test_raw_df.loc[test_raw_df["content_type_id"] == True, "content_id"] = mapping_lecture(test_raw_df.loc[test_raw_df["content_type_id"] == True, "content_id"].values)
 if past_df is not None:
past_df["user_answer"] = eval(test_raw_df["prior_group_responses"].values[0])
 past_df["choice_id"] = past_df["content_id"]*4 + past_df["user_answer"] *(past_df["user_answer"] >= 0)
 for user_ix, choice_id, timestamp, cid, part in zip(past_df["user_ix"], past_df["choice_id"], past_df["timestamp"], past_df["content_id"], past_df["part"]):
choices[user_ix].append(choice_id)
 timestamps[user_ix].append(timestamp)
 elapsed_times[user_ix].append(0 if cid > 13523 else -1)
 explanations[user_ix].append(0 if cid > 13523 else -1)
 orders[user_ix] += 1
if cid <= 13523:
part = int(part)
 qp_agg_df.at[user_ix, f"part{part}_count"] += 1
qp_agg_df.at[user_ix, f"part{part}_sum"] += int(choice_is_correct[choice_id])
 qp_agg_df.at[user_ix, f"question_count"] += 1
qp_agg_df.at[user_ix, f"question_sum"] += int(choice_is_correct[choice_id])
 for user_id in test_raw_df["user_id"].values:
if not user_id in user_id_map.keys() :
user_id_map[user_id] = user_ix_last
choices[user_ix_last] = []
timestamps[user_ix_last] = []
elapsed_times[user_ix_last] = []
explanations[user_ix_last] = []
orders[user_ix_last] = 0
qp_agg_df.loc[user_ix_last, :] = 0
user_ix_last += 1
get = np.vectorize(lambda x: user_id_map[x])
 test_raw_df["user_ix"] = get(test_raw_df["user_id"])
 test_users = test_raw_df["user_ix"].unique()
test_questions = test_raw_df[test_raw_df["content_type_id"] == False]["content_id"].unique()
test_raw_df = test_raw_df.merge(questions_df.loc[test_questions, ["part", 'content_2_accuracy', 'content_3_accuracy', 'content_4_accuracy', "idf"]],
on="content_id",
how="left")
 test_df = test_raw_df[test_raw_df["content_type_id"] == 0]
test_df["part"] = test_df["part"].astype(np.int32)
 test_df = test_df.merge(qp_agg_df.loc[test_users, :],
on="user_ix",
how="left")
 for user_ix, time, expl in zip(test_df["user_ix"], test_df["prior_question_elapsed_time"], test_df["prior_question_had_explanation"]):
if pd.isna(time):
continue
elapsed_times[user_ix] = [time if(t == -1 and c <= 54095)else t for t, c in zip(elapsed_times.get(user_ix, []), choices.get(user_ix, [])) ]
explanations[user_ix] = [expl if(t == -1 and c <= 54095)else t for t, c in zip(explanations.get(user_ix, []), choices.get(user_ix, [])) ]
order = []
choice_history = []
delta_timestamp_history = []
elapsed_time_history = []
explanation_history = []
for user_ix in test_df["user_ix"]:
order.append(orders.get(user_ix, 0))
choice_history.append(choices.get(user_ix, []))
delta_timestamp_history.append(timestamps.get(user_ix, []))
elapsed_time_history.append(elapsed_times.get(user_ix, []))
explanation_history.append(explanations.get(user_ix, []))
choice_history = tf.keras.preprocessing.sequence.pad_sequences(choice_history, maxlen=500, dtype="int32")
 raw_timestamp_history= tf.keras.preprocessing.sequence.pad_sequences(delta_timestamp_history, maxlen=500, value=-1, dtype="int64")
 raw_elapsed_time_history = tf.keras.preprocessing.sequence.pad_sequences(elapsed_time_history, maxlen=500, value=-1, dtype="float32")
 explanation_history = tf.keras.preprocessing.sequence.pad_sequences(explanation_history, maxlen=500, value=-1, dtype="float32")
 choice_history = tf.constant(choice_history, dtype="int32")
 raw_timestamp_history = tf.constant(raw_timestamp_history, dtype="int64")
 raw_elapsed_time_history = tf.constant(raw_elapsed_time_history, dtype="float32")
 explanation_history = tf.constant(explanation_history, dtype="float32")
 user_ix = tf.constant(test_df["user_ix"], dtype="int32")
 content_id = tf.constant(test_df["content_id"], dtype="int32")
 timestamp = tf.constant(test_df["timestamp"], dtype="int64")
 part = tf.constant(test_df["part"], dtype="int32")
 order = tf.constant(order, dtype="int32")
 feat = tf.constant(test_df[feat_col].values, dtype="float32")
 raw_feat = tf.constant(test_df[raw_feat_col].values, dtype="float32")
 start = timer()
input_ = predictor(user_ix, content_id, timestamp, order, choice_history, raw_timestamp_history, raw_elapsed_time_history, explanation_history, part, feat, raw_feat)
 print(timer() - start)
 start = timer()
nn_pred = model(*input_, training=False)
 print(timer() - start)
 test_df["answered_correctly"] =(np.squeeze(nn_pred.numpy()))
past_df = test_raw_df
env.predict(test_df.loc[test_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
<train_model>
print('Making Hour, Day and Weekday Columns...')
 start_time = datetime.now()
date_time = pd.to_datetime(train.click_time)
 train['hour'] = date_time.dt.hour.astype('uint8')
 train['day'] = date_time.dt.day.astype('uint8')
 train['wday'] = date_time.dt.dayofweek.astype('uint8')
 print('End of Making HDWd: {time_taken}'.format(time_taken=datetime.now() - start_time))
kernel_start = datetime.datetime.now()
print(f'kernel_start: {kernel_start}' )<install_modules>
print('Making number of channels by IP, Day, HOUR...')
 group_ip_dhc = train[['ip', 'day', 'hour', 'channel']].groupby(by=['ip', 'day', 'hour'])
 group_ip_dhc = group_ip_dhc[['channel']].count().reset_index().rename(index=str, columns={'channel': 'qty'})
 train = train.merge(group_ip_dhc, on=['ip', 'day', 'hour'], how='left')
 del group_ip_dhc
gc.collect
train.head()
print('End of Making channel count by IDH: {time_taken}'.format(time_taken=datetime.now() - start_time))
!pip uninstall -y cloud-tpu-client
if not IS_KAGGLE:
!pip uninstall -y tensorflow
!pip install --upgrade tensorflow==2.3.0
!pip uninstall -y tensorflow-gcs-config
!pip install --upgrade tensorflow-gcs-config==2.3.0
elif IS_KAGGLE and not USE_GPU:
!pip uninstall -y tensorflow
!pip install --upgrade tensorflow==2.3.1
print(f'TensorFlow: {tf.__version__}')
 print(client.Client )<set_options>
print('Making number of channels by IP, APP...')
 start_time = datetime.now()
group_ip_ac = train[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])
 group_ip_ac = group_ip_ac[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})
 train = train.merge(group_ip_ac, on=['ip', 'app'], how='left')
 del group_ip_ac
gc.collect()
train.head()
print('End of Making channel count by IA: {time_taken}'.format(time_taken=datetime.now() - start_time))
if IS_KAGGLE:
BASE_DIR = '/kaggle/input'
else:
BUCKET_DIR = 'gs://shieh-tpu/r3id'
BASE_DIR = '/content/drive/My Drive/r3id'
BASE_DIR_QUOTED = '"/content/drive/My Drive/r3id"'
if not IS_KAGGLE:
auth.authenticate_user()
project_id = 'shieh-tpu'
!gcloud config set project {project_id}
!gsutil ls {BUCKET_DIR}
drive.mount('/content/drive')
 if not os.path.isdir(BASE_DIR):
!mkdir {BASE_DIR_QUOTED}
!gsutil -m cp -r 'gs://shieh-tpu/r3id' "/content/drive/My Drive"
!ls -l '{BASE_DIR}'<set_options>
print('Making number of channels by IP, APP, OS...')
 start_time = datetime.now()
group_ip_aoc = train[['ip', 'app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])
 group_ip_aoc = group_ip_aoc[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})
 train = train.merge(group_ip_aoc, on=['ip', 'app', 'os'], how='left')
 del group_ip_aoc
gc.collect()
print('End of Making channel count by IAO: {time_taken}'.format(time_taken=datetime.now() - start_time))
try:
client.Client().configure_tpu_version(tf.__version__, restart_type='ifNeeded')
 if IS_KAGGLE:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
else:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
 print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<install_modules>
from sklearn.preprocessing import LabelEncoder
!pip uninstall -y datatable
!pip uninstall -y tokenizers
!pip uninstall -y transformers
if IS_KAGGLE:
!pip install '{BASE_DIR + '/r3id-packages/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl'}'
!pip install --upgrade '{BASE_DIR + '/r3id-packages/tokenizers-0.9.2-cp37-cp37m-manylinux1_x86_64.whl'}'
else:
!pip install '{BASE_DIR + '/r3id-packages/datatable-0.11.0-cp36-cp36m-manylinux2010_x86_64.whl'}'
!pip install --upgrade '{BASE_DIR + '/r3id-packages/tokenizers-0.9.2-cp36-cp36m-manylinux1_x86_64.whl'}'
!pip install --upgrade '{BASE_DIR + '/r3id-packages/transformers-3.4.0-py3-none-any.whl'}'<set_options>
print('Label encoding and fit transformation...')
 start_time = datetime.now()
train[['app', 'device', 'os', 'channel', 'hour', 'day', 'wday']].apply(LabelEncoder().fit_transform)
 print('End of Making channel count by IAO: {time_taken}'.format(time_taken=datetime.now() - start_time))
os.environ['TF_DETERMINISTIC_OPS'] = '1'
if IS_KAGGLE and USE_GPU:
gc.enable()
TFPreTrainedModel,
keras_serializable,
shape_list, 
)
 if IS_KAGGLE: 
<define_variables>
print('Splitting train and test data...')
 start_time = datetime.now()
test = train[train_size:]
print(len(test))
train = train[:train_size]
y_train = train['is_attributed'].values
print(len(train))
train.drop(['click_id', 'click_time', 'ip', 'is_attributed'], 1, inplace=True)
 print('End of Splitting Train and Test Dataset: {time_taken}'.format(time_taken=datetime.now() - start_time))
train_dt_path_or_obj = f'{BASE_DIR}/r3id-trainjay/train.jay'
unique_user_id_train_path = f'{BASE_DIR}/r3id-info/unique_user_id_train.json'
unique_question_id_train_path = f'{BASE_DIR}/r3id-info/unique_question_id_train.json'
unique_lecture_id_train_path = f'{BASE_DIR}/r3id-info/unique_lecture_id_train.json'
user_id_to_row_id_train_path = f'{BASE_DIR}/r3id-info/user_id_to_row_id_train.json'
unique_user_id_splitted_train_path = f'{BASE_DIR}/r3id-info-shieh/unique_user_id_splitted_train.json'
unique_question_id_splitted_train_path = f'{BASE_DIR}/r3id-info-shieh/unique_question_id_splitted_train.json'
unique_lecture_id_splitted_train_path = f'{BASE_DIR}/r3id-info-shieh/unique_lecture_id_splitted_train.json'
n_contents_dict_path = f'{BASE_DIR}/r3id-info-shieh/n_contents_dict.json'
question_tags_info_path = f'{BASE_DIR}/r3id-info-shieh/question_tags_info.json'
lecture_tags_info_path = f'{BASE_DIR}/r3id-info-shieh/lecture_tags_info.json'
question_part_info_path = f'{BASE_DIR}/r3id-info-shieh/question_part_info.json'
lecture_part_info_path = f'{BASE_DIR}/r3id-info-shieh/lecture_part_info.json'
correct_answer_info_path = f'{BASE_DIR}/r3id-info-shieh/correct_answer_info.json'
question_history_path = f'{BASE_DIR}/r3id-info-shieh/question_history.json'
question_history_at_training_end_path = f'{BASE_DIR}/r3id-info-shieh/question_history_at_training_end.json'
single_question_history_at_training_end_optimized_path = f'{BASE_DIR}/r3id-info-shieh/single_question_history_at_training_end.json'
user_performance_hdf5_path = f'{BASE_DIR}/user-performance/user_performance.hdf5'
question_history_for_valid_path = f'{BASE_DIR}/r3id-info-shieh/question_history_fold_1.json'
question_history_at_training_end_for_valid_path = f'{BASE_DIR}/r3id-info-shieh/question_history_at_training_end_fold_1.json'
single_question_history_at_training_end_optimized_for_valid_path = f'{BASE_DIR}/r3id-info-shieh/single_question_history_at_training_end_fold_1.json'
valid_info_paths = [
f'{BASE_DIR}/r3id-info-shieh/valid_info.json',
f'{BASE_DIR}/r3id-info-shieh/valid_info_fold_1.json',
f'{BASE_DIR}/r3id-info-shieh/valid_info_fold_2.json',
f'{BASE_DIR}/r3id-info-shieh/valid_info_fold_3.json',
]
train_valid_split_indices_paths = [
f'{BASE_DIR}/r3id-info-shieh/train_valid_split_indices.json',
f'{BASE_DIR}/r3id-info-shieh/train_valid_split_indices_fold_1.json',
f'{BASE_DIR}/r3id-info-shieh/train_valid_split_indices_fold_2.json',
f'{BASE_DIR}/r3id-info-shieh/train_valid_split_indices_fold_3.json',
]
if tpu is None:
TFREC_DIR = BASE_DIR
train_tfrec_dir = f'{BASE_DIR}/ednet-tfrecords-sequential-more'
valid_tfrec_dir = f'{BASE_DIR}/r3id-tfrecords-valid-more'
elif IS_KAGGLE:
train_tfrec_dir = KaggleDatasets().get_gcs_path('ednet-tfrecords-sequential-more')
 valid_tfrec_dir = KaggleDatasets().get_gcs_path('r3id-tfrecords-valid-more')
 print(f'train_tfrec_dir from Kaggle = {train_tfrec_dir}
')
 print(f'valid_tfrec_di from Kaggle = {valid_tfrec_dir}
')
 !gsutil ls $train_tfrec_dir
!gsutil ls $valid_tfrec_dir
else:
TFREC_DIR = BUCKET_DIR
train_tfrec_dir = f'{BUCKET_DIR}/ednet-tfrecords-sequential-more'
valid_tfrec_dir = f'{BUCKET_DIR}/r3id-tfrecords-valid-more'
train_dt = dt.fread(train_dt_path_or_obj )<define_variables>
from keras.layers import Input, Embedding, Dense, Flatten, Dropout, concatenate
from keras.layers import BatchNormalization, SpatialDropout1D, GRU
from keras.callbacks import Callback
from keras.models import Model
from keras.optimizers import Adam
MODEL_TYPE = 'ed'
MODEL_SIZE = 'b-2'
MODEL_DESC = 'raw'
ACTIVATION = 'relu'
USE_PRE_CLASSIFIER = True
USE_SOFTMAX = False
USE_USER_ANSWER = False
USE_USER_ANSWER_LOSS = False
USE_CORRECT_ANSWER_FOR_ENCODER = False
USE_CORRECT_ANSWER_FOR_DECODER = False
USE_ABS_POS = False
USE_TASK_CONTAINER_POS = False
SHARE_POS_EMBEDDING = True
USE_TAGS = False
USE_PART = False
USE_PRIOR_EXPLANATION = False
USE_PRIOR_QUESTION_ELAPSED_TIME_INPUT = False
USE_LAG_TIME = False
USE_LAG_TIME_FOR_ENCODER = False
USE_USER_LEVEL_AGGREGATED_HISTORICAL_INFO = False
USE_PART_AGGREGATED_HISTORICAL_INFO = False
USE_CORRECT_ANSWER_AGGREGATED_HISTORICAL_INFO = False
USE_QUESTION_LEVEL_AGGREGATED_HISTORICAL_INFO = False
ALLOW_BUNDLE_ATTEN = False
GENERATIVE = False
VALID_FOLD = 0
WINDOW_SIZE = 128
LOSS_WEIGHT_WINDOW_SIZE = None
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
if not tpu:
BATCH_SIZE = 8 * BATCH_SIZE
PRED_BATCH_SIZE = 256 * strategy.num_replicas_in_sync
if tpu:
assert BATCH_SIZE // strategy.num_replicas_in_sync != WINDOW_SIZE
assert PRED_BATCH_SIZE // strategy.num_replicas_in_sync != WINDOW_SIZE
N_EPOCHS = 2
STEPS_PER_CALL = 1000
MAX_N_CONTENTS_PER_USER_FOR_SAMPLING_PROB = 512
LR = 1e-4
END_LR = 1e-4
WARMUP_STEPS = 5000
DETERMINISTIC = False
if DETERMINISTIC:
SEED = 2021
N_PARALLEL_READS = None
N_PARALLEL_CALLS = None
SHUFFLE_BUFFER_SIZE = 1
else:
SEED = None
N_PARALLEL_READS = 16
N_PARALLEL_CALLS = tf.data.experimental.AUTOTUNE
SHUFFLE_BUFFER_SIZE = 4096
MAX_TRAIN_ITER_STEPS = None
MAX_VALID_ITER_STEPS = None
PRINTING_STEPS = 1000
CKPT_DIR = f'{MODEL_TYPE}-{MODEL_SIZE}-{MODEL_DESC}/'<define_variables>
max_app = np.max([train['app'].max() , test['app'].max() ])+ 1
max_channel = np.max([train['channel'].max() , test['channel'].max() ])+ 1
max_device = np.max([train['device'].max() , test['device'].max() ])+ 1
max_os = np.max([train['os'].max() , test['os'].max() ])+ 1
max_hour = np.max([train['hour'].max() , test['hour'].max() ])+ 1
max_day = np.max([train['day'].max() , test['day'].max() ])+ 1
max_wday = np.max([train['wday'].max() , test['wday'].max() ])+ 1
max_qty = np.max([train['qty'].max() , test['qty'].max() ])+ 1
max_c1 = np.max([train['ip_app_count'].max() , test['ip_app_count'].max() ])+ 1
max_c2 = np.max([train['ip_app_os_count'].max() , test['ip_app_os_count'].max() ])+ 1
DTYPE = tf.float32
tf.keras.backend.set_floatx('float32')
 TRAIN = True
VALID = False
PRED = False
RESUME_TRAINING = False
N_FILES = 6
SUBMISSION = False
if IS_KAGGLE:
N_FILES = len(os.listdir('/kaggle/input/riiid-test-answer-prediction'))
SUBMISSION =(N_FILES != 6)
 else:
PRED = False
if SUBMISSION:
TRAIN = False
VALID = False
PRED = True
if not TRAIN:
RESUME_TRAINING = False
DEBUG = True
PROBE = False
CKPT_TRAIN_PATH = None
CKPT_PRED_PATH = None
if CKPT_TRAIN_PATH is None:
if IS_KAGGLE:
CKPT_TRAIN_PATH = './'
else:
CKPT_TRAIN_PATH = f'{BUCKET_DIR}/r3id-ckpts/{CKPT_DIR}'
if TRAIN and not RESUME_TRAINING:
_state = !gsutil -q stat {CKPT_TRAIN_PATH}*; echo $?
already_existed = 1 - int(_state[0])
 assert not already_existed
if CKPT_PRED_PATH is None:
if IS_KAGGLE:
CKPT_PRED_PATH = f'{BASE_DIR}/{CKPT_DIR}'
else:
CKPT_PRED_PATH = f'{BUCKET_DIR}/r3id-ckpts/{CKPT_DIR}'<define_variables>
print('Preparing dataset for training...')
 start_time = datetime.now()
def get_keras_data(dataset):
X = {
'app': np.array(dataset.app),
'channel': np.array(dataset.channel),
'device': np.array(dataset.device),
'os': np.array(dataset.os),
'hour': np.array(dataset.hour),
'day': np.array(dataset.day),
'wday': np.array(dataset.wday),
'qty': np.array(dataset.qty),
'c1': np.array(dataset.ip_app_count),
'c2': np.array(dataset.ip_app_os_count)
 }
return X
train = get_keras_data(train)
 in_app = Input(shape=[1], name='app')
 emb_app = Embedding(max_app, EMBEDDING_N )(in_app)
 in_channel = Input(shape=[1], name='channel')
 emb_channel = Embedding(max_channel, EMBEDDING_N )(in_channel)
 in_device = Input(shape=[1], name='device')
 emb_device = Embedding(max_device, EMBEDDING_N )(in_device)
 in_os = Input(shape=[1], name='os')
 emb_os = Embedding(max_os, EMBEDDING_N )(in_os)
 in_hour = Input(shape=[1], name='hour')
 emb_hour = Embedding(max_hour, EMBEDDING_N )(in_hour)
 in_day = Input(shape=[1], name='day')
 emb_day = Embedding(max_day, EMBEDDING_N )(in_day)
 in_wday = Input(shape=[1], name='wday')
 emb_wday = Embedding(max_wday, EMBEDDING_N )(in_wday)
 in_qty = Input(shape=[1], name='qty')
 emb_qty = Embedding(max_qty, EMBEDDING_N )(in_qty)
 in_c1 = Input(shape=[1], name='c1')
 emb_c1 = Embedding(max_c1, EMBEDDING_N )(in_c1)
 in_c2 = Input(shape=[1], name='c2')
 emb_c2 = Embedding(max_c2, EMBEDDING_N )(in_c2)
 print('End of Dataset Preparation: {time_taken}'.format(time_taken=datetime.now() - start_time))
PAD_TOKEN = -2
START_TOKEN = -3
END_TOKEN = -4
MASK_TOKEN = -5
PAD_ID = 0
START_ID = 1
END_ID = 2
MASK_ID = 3
RESPONSE_LECTURE_TOKEN = -1
RESPONSE_FALSE_TOKEN = 0
RESPONSE_TRUE_TOKEN = 1
ANSWER_LECTURE_TOKEN = -1
ANSWER_0_TOKEN = 0
ANSWER_1_TOKEN = 1
ANSWER_2_TOKEN = 2
ANSWER_3_TOKEN = 3
RESPONSE_LECTURE_ID = 4
RESPONSE_FALSE_ID = 5
RESPONSE_TRUE_ID = 6
ANSWER_LECTURE_ID = 4
ANSWER_0_ID = 5
ANSWER_1_ID = 6
ANSWER_2_ID = 7
ANSWER_3_ID = 8
NON_TARGET_ID = -100
TAG_VOCAB_SIZE = 189
PART_VOCAB_SIZE = 9
PRIOR_EXPLANATION_VOCAB_SIZE = 3
N_TAGS_PER_CONTENT = 6
MAX_TRAIN_HISTORY_LEN = 17917
MAX_HISTORY_LEN = 20480
MAX_TASK_CONTAINER_ID = 10000
N_AGGREGATED_QUESTION_SCALING_FACTOR = 1000<define_variables>
print('Create RNN Layers...')
 start_time = datetime.now()
rnn_layer_1 = GRU(16 )(emb_app)
 rnn_layer_2 = GRU(8 )(emb_channel)
 rnn_layer_3 = GRU(8 )(emb_device)
 rnn_layer_4 = GRU(8 )(emb_os)
 fe = concatenate([
Flatten()(emb_app),
Flatten()(emb_channel),
Flatten()(emb_device),
Flatten()(emb_os),
Flatten()(emb_hour),
Flatten()(emb_day),
Flatten()(emb_wday),
Flatten()(emb_qty),
Flatten()(emb_c1),
Flatten()(emb_c2),
rnn_layer_1,
rnn_layer_2,
rnn_layer_3,
rnn_layer_4
])
 print('End of RNN Layer Creation: {time_taken}'.format(time_taken=datetime.now() - start_time))
MAX_PRED_TIME_QUESTION_BUNDLE_LEN = 10<categorify>
print('Building TF architecture...')
 start_time = datetime.now()
x = Dropout(DROPOUT_1 )(Dense(DENSE_N, activation='relu' )(fe))
x = Dropout(DROPOUT_1 )(Dense(512, activation='relu' )(x))
x = Dropout(DROPOUT_2 )(Dense(256, activation='relu' )(fe))
x = Dropout(DROPOUT_2 )(Dense(128, activation='relu' )(x))
outp = Dense(1,activation='sigmoid' )(x)
 model = Model(inputs=[in_app, in_channel, in_device, in_os,
in_hour, in_day, in_wday, in_qty, in_c1, in_c2], outputs=outp)
 print('End of Building TF Architecture: {time_taken}'.format(time_taken=datetime.now() - start_time))
def convert_valid_info(valid_info):
data = {}
user_ids = list(valid_info.keys())
 for user_id in user_ids:
user_valid_info = valid_info[user_id]
user_valid_info['bundle_info']['block_starting_index_dict'] = {int(k): v for k, v in user_valid_info['bundle_info']['block_starting_index_dict'].items() }
user_valid_info['bundle_info']['bundle_starting_index_dict'] = {int(k): v for k, v in user_valid_info['bundle_info']['bundle_starting_index_dict'].items() }
data[int(user_id)] = user_valid_info
del valid_info[user_id]
return data
def load_data(path):
if path.endswith('.json'):
with open(path, 'r', encoding='UTF-8')as fp:
data = json.load(fp)
 if path.endswith('valid_info.json')or 'valid_info_fold_' in path:
data = convert_valid_info(data)
 elif path.endswith('n_contents_dict.json'):
data = {int(k): v for k, v in data.items() }
elif path.endswith('user_id_to_row_id_train.json')or 'train_valid_split_indices' in path:
data = {int(k): v for k, v in data.items() }
elif path.endswith('question_tags_info.json')or path.endswith('lecture_tags_info.json'):
data = {int(k): v for k, v in data.items() }
elif path.endswith('question_part_info.json')or path.endswith('lecture_part_info.json'):
data = {int(k): v for k, v in data.items() }
elif path.endswith('correct_answer_info.json'):
data = {int(k): v for k, v in data.items() }
elif path.endswith('question_history.json'):
pass
elif path.endswith('question_history_at_training_end.json'):
pass
elif path.endswith('single_question_history_at_training_end.json'):
pass
return data
elif path.endswith('train.jay'):
train_dt = dt.fread(train_dt_path_or_obj)
 return train_dt
else:
raise ValueError('The path provided is not the one needed to create an instance of `Valid_Manager`!' )<load_from_csv>
exp_decay = lambda init, fin, steps:(init/fin)**(1/(steps -1)) - 1
questioin_df = pd.read_csv(f'{BASE_DIR}/riiid-test-answer-prediction/questions.csv')
 lecture_df = pd.read_csv(f'{BASE_DIR}/riiid-test-answer-prediction/lectures.csv')
 question_ids = questioin_df['question_id'].tolist()
lecture_ids = lecture_df['lecture_id'].tolist()
question_ids = sorted(question_ids)
 lecture_ids = sorted(lecture_ids)
 assert question_ids == sorted(question_ids)
 assert lecture_ids == sorted(lecture_ids)
 special_vocab = [PAD_TOKEN, START_TOKEN, END_TOKEN, MASK_TOKEN]
question_vocab = question_ids
lecture_vocab = lecture_ids
content_vocab = special_vocab + question_vocab + lecture_vocab
input_ids = tf.range(len(special_vocab), len(special_vocab)+ len(question_vocab))
initializer = tf.lookup.KeyValueTensorInitializer(tf.constant(question_ids, dtype=tf.int32), input_ids)
 question_id_to_input_id_table = tf.lookup.StaticHashTable(
 initializer, default_value=tf.int32.limits[1], name=None 
)
 input_ids = tf.range(len(special_vocab)+ len(question_vocab), len(content_vocab))
initializer = tf.lookup.KeyValueTensorInitializer(tf.constant(lecture_ids, dtype=tf.int32), input_ids)
 lecture_id_to_input_id_table = tf.lookup.StaticHashTable(
 initializer, default_value=tf.int32.limits[1], name=None 
)
 response_vocab = [PAD_TOKEN, START_TOKEN, END_TOKEN, MASK_TOKEN, RESPONSE_LECTURE_TOKEN, RESPONSE_TRUE_TOKEN, RESPONSE_FALSE_TOKEN]
answer_vocab = [
PAD_TOKEN, START_TOKEN, END_TOKEN, MASK_TOKEN,
ANSWER_LECTURE_TOKEN,
ANSWER_0_TOKEN, ANSWER_1_TOKEN, ANSWER_2_TOKEN, ANSWER_3_TOKEN
]
CONTENT_VOCAB_SIZE = len(content_vocab)
 RESPONSE_VOCAB_SIZE = len(response_vocab)
 ANSWER_VOCAB_SIZE = len(answer_vocab)
 question_tags_info = load_data(question_tags_info_path)
 lecture_tags_info = load_data(lecture_tags_info_path)
 correct_answer_info = load_data(correct_answer_info_path)
 question_part_info = load_data(question_part_info_path)
 lecture_part_info = load_data(lecture_part_info_path)
 tags_database = []
part_database = []
correct_answer_table = []
for idx in range(len(content_vocab)) :
if idx < len(special_vocab):
tags_database.append([-1] * N_TAGS_PER_CONTENT)
 part_database.append(-1)
 correct_answer_table.append(idx)
 elif idx < len(special_vocab)+ len(question_vocab):
index_in_questions_ids = idx - len(special_vocab)
 question_id = question_ids[index_in_questions_ids]
tags_database.append(question_tags_info[question_id])
 part_database.append(question_part_info[question_id])
 correct_answer_table.append(correct_answer_info[question_id] + ANSWER_0_ID)
 elif idx < len(content_vocab):
index_in_lecture_ids = idx -(len(special_vocab)+ len(question_vocab))
lecture_id = lecture_ids[index_in_lecture_ids]
tags_database.append(lecture_tags_info[lecture_id])
 part_database.append(lecture_part_info[lecture_id])
 correct_answer_table.append(ANSWER_LECTURE_ID)
 c_inputs_ids_to_tags = tf.constant(tags_database, dtype=tf.int32)
 c_inputs_ids_to_part = tf.constant(part_database, dtype=tf.int32)
 c_inputs_ids_to_correct_answer_id = tf.constant(correct_answer_table, dtype=tf.int32)
 question_id_to_input_id_dict = {
k: v for k, v in zip(question_ids, range(len(special_vocab), len(special_vocab)+ len(question_vocab)))
 }
c_inputs_ids_to_part_dict = {k: v for k, v in enumerate(part_database)}
c_inputs_ids_to_correct_answer_id_dict = {k: v for k, v in enumerate(correct_answer_table)}<define_variables>
steps = int(len(train)/ BATCH_SIZE)* EPOCHS
lr_init, lr_fin = LEARNING_RATE_INIT, LEARNING_RATE_END
lr_decay = exp_decay(lr_init, lr_fin, steps )
if not IS_KAGGLE:
train_tfrec_fns = [f'EdNet-user-history-{idx}.tfrecord' for idx in range(41)]
train_tfrec_fns = sorted(train_tfrec_fns, key=lambda x: int(x.replace('EdNet-user-history-', '' ).replace('.tfrecord', '')))
 if tpu is None or IS_KAGGLE:
train_tfrec_paths = [os.path.join(train_tfrec_dir, fn)for fn in train_tfrec_fns]
else:
train_tfrec_paths = [train_tfrec_dir + f'/{fn}' for fn in train_tfrec_fns]
train_tfrec_paths<define_variables>
print('Creating the model...')
 start_time = datetime.now()
optimizer_adam = Adam(lr=0.001, decay=lr_decay)
 model.compile(loss='binary_crossentropy', optimizer=optimizer_adam, metrics=['accuracy'])
 model.summary()
print('End of Creating the Model: {time_taken}'.format(time_taken=datetime.now() - start_time))
if not IS_KAGGLE:
valid_tfrec_fns = [
'EdNet-user-history-with-valid-block-pos.tfrecord',
]
if tpu is None:
valid_tfrec_paths = [os.path.join(valid_tfrec_dir, fn)for fn in valid_tfrec_fns]
else:
valid_tfrec_paths = [valid_tfrec_dir + f'/{fn}' for fn in valid_tfrec_fns]
valid_tfrec_paths<data_type_conversions>
print('Training...')
 start_time = datetime.now()
class_weight = {0:0.01, 1:0.99}
model.fit(train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, class_weight=class_weight, shuffle=True, verbose=1)
 del train, y_train
gc.collect()
print('End of Training: {time_taken}'.format(time_taken=datetime.now() - start_time))
train_raw_features = {
'user_id': tf.io.FixedLenFeature([], dtype=tf.int64),
'row_id': tf.io.RaggedFeature(value_key='row_id', dtype=tf.int64),
'timestamp': tf.io.RaggedFeature(value_key='timestamp', dtype=tf.int64),
'content_id': tf.io.RaggedFeature(value_key='content_id', dtype=tf.int64),
'content_type_id': tf.io.RaggedFeature(value_key='content_type_id', dtype=tf.int64),
'task_container_id': tf.io.RaggedFeature(value_key='task_container_id', dtype=tf.int64),
'user_answer': tf.io.RaggedFeature(value_key='user_answer', dtype=tf.int64),
'answered_correctly': tf.io.RaggedFeature(value_key='answered_correctly', dtype=tf.int64),
'prior_question_elapsed_time': tf.io.RaggedFeature(value_key='prior_question_elapsed_time', dtype=tf.float32),
'prior_question_had_explanation': tf.io.RaggedFeature(value_key='prior_question_had_explanation', dtype=tf.int64),
'n_prev_seen': tf.io.RaggedFeature(value_key='n_prev_seen', dtype=tf.int64),
'n_prev_correctness': tf.io.RaggedFeature(value_key='n_prev_correctness', dtype=tf.int64),
}
def parse_train_example(example):

 _parsed = tf.io.parse_single_example(example, train_raw_features)
 parsed = {}
parsed['user_id'] = _parsed['user_id']
parsed['seq_len'] = tf.reduce_sum(tf.ones_like(_parsed['row_id'], dtype=tf.int32))
parsed['prev_seq_len'] = parsed['seq_len']
parsed['start'] = tf.constant(0, dtype=tf.int32)
 parsed['end'] = parsed['seq_len'] - 1
for k in train_raw_features:
data = _parsed[k]
if k not in ['row_id', 'user_id', 'timestamp', 'prior_question_elapsed_time']:
data = tf.cast(data, dtype=tf.int32)
 elif k == 'prior_question_elapsed_time':
data = tf.cast(data, dtype=DTYPE)
 if k != 'user_id':
parsed[k] = data
parsed['shifted_answered_correctly'] = tf.concat([[START_TOKEN], parsed['answered_correctly']], axis=0)
 parsed['shifted_user_answer'] = tf.concat([[START_TOKEN], parsed['user_answer']], axis=0)
 pad_mask = tf.cast(parsed['row_id'] == PAD_TOKEN, dtype=tf.int32)
 pred_time_mask = tf.zeros_like(parsed['timestamp'], dtype=tf.int32)
 pred_time_mask = pred_time_mask *(1 - pad_mask)+(PAD_TOKEN)* pad_mask
parsed['pred_time_mask'] = pred_time_mask
parsed['abs_pos'] = tf.range(parsed['seq_len'], dtype=tf.int32)
 parsed['shifted_abs_pos'] = tf.concat([[START_TOKEN], parsed['abs_pos'][:-1]], axis=0)
 diff_t = parsed['timestamp'] - tf.concat([[1], parsed['timestamp'][:-1]], axis=0)
 bundle_start_mask = tf.cast(diff_t != 0, dtype=tf.int32)
 jump_t = tf.boolean_mask(diff_t, bundle_start_mask > 0)
 task_container_id_at_jumps = tf.boolean_mask(parsed['task_container_id'], bundle_start_mask > 0)
 task_containder_id_to_jump_t = tf.scatter_nd(
 indices=task_container_id_at_jumps[:, tf.newaxis], updates=jump_t, shape=tf.math.reduce_max(task_container_id_at_jumps + 1)[tf.newaxis], name=None 
)
 lag_time = tf.gather(params=task_containder_id_to_jump_t , indices=parsed['task_container_id'])
 lag_time = tf.math.maximum(tf.cast(0, dtype=tf.int64), lag_time)
 _mask = tf.cast(tf.math.logical_and(lag_time == 0, parsed['timestamp'] > 0), dtype=tf.int64)
 lag_time = lag_time *(1 - _mask)+ 1000 * _mask
parsed['lag_time'] = lag_time
return parsed
train_features_with_valid_info = {
'user_id': tf.io.FixedLenFeature([], dtype=tf.int64),
'row_id': tf.io.RaggedFeature(value_key='row_id', dtype=tf.int64),
'timestamp': tf.io.RaggedFeature(value_key='timestamp', dtype=tf.int64),
'content_id': tf.io.RaggedFeature(value_key='content_id', dtype=tf.int64),
'content_type_id': tf.io.RaggedFeature(value_key='content_type_id', dtype=tf.int64),
'task_container_id': tf.io.RaggedFeature(value_key='task_container_id', dtype=tf.int64),
'user_answer': tf.io.RaggedFeature(value_key='user_answer', dtype=tf.int64),
'answered_correctly': tf.io.RaggedFeature(value_key='answered_correctly', dtype=tf.int64),
'prior_question_elapsed_time': tf.io.RaggedFeature(value_key='prior_question_elapsed_time', dtype=tf.float32),
'prior_question_had_explanation': tf.io.RaggedFeature(value_key='prior_question_had_explanation', dtype=tf.int64),
'n_prev_seen': tf.io.RaggedFeature(value_key='n_prev_seen', dtype=tf.int64),
'n_prev_correctness': tf.io.RaggedFeature(value_key='n_prev_correctness', dtype=tf.int64),
'n_valid_blocks': tf.io.FixedLenFeature([], dtype=tf.int64),
'valid_blocks_start_pos': tf.io.RaggedFeature(value_key='valid_blocks_start_pos', dtype=tf.int64),
'valid_blocks_end_pos': tf.io.RaggedFeature(value_key='valid_blocks_end_pos', dtype=tf.int64),
}
def parse_train_example_with_valid_info(example):
_parsed = tf.io.parse_single_example(example, train_features_with_valid_info)
 parsed = {}
parsed['user_id'] = _parsed['user_id']
parsed['seq_len'] = tf.reduce_sum(tf.ones_like(_parsed['row_id'], dtype=tf.int32))
parsed['prev_seq_len'] = parsed['seq_len']
parsed['start'] = tf.constant(0, dtype=tf.int32)
 parsed['end'] = parsed['seq_len'] - 1
for k in train_features_with_valid_info:
data = _parsed[k]
if k not in ['row_id', 'user_id', 'timestamp', 'prior_question_elapsed_time']:
data = tf.cast(data, dtype=tf.int32)
 elif k == 'prior_question_elapsed_time':
data = tf.cast(data, dtype=DTYPE)
 if k != 'user_id':
parsed[k] = data
parsed['shifted_answered_correctly'] = tf.concat([[START_TOKEN], parsed['answered_correctly']], axis=0)
 parsed['shifted_user_answer'] = tf.concat([[START_TOKEN], parsed['user_answer']], axis=0)
 pad_mask = tf.cast(parsed['row_id'] == PAD_TOKEN, dtype=tf.int32)
 pred_time_mask = tf.zeros_like(parsed['timestamp'], dtype=tf.int32)
 pred_time_mask = pred_time_mask *(1 - pad_mask)+(PAD_TOKEN)* pad_mask
parsed['pred_time_mask'] = pred_time_mask
parsed['abs_pos'] = tf.range(parsed['seq_len'], dtype=tf.int32)
 parsed['shifted_abs_pos'] = tf.concat([[START_TOKEN], parsed['abs_pos'][:-1]], axis=0)
 diff_t = parsed['timestamp'] - tf.concat([[1], parsed['timestamp'][:-1]], axis=0)
 bundle_start_mask = tf.cast(diff_t != 0, dtype=tf.int32)
 jump_t = tf.boolean_mask(diff_t, bundle_start_mask > 0)
 task_container_id_at_jumps = tf.boolean_mask(parsed['task_container_id'], bundle_start_mask > 0)
 task_containder_id_to_jump_t = tf.scatter_nd(
 indices=task_container_id_at_jumps[:, tf.newaxis], updates=jump_t, shape=tf.math.reduce_max(task_container_id_at_jumps + 1)[tf.newaxis], name=None 
)
 lag_time = tf.gather(params=task_containder_id_to_jump_t , indices=parsed['task_container_id'])
 lag_time = tf.math.maximum(tf.cast(0, dtype=tf.int64), lag_time)
 _mask = tf.cast(tf.math.logical_and(lag_time == 0, parsed['timestamp'] > 0), dtype=tf.int64)
 lag_time = lag_time *(1 - _mask)+ 1000 * _mask
parsed['lag_time'] = lag_time
return parsed<create_dataframe>
print('Preparing test dataset...')
 start_time = datetime.now()
submit = pd.DataFrame()
submit['click_id'] = test['click_id'].astype('int')
 test.drop(['click_id', 'click_time', 'ip', 'is_attributed'], 1, inplace=True)
 test = get_keras_data(test)
 print('End of Test Dataset: {time_taken}'.format(time_taken=datetime.now() - start_time))
if not IS_KAGGLE:
train_raw_ds = tf.data.TFRecordDataset(train_tfrec_paths, num_parallel_reads=1)
 train_raw_ds = train_raw_ds.map(parse_train_example, num_parallel_calls=1, deterministic=True)
 for x in train_raw_ds.take(1):
print(x )<create_dataframe>
print('Preparing predicted output...')
 start_time = datetime.now()
submit['is_attributed'] = model.predict(test, batch_size=BATCH_SIZE, verbose=2)
 del test
gc.collect()
submit.to_csv('output.csv', index=False)
 print('End of Predicted Output: {time_taken}'.format(time_taken=datetime.now() - start_time))
if not IS_KAGGLE:
valid_raw_ds = tf.data.TFRecordDataset(valid_tfrec_paths, num_parallel_reads=1)
 valid_raw_ds = valid_raw_ds.map(parse_train_example_with_valid_info, num_parallel_calls=1, deterministic=True)
 for x in valid_raw_ds.take(1):
print(x )<split>
print('The END: {time_taken}'.format(time_taken=datetime.now() - init_time))
<string_transform><EOS> 
if(IS_DEV == False):
model.save_weights('.. /output/weights.h5')
 model.save('.. /output/model.h5' )