Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
<SOS> metric: AUC Kaggle data source: talkingdata-adtracking-fraud-detection-challenge<categorify>
dtypes = {
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
path = '.. /input/'
def diff_timestamp(df, drop_na = False):
df['click_time'] = pd.to_datetime(df['click_time'])
 df['diff'] = df.groupby('ip' ).click_time.diff().dt.total_seconds()
if drop_na == True:
nulls = df['diff'].isnull()
df.drop(nulls[nulls == True].index, inplace = True)
 return df
def handleClickHour_raul(df):
df['click_hour']= pd.to_datetime(df['click_time'] ).dt.hour.astype('uint8')
 df = df.drop(['click_time'], axis=1)
 return df
def get_causal_attention_mask(nd, ns, dtype, only_before):

 i = tf.range(nd)[:, tf.newaxis]
j = tf.range(ns)
 m = i >=(j - ns + nd)+ tf.cast(only_before, dtype=tf.int32)
 return tf.cast(m, dtype)
 def get_attention_mask_from_timestamp_batch(timestamp_tensors, dtype, only_before):

 t = timestamp_tensors
batch_size = tf.math.reduce_sum(tf.ones_like(t[:, :1], dtype=tf.int32))
seq_len = tf.math.reduce_sum(tf.ones_like(t[:1, :], dtype=tf.int32))
x = tf.broadcast_to(t[:, :, tf.newaxis], shape=[batch_size, seq_len, seq_len])
 y = tf.broadcast_to(t[:, tf.newaxis, :], shape=[batch_size, seq_len, seq_len])+ tf.cast(only_before, dtype=tf.int64)
 m = x >= y
return tf.cast(m, dtype )<categorify>
train_columns = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']
start_time = time.time()
df_train_30m = pd.read_csv(path + 'train.csv', dtype=dtypes, skiprows=range(1,163333333),
nrows=35000000, usecols=train_columns)
 print('Loaded df_train_30m with {:f} seconds'.format(time.time() - start_time))
if not IS_KAGGLE:
print(get_causal_attention_mask(3, 3, tf.int32, only_before=tf.constant(False)))
 print(get_causal_attention_mask(3, 3, tf.int32, only_before=tf.constant(True)))
 timestamp_tensors = tf.constant([[0, 0, 1, 2, 2, 2, 3, 3], [0, 1, 1, 2, 2, 3, 3, 3]], dtype=tf.int64)
 print(get_attention_mask_from_timestamp_batch(timestamp_tensors, tf.int32, only_before=tf.constant(False)))
 print(get_attention_mask_from_timestamp_batch(timestamp_tensors, tf.int32, only_before=tf.constant(True)) )<categorify>
start_time = time.time()
df_test = pd.read_csv(path + 'test.csv', dtype=dtypes)
 print('Loaded df_test in {:.2f} seconds'.format(round(time.time() - start_time, 3)))
 train_record_index = df_train_30m.shape[0]
df_train_30m = diff_timestamp(df_train_30m)
 df_train_30m = handleClickHour_raul(df_train_30m)
 df_test = diff_timestamp(df_test)
 df_test = handleClickHour_raul(df_test)
 gc.collect() ;
print('ClickTime data correctly handled.')
 df_submit = pd.DataFrame()
df_submit['click_id'] = df_test['click_id']
Learning_Y = df_train_30m['is_attributed']
print('Training target correctly extracted.')
 df_test = df_test.drop(['click_id'], axis=1)
 df_train_30m = df_train_30m.drop(['is_attributed'], axis=1)
 gc.collect() ;
df_merge = pd.concat([df_train_30m, df_test])
 del df_train_30m, df_test
gc.collect() ;
print('Data was correctly concatenated' )
@tf.function
def get_attention_masks(batch, training, encoder_decoder, generative, allow_bundle_atten):
pad_mask = tf.cast(( batch['content_type_id'] == PAD_TOKEN), dtype=tf.int32)
 question_mask = tf.cast(( batch['content_type_id'] == 0), dtype=tf.int32)
 pred_time_mask = batch['pred_time_mask'] * tf.cast(batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 content_ids = batch['content_id']
_pad_mask = tf.cast(( batch['timestamp'] == PAD_TOKEN), dtype=tf.int32)
 seq_len = tf.math.reduce_sum(tf.ones_like(content_ids[0, :], dtype=tf.int32))
non_pad_mask = 1 - pad_mask
causal_attention_mask = get_causal_attention_mask(nd=seq_len, ns=seq_len, dtype=tf.int32, only_before=tf.constant(False))
timestamp_attention_mask = get_attention_mask_from_timestamp_batch(batch['timestamp'], dtype=tf.int32, only_before=tf.constant(False))
timestamp_attention_mask_only_before = get_attention_mask_from_timestamp_batch(batch['timestamp'], dtype=tf.int32, only_before=tf.constant(True))
if tf.cast(allow_bundle_atten, dtype=tf.bool):
c_mask = non_pad_mask[:, tf.newaxis, :] * timestamp_attention_mask
else:
c_mask = non_pad_mask[:, tf.newaxis, :] * causal_attention_mask[tf.newaxis, :, :]
r_mask = c_mask
if tf.cast(encoder_decoder, dtype=tf.bool):
d_mask = non_pad_mask[:, tf.newaxis, :] * causal_attention_mask[tf.newaxis, :, :]
if(1 - training)*(1 - generative)== 1:
pred_time_question_mask = pred_time_mask * question_mask
d_mask = d_mask * tf.math.maximum(( 1 - pred_time_question_mask)[:, tf.newaxis, :], tf.eye(seq_len, dtype=tf.int32)[tf.newaxis, :, :])
 r_mask = d_mask
c_mask = c_mask * tf.math.maximum(( 1 - pred_time_question_mask)[:, tf.newaxis, :], tf.eye(seq_len, dtype=tf.int32)[tf.newaxis, :, :])
 else:
r_mask = d_mask
r_c_mask = c_mask
c_r_mask = non_pad_mask[:, tf.newaxis, :] * timestamp_attention_mask_only_before
return c_mask, r_mask, r_c_mask, c_r_mask
def add_input_ids_and_targets(batch, training, generative, use_abs_pos):

 content_ids = batch['content_id']
content_type_ids = batch['content_type_id']
answered_correctly = batch['answered_correctly']
user_answer = batch['user_answer']
question_mask = tf.cast(( content_type_ids == 0), dtype=tf.int32)
 lecture_mask = tf.cast(( content_type_ids == 1), dtype=tf.int32)
 pad_mask = tf.cast(( content_type_ids == PAD_TOKEN), dtype=tf.int32)
 _pad_mask = tf.cast(( batch['timestamp'] == PAD_TOKEN), dtype=tf.int32)
 pred_time_mask = batch['pred_time_mask'] * tf.cast(batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 pred_time_question_mask = pred_time_mask * question_mask
n_questions_in_pred_time = tf.math.reduce_sum(pred_time_question_mask, axis=1)
 response_false_mask = tf.cast(( answered_correctly == RESPONSE_FALSE_TOKEN), dtype=tf.int32)
 response_true_mask = tf.cast(( answered_correctly == RESPONSE_TRUE_TOKEN), dtype=tf.int32)
 _batch = {}
for k in batch:
_batch[k] = batch[k]
_batch['abs_pos_ids'] = tf.cast(tf.math.minimum(tf.math.maximum(0, _batch['abs_pos'] + 1), MAX_HISTORY_LEN), dtype=DTYPE)/ tf.cast(MAX_HISTORY_LEN, dtype=DTYPE)* WINDOW_SIZE
_batch['shifted_abs_pos_ids'] = tf.cast(tf.math.minimum(tf.math.maximum(0, _batch['shifted_abs_pos'] + 1), MAX_HISTORY_LEN - 1), dtype=DTYPE)/ tf.cast(MAX_HISTORY_LEN, dtype=DTYPE)* WINDOW_SIZE
_batch['pos_ids'] = tf.math.cumsum(1 - pad_mask, axis=1)
 _batch['shifted_pos_ids'] = tf.math.maximum(0, _batch['pos_ids'] - 1)
 content_input_ids = question_mask * question_id_to_input_id_table.lookup(content_ids)+ \
lecture_mask * lecture_id_to_input_id_table.lookup(content_ids)+ \
pad_mask * PAD_ID
_batch['c_input_ids'] = content_input_ids
response_input_ids_masked = pad_mask * PAD_ID + pred_time_question_mask * MASK_ID + lecture_mask * RESPONSE_LECTURE_ID +(1 - pred_time_question_mask)* response_false_mask * RESPONSE_FALSE_ID +(1 - pred_time_question_mask)* response_true_mask * RESPONSE_TRUE_ID
_batch['r_input_ids'] = response_input_ids_masked
shifted_answered_correctly = batch['shifted_answered_correctly']
shifted_pad_mask = tf.cast(( shifted_answered_correctly == PAD_TOKEN), dtype=tf.int32)
 shifted_start_mask = tf.cast(( shifted_answered_correctly == START_TOKEN), dtype=tf.int32)
 shifted_masking_mask = tf.cast(( shifted_answered_correctly == MASK_TOKEN), dtype=tf.int32)
 shifted_response_lecture_mask = tf.cast(( shifted_answered_correctly == RESPONSE_LECTURE_TOKEN), dtype=tf.int32)
 shifted_response_false_mask = tf.cast(( shifted_answered_correctly == RESPONSE_FALSE_TOKEN), dtype=tf.int32)
 shifted_response_true_mask = tf.cast(( shifted_answered_correctly == RESPONSE_TRUE_TOKEN), dtype=tf.int32)
 decoder_input_ids = shifted_pad_mask * PAD_ID + shifted_start_mask * START_ID + shifted_masking_mask * MASK_ID + shifted_response_lecture_mask * RESPONSE_LECTURE_ID + shifted_response_false_mask * RESPONSE_FALSE_ID + shifted_response_true_mask * RESPONSE_TRUE_ID
shifted_user_answer = batch['shifted_user_answer']
shifted_answer_0_mask = tf.cast(( shifted_user_answer == ANSWER_0_TOKEN), dtype=tf.int32)
 shifted_answer_1_mask = tf.cast(( shifted_user_answer == ANSWER_1_TOKEN), dtype=tf.int32)
 shifted_answer_2_mask = tf.cast(( shifted_user_answer == ANSWER_2_TOKEN), dtype=tf.int32)
 shifted_answer_3_mask = tf.cast(( shifted_user_answer == ANSWER_3_TOKEN), dtype=tf.int32)
 decoder_answer_input_ids = shifted_pad_mask * PAD_ID + shifted_start_mask * START_ID + shifted_masking_mask * MASK_ID + \
shifted_response_lecture_mask * ANSWER_LECTURE_ID + \
shifted_answer_0_mask * ANSWER_0_ID + shifted_answer_1_mask * ANSWER_1_ID + \
shifted_answer_2_mask * ANSWER_2_ID + shifted_answer_3_mask * ANSWER_3_ID
if not generative:
pred_time_question_start_mask = tf.cast(tf.math.cumsum(pred_time_question_mask, axis=1)== 1, dtype=tf.int32)
 pred_time_question_start_value_response = tf.math.reduce_sum(pred_time_question_start_mask * decoder_input_ids, axis=1)
 pred_time_question_start_value_answer = tf.math.reduce_sum(pred_time_question_start_mask * decoder_answer_input_ids, axis=1)
 prev_lecture_mask = tf.cast(pred_time_question_start_value_response == MASK_ID, tf.int32)
 pred_time_question_start_value_response = RESPONSE_LECTURE_ID * prev_lecture_mask + pred_time_question_start_value_response *(1 - prev_lecture_mask)
 pred_time_question_start_value_answer = ANSWER_LECTURE_ID * prev_lecture_mask + pred_time_question_start_value_answer *(1 - prev_lecture_mask)
 decoder_input_ids = decoder_input_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pred_time_question_start_value_response[:, tf.newaxis]
decoder_answer_input_ids = decoder_answer_input_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pred_time_question_start_value_answer[:, tf.newaxis]
decoder_input_ids = decoder_input_ids * tf.cast(decoder_input_ids != MASK_ID, dtype=tf.int32)+ RESPONSE_LECTURE_ID * tf.cast(decoder_input_ids == MASK_ID, dtype=tf.int32)
 decoder_answer_input_ids = decoder_answer_input_ids * tf.cast(decoder_answer_input_ids != MASK_ID, dtype=tf.int32)+ ANSWER_LECTURE_ID * tf.cast(decoder_answer_input_ids == MASK_ID, dtype=tf.int32)
 _batch['d_input_ids'] = decoder_input_ids
_batch['d_ans_input_ids'] = decoder_answer_input_ids
if not generative:
pos_ids = _batch['pos_ids']
shifted_pos_ids = _batch['shifted_pos_ids']
abs_pos_ids = _batch['abs_pos_ids']
shifted_abs_pos_ids = _batch['shifted_abs_pos_ids']
pos_ids_question_start_value = tf.math.reduce_sum(pred_time_question_start_mask * pos_ids, axis=1)
 shifted_pos_ids_question_start_value = tf.math.reduce_sum(pred_time_question_start_mask * shifted_pos_ids, axis=1)
 pos_ids = pos_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pos_ids_question_start_value[:, tf.newaxis]
shifted_pos_ids = shifted_pos_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * shifted_pos_ids_question_start_value[:, tf.newaxis]
abs_pos_ids_question_start_value = tf.math.reduce_sum(tf.cast(pred_time_question_start_mask, dtype=DTYPE)* abs_pos_ids, axis=1)
 shifted_abs_pos_ids_question_start_value = tf.math.reduce_sum(tf.cast(pred_time_question_start_mask, dtype=DTYPE)* shifted_abs_pos_ids, axis=1)
 abs_pos_ids = abs_pos_ids * tf.cast(1 - pred_time_question_mask, dtype=DTYPE)+ tf.cast(pred_time_question_mask, dtype=DTYPE)* abs_pos_ids_question_start_value[:, tf.newaxis]
shifted_abs_pos_ids = shifted_abs_pos_ids * tf.cast(1 - pred_time_question_mask, dtype=DTYPE)+ tf.cast(pred_time_question_mask, dtype=DTYPE)* shifted_abs_pos_ids_question_start_value[:, tf.newaxis]
_batch['pos_ids'] = pos_ids
_batch['shifted_pos_ids'] = shifted_pos_ids
_batch['abs_pos_ids'] = abs_pos_ids
_batch['shifted_abs_pos_ids'] = shifted_abs_pos_ids
_batch['tag_ids'] = tf.gather(params=c_inputs_ids_to_tags, indices=_batch['c_input_ids'])+ 1
_batch['part_ids'] = tf.gather(params=c_inputs_ids_to_part, indices=_batch['c_input_ids'])+ 1
_batch['prior_explanation_ids'] = tf.math.maximum(0, _batch['prior_question_had_explanation'] + 1)
 _batch['prior_question_elapsed_time_input'] = tf.cast(_batch['prior_question_elapsed_time'], dtype=DTYPE)/ 1000.0 / 300.0
lag_time = tf.cast(_batch['lag_time'], dtype=DTYPE)
 lag_time = lag_time / 1000.0 / 3600.0
lag_time = tf.math.minimum(lag_time, 72.0)
 _batch['lag_time'] = lag_time
task_container_pos_ids = tf.cast(_batch['task_container_id'], dtype=DTYPE)/ MAX_TASK_CONTAINER_ID * 10.0
_batch['task_container_pos_ids'] = task_container_pos_ids
answer_mask = tf.cast(batch['answered_correctly'] > -1, dtype=tf.int32)
 _batch['target'] = batch['answered_correctly'] * answer_mask +(NON_TARGET_ID)*(1 - answer_mask)
 _batch['answer_target'] = batch['user_answer'] * answer_mask +(NON_TARGET_ID)*(1 - answer_mask)
 _batch['correct_answer_id'] = tf.gather(params=c_inputs_ids_to_correct_answer_id, indices=_batch['c_input_ids'])
 targets = _batch['target']
if training == 0:
targets = targets * pred_time_mask + NON_TARGET_ID *(1 - pred_time_mask)
 pred_mask = targets != NON_TARGET_ID
nb_pred_places = tf.math.reduce_sum(tf.cast(pred_mask, dtype=tf.int32))
_batch['nb_pred_places'] = nb_pred_places * tf.ones_like(_batch['user_id'], dtype=tf.int32)
 scaling_factor = tf.constant(N_AGGREGATED_QUESTION_SCALING_FACTOR, dtype=tf.float32)
 _batch['n_questions_answered_scaled'] = tf.cast(_batch['n_questions_answered'], dtype=tf.float32)/ scaling_factor
_batch['n_lectures_watched_scaled'] = tf.cast(_batch['n_lectures_watched'], dtype=tf.float32)/ scaling_factor
n_questions_answered = tf.cast(tf.math.maximum(_batch['n_questions_answered'], 1), dtype=tf.float32)
 _batch['answered_correctly_ratio'] = tf.cast(_batch['n_questions_answered_correctly'], dtype=tf.float32)/ n_questions_answered
part_count_scaled = []
part_correctness_ratio = []
for part_idx in range(2, 9):
key = f'part_{part_idx}_count'
_part_count = tf.cast(_batch[key], dtype=tf.float32)
 _part_count_scaled = _part_count / scaling_factor
part_count_scaled.append(_part_count_scaled)
 key_2 = f'part_{part_idx}_correctness_count'
_part_correctness_count = tf.cast(_batch[key_2], dtype=tf.float32)
 _part_correctness_ratio = _part_correctness_count / tf.math.maximum(_part_count, 1.0)
 part_correctness_ratio.append(_part_correctness_ratio)
 del _batch[key]
del _batch[key_2]
part_count_scaled = tf.stack(part_count_scaled, axis=-1)
 part_correctness_ratio = tf.stack(part_correctness_ratio, axis=-1)
 _batch['part_count_scaled'] = part_count_scaled
_batch['part_correctness_ratio'] = part_correctness_ratio
_part_count_scaled = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
part_count_scaled
],
axis=-1 
)
 _part_correctness_ratio = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
part_correctness_ratio
],
axis=-1 
)
 current_part_count_scaled = tf.gather(params=_part_count_scaled, indices=_batch['part_ids'], batch_dims=2)
 current_part_correctness_ratio = tf.gather(params=_part_correctness_ratio, indices=_batch['part_ids'], batch_dims=2)
 _batch['current_part_count_scaled'] = current_part_count_scaled
_batch['current_part_correctness_ratio'] = current_part_correctness_ratio
correct_answer_count_scaled = []
correct_answer_correctness_ratio = []
for correct_answer_idx in range(ANSWER_0_ID, ANSWER_3_ID + 1):
key = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_count'
_correct_answer_count = tf.cast(_batch[key], dtype=tf.float32)
 _correct_answer_count_scaled = _correct_answer_count / scaling_factor
correct_answer_count_scaled.append(_correct_answer_count_scaled)
 key_2 = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_answered_correctly_count'
_correct_answer_answered_correctly_count = tf.cast(_batch[key_2], dtype=tf.float32)
 _correct_answer_answered_correctly_ratio = _correct_answer_answered_correctly_count / tf.math.maximum(_correct_answer_count, 1.0)
 correct_answer_correctness_ratio.append(_correct_answer_answered_correctly_ratio)
 del _batch[key]
del _batch[key_2]
correct_answer_count_scaled = tf.stack(correct_answer_count_scaled, axis=-1)
 correct_answer_correctness_ratio = tf.stack(correct_answer_correctness_ratio, axis=-1)
 _batch['correct_answer_count_scaled'] = correct_answer_count_scaled
_batch['correct_answer_correctness_ratio'] = correct_answer_correctness_ratio
_correct_answer_count_scaled = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
correct_answer_count_scaled
],
axis=-1 
)
 _correct_answer_correctness_ratio = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
correct_answer_correctness_ratio
],
axis=-1 
)
 _indices = tf.math.maximum(_batch['correct_answer_id'] - ANSWER_LECTURE_ID + 1, 0)
 current_correct_answer_count_scaled = tf.gather(params=_correct_answer_count_scaled, indices=_indices, batch_dims=2)
 current_correct_answer_correctness_ratio = tf.gather(params=_correct_answer_correctness_ratio, indices=_indices, batch_dims=2)
 _batch['current_correct_answer_count_scaled'] = current_correct_answer_count_scaled
_batch['current_correct_answer_correctness_ratio'] = current_correct_answer_correctness_ratio
_batch['current_question_count_scaled'] = tf.cast(_batch['n_prev_seen'], dtype=tf.float32)/ scaling_factor
current_question_count = tf.cast(tf.math.maximum(_batch['n_prev_seen'], 1), dtype=tf.float32)
 _batch['current_question_correctness_ratio'] = tf.cast(_batch['n_prev_correctness'], dtype=tf.float32)/ current_question_count
return _batch
def prepare_training_dataset(batched_ds, generative=False, use_abs_pos=False, num_parallel_calls=None, deterministic=None):
training = tf.constant(1, dtype=tf.int32)
 return batched_ds.map(lambda batch, _: add_input_ids_and_targets(batch, training, generative, use_abs_pos), num_parallel_calls=num_parallel_calls, deterministic=deterministic )<categorify>
start_time = time.time()
df_ip_count = df_merge['ip'].value_counts().reset_index(name = 'ip_count')
 df_ip_count.columns = ['ip', 'ip_count']
print('Loaded df_ip_count with {:.2f} seconds'.format(time.time() - start_time))
print('Starting to merge with main dataset...')
 df_merge = df_merge.merge(df_ip_count, on='ip', how='left', sort=False)
 df_merge['ip_count'] = df_merge['ip_count'].astype('uint16')
 print('Merging operation completed.')
 del df_ip_count
df_merge = df_merge.drop(['ip'], axis=1)
 gc.collect() ;
def get_input_signatures_2() :
batch_size = None
seq_len = 128
input_signatures = {
'user_id': tf.TensorSpec(shape=[batch_size], dtype=tf.int64, name='user_id'),
'seq_len': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='seq_len'),
'prev_seq_len': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='prev_seq_len'),
'start': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='start'),
'end': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='end'),
'row_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='row_id'),
'timestamp': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='timestamp'),
'content_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='content_id'),
'content_type_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='content_type_id'),
'task_container_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='task_container_id'),
'user_answer': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='user_answer'),
'shifted_user_answer': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_user_answer'),
'answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='answered_correctly'),
'shifted_answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_answered_correctly'),
'prior_question_elapsed_time': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='prior_question_elapsed_time'),
'prior_question_had_explanation': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='prior_question_had_explanation'),
'pred_time_mask': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='pred_time_mask'),
'abs_pos': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='abs_pos'),
'shifted_abs_pos': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_abs_pos'),
'lag_time': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='lag_time'),
'n_questions_answered': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_questions_answered'),
'n_lectures_watched': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_lectures_watched'),
'n_questions_answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_questions_answered_correctly'),
'n_prev_seen': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_prev_seen'),
'n_prev_correctness': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_prev_correctness'),
'part_2_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_2_count'),
'part_3_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_3_count'),
'part_4_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_4_count'),
'part_5_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_5_count'),
'part_6_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_6_count'),
'part_7_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_7_count'),
'part_8_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_8_count'),
'part_2_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_2_correctness_count'),
'part_3_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_3_correctness_count'),
'part_4_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_4_correctness_count'),
'part_5_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_5_correctness_count'),
'part_6_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_6_correctness_count'),
'part_7_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_7_correctness_count'),
'part_8_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_8_correctness_count'),
'correct_answer_0_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_0_count'),
'correct_answer_1_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_1_count'),
'correct_answer_2_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_2_count'),
'correct_answer_3_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_3_count'),
'correct_answer_0_answered_correctly_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_0_answered_correctly_count'),
'correct_answer_1_answered_correctly_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_1_answered_correctly_count'),
'correct_answer_2_answered_correctly_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_2_answered_correctly_count'),
'correct_answer_3_answered_correctly_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_3_answered_correctly_count'),
}
return [input_signatures]<categorify>
def clf_eval(y_true, y_pred):
print('Classification Report')
 print('ROC Score: {}'.format(roc_auc_score(y_true, y_pred)))
 return roc_auc_score(y_true, y_pred)
 df_train = df_merge[:train_record_index]
df_test = df_merge[train_record_index:]
del df_merge
gc.collect() ;
@tf.function(
 input_signature=get_input_signatures_2() 
)
 def add_input_ids_and_targets_2(batch):

 training = tf.constant(0, dtype=tf.int32)
 generative = tf.constant(False, dtype=tf.bool)
 content_ids = batch['content_id']
content_type_ids = batch['content_type_id']
answered_correctly = batch['answered_correctly']
user_answer = batch['user_answer']
question_mask = tf.cast(( content_type_ids == 0), dtype=tf.int32)
 lecture_mask = tf.cast(( content_type_ids == 1), dtype=tf.int32)
 pad_mask = tf.cast(( content_type_ids == PAD_TOKEN), dtype=tf.int32)
 _pad_mask = tf.cast(( batch['timestamp'] == PAD_TOKEN), dtype=tf.int32)
 pred_time_mask = batch['pred_time_mask'] * tf.cast(batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 pred_time_question_mask = pred_time_mask * question_mask
n_questions_in_pred_time = tf.math.reduce_sum(pred_time_question_mask, axis=1)
 response_false_mask = tf.cast(( answered_correctly == RESPONSE_FALSE_TOKEN), dtype=tf.int32)
 response_true_mask = tf.cast(( answered_correctly == RESPONSE_TRUE_TOKEN), dtype=tf.int32)
 _batch = {}
for k in batch:
_batch[k] = batch[k]
_batch['abs_pos_ids'] = tf.cast(tf.math.minimum(tf.math.maximum(0, _batch['abs_pos'] + 1), MAX_HISTORY_LEN), dtype=DTYPE)/ tf.cast(MAX_HISTORY_LEN, dtype=DTYPE)* WINDOW_SIZE
_batch['shifted_abs_pos_ids'] = tf.cast(tf.math.minimum(tf.math.maximum(0, _batch['shifted_abs_pos'] + 1), MAX_HISTORY_LEN - 1), dtype=DTYPE)/ tf.cast(MAX_HISTORY_LEN, dtype=DTYPE)* WINDOW_SIZE
_batch['pos_ids'] = tf.math.cumsum(1 - pad_mask, axis=1)
 _batch['shifted_pos_ids'] = tf.math.maximum(0, _batch['pos_ids'] - 1)
 content_input_ids = question_mask * question_id_to_input_id_table.lookup(content_ids)+ \
lecture_mask * lecture_id_to_input_id_table.lookup(content_ids)+ \
pad_mask * PAD_ID
_batch['c_input_ids'] = content_input_ids
response_input_ids_masked = pad_mask * PAD_ID + pred_time_question_mask * MASK_ID + lecture_mask * RESPONSE_LECTURE_ID +(1 - pred_time_question_mask)* response_false_mask * RESPONSE_FALSE_ID +(1 - pred_time_question_mask)* response_true_mask * RESPONSE_TRUE_ID
_batch['r_input_ids'] = response_input_ids_masked
shifted_answered_correctly = batch['shifted_answered_correctly']
shifted_pad_mask = tf.cast(( shifted_answered_correctly == PAD_TOKEN), dtype=tf.int32)
 shifted_start_mask = tf.cast(( shifted_answered_correctly == START_TOKEN), dtype=tf.int32)
 shifted_masking_mask = tf.cast(( shifted_answered_correctly == MASK_TOKEN), dtype=tf.int32)
 shifted_response_lecture_mask = tf.cast(( shifted_answered_correctly == RESPONSE_LECTURE_TOKEN), dtype=tf.int32)
 shifted_response_false_mask = tf.cast(( shifted_answered_correctly == RESPONSE_FALSE_TOKEN), dtype=tf.int32)
 shifted_response_true_mask = tf.cast(( shifted_answered_correctly == RESPONSE_TRUE_TOKEN), dtype=tf.int32)
 decoder_input_ids = shifted_pad_mask * PAD_ID + shifted_start_mask * START_ID + shifted_masking_mask * MASK_ID + shifted_response_lecture_mask * RESPONSE_LECTURE_ID + shifted_response_false_mask * RESPONSE_FALSE_ID + shifted_response_true_mask * RESPONSE_TRUE_ID
shifted_user_answer = batch['shifted_user_answer']
shifted_answer_0_mask = tf.cast(( shifted_user_answer == ANSWER_0_TOKEN), dtype=tf.int32)
 shifted_answer_1_mask = tf.cast(( shifted_user_answer == ANSWER_1_TOKEN), dtype=tf.int32)
 shifted_answer_2_mask = tf.cast(( shifted_user_answer == ANSWER_2_TOKEN), dtype=tf.int32)
 shifted_answer_3_mask = tf.cast(( shifted_user_answer == ANSWER_3_TOKEN), dtype=tf.int32)
 decoder_answer_input_ids = shifted_pad_mask * PAD_ID + shifted_start_mask * START_ID + shifted_masking_mask * MASK_ID + \
shifted_response_lecture_mask * ANSWER_LECTURE_ID + \
shifted_answer_0_mask * ANSWER_0_ID + shifted_answer_1_mask * ANSWER_1_ID + \
shifted_answer_2_mask * ANSWER_2_ID + shifted_answer_3_mask * ANSWER_3_ID
pred_time_question_start_mask = tf.cast(tf.math.cumsum(pred_time_question_mask, axis=1)== 1, dtype=tf.int32)
 pred_time_question_start_value_response = tf.math.reduce_sum(pred_time_question_start_mask * decoder_input_ids, axis=1)
 pred_time_question_start_value_answer = tf.math.reduce_sum(pred_time_question_start_mask * decoder_answer_input_ids, axis=1)
 prev_lecture_mask = tf.cast(pred_time_question_start_value_response == MASK_ID, tf.int32)
 pred_time_question_start_value_response = RESPONSE_LECTURE_ID * prev_lecture_mask + pred_time_question_start_value_response *(1 - prev_lecture_mask)
 pred_time_question_start_value_answer = ANSWER_LECTURE_ID * prev_lecture_mask + pred_time_question_start_value_answer *(1 - prev_lecture_mask)
 decoder_input_ids = decoder_input_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pred_time_question_start_value_response[:, tf.newaxis]
decoder_answer_input_ids = decoder_answer_input_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pred_time_question_start_value_answer[:, tf.newaxis]
decoder_input_ids = decoder_input_ids * tf.cast(decoder_input_ids != MASK_ID, dtype=tf.int32)+ RESPONSE_LECTURE_ID * tf.cast(decoder_input_ids == MASK_ID, dtype=tf.int32)
 decoder_answer_input_ids = decoder_answer_input_ids * tf.cast(decoder_answer_input_ids != MASK_ID, dtype=tf.int32)+ ANSWER_LECTURE_ID * tf.cast(decoder_answer_input_ids == MASK_ID, dtype=tf.int32)
 _batch['d_input_ids'] = decoder_input_ids
_batch['d_ans_input_ids'] = decoder_answer_input_ids
pos_ids = _batch['pos_ids']
shifted_pos_ids = _batch['shifted_pos_ids']
abs_pos_ids = _batch['abs_pos_ids']
shifted_abs_pos_ids = _batch['shifted_abs_pos_ids']
pos_ids_question_start_value = tf.math.reduce_sum(pred_time_question_start_mask * pos_ids, axis=1)
 shifted_pos_ids_question_start_value = tf.math.reduce_sum(pred_time_question_start_mask * shifted_pos_ids, axis=1)
 pos_ids = pos_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pos_ids_question_start_value[:, tf.newaxis]
shifted_pos_ids = shifted_pos_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * shifted_pos_ids_question_start_value[:, tf.newaxis]
abs_pos_ids_question_start_value = tf.math.reduce_sum(tf.cast(pred_time_question_start_mask, dtype=DTYPE)* abs_pos_ids, axis=1)
 shifted_abs_pos_ids_question_start_value = tf.math.reduce_sum(tf.cast(pred_time_question_start_mask, dtype=DTYPE)* shifted_abs_pos_ids, axis=1)
 abs_pos_ids = abs_pos_ids * tf.cast(1 - pred_time_question_mask, dtype=DTYPE)+ tf.cast(pred_time_question_mask, dtype=DTYPE)* abs_pos_ids_question_start_value[:, tf.newaxis]
shifted_abs_pos_ids = shifted_abs_pos_ids * tf.cast(1 - pred_time_question_mask, dtype=DTYPE)+ tf.cast(pred_time_question_mask, dtype=DTYPE)* shifted_abs_pos_ids_question_start_value[:, tf.newaxis]
_batch['pos_ids'] = pos_ids
_batch['shifted_pos_ids'] = shifted_pos_ids
_batch['abs_pos_ids'] = abs_pos_ids
_batch['shifted_abs_pos_ids'] = shifted_abs_pos_ids
_batch['tag_ids'] = tf.gather(params=c_inputs_ids_to_tags, indices=_batch['c_input_ids'])+ 1
_batch['part_ids'] = tf.gather(params=c_inputs_ids_to_part, indices=_batch['c_input_ids'])+ 1
_batch['prior_explanation_ids'] = tf.math.maximum(0, _batch['prior_question_had_explanation'] + 1)
 _batch['prior_question_elapsed_time_input'] = tf.cast(_batch['prior_question_elapsed_time'], dtype=DTYPE)/ 1000.0 / 300.0
lag_time = tf.cast(_batch['lag_time'], dtype=DTYPE)
 lag_time = lag_time / 1000.0 / 3600.0
lag_time = tf.math.minimum(lag_time, 72.0)
 _batch['lag_time'] = lag_time
task_container_pos_ids = tf.cast(_batch['task_container_id'], dtype=DTYPE)/ MAX_TASK_CONTAINER_ID * 10.0
_batch['task_container_pos_ids'] = task_container_pos_ids
answer_mask = tf.cast(batch['answered_correctly'] > -1, dtype=tf.int32)
 _batch['target'] = batch['answered_correctly'] * answer_mask +(NON_TARGET_ID)*(1 - answer_mask)
 _batch['answer_target'] = batch['user_answer'] * answer_mask +(NON_TARGET_ID)*(1 - answer_mask)
 _batch['correct_answer_id'] = tf.gather(params=c_inputs_ids_to_correct_answer_id, indices=_batch['c_input_ids'])
 targets = _batch['target']
if training == 0:
targets = targets * pred_time_mask + NON_TARGET_ID *(1 - pred_time_mask)
 pred_mask = targets != NON_TARGET_ID
nb_pred_places = tf.math.reduce_sum(tf.cast(pred_mask, dtype=tf.int32))
_batch['nb_pred_places'] = nb_pred_places * tf.ones_like(_batch['user_id'], dtype=tf.int32)
 scaling_factor = tf.constant(N_AGGREGATED_QUESTION_SCALING_FACTOR, dtype=tf.float32)
 _batch['n_questions_answered_scaled'] = tf.cast(_batch['n_questions_answered'], dtype=tf.float32)/ scaling_factor
_batch['n_lectures_watched_scaled'] = tf.cast(_batch['n_lectures_watched'], dtype=tf.float32)/ scaling_factor
n_questions_answered = tf.cast(tf.math.maximum(_batch['n_questions_answered'], 1), dtype=tf.float32)
 _batch['answered_correctly_ratio'] = tf.cast(_batch['n_questions_answered_correctly'], dtype=tf.float32)/ n_questions_answered
part_count_scaled = []
part_correctness_ratio = []
for part_idx in range(2, 9):
key = f'part_{part_idx}_count'
_part_count = tf.cast(_batch[key], dtype=tf.float32)
 _part_count_scaled = _part_count / scaling_factor
part_count_scaled.append(_part_count_scaled)
 key_2 = f'part_{part_idx}_correctness_count'
_part_correctness_count = tf.cast(_batch[key_2], dtype=tf.float32)
 _part_correctness_ratio = _part_correctness_count / tf.math.maximum(_part_count, 1.0)
 part_correctness_ratio.append(_part_correctness_ratio)
 del _batch[key]
del _batch[key_2]
part_count_scaled = tf.stack(part_count_scaled, axis=-1)
 part_correctness_ratio = tf.stack(part_correctness_ratio, axis=-1)
 _batch['part_count_scaled'] = part_count_scaled
_batch['part_correctness_ratio'] = part_correctness_ratio
_part_count_scaled = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
part_count_scaled
],
axis=-1 
)
 _part_correctness_ratio = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
part_correctness_ratio
],
axis=-1 
)
 current_part_count_scaled = tf.gather(params=_part_count_scaled, indices=_batch['part_ids'], batch_dims=2)
 current_part_correctness_ratio = tf.gather(params=_part_correctness_ratio, indices=_batch['part_ids'], batch_dims=2)
 _batch['current_part_count_scaled'] = current_part_count_scaled
_batch['current_part_correctness_ratio'] = current_part_correctness_ratio
correct_answer_count_scaled = []
correct_answer_correctness_ratio = []
for correct_answer_idx in range(ANSWER_0_ID, ANSWER_3_ID + 1):
key = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_count'
_correct_answer_count = tf.cast(_batch[key], dtype=tf.float32)
 _correct_answer_count_scaled = _correct_answer_count / scaling_factor
correct_answer_count_scaled.append(_correct_answer_count_scaled)
 key_2 = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_answered_correctly_count'
_correct_answer_answered_correctly_count = tf.cast(_batch[key_2], dtype=tf.float32)
 _correct_answer_answered_correctly_ratio = _correct_answer_answered_correctly_count / tf.math.maximum(_correct_answer_count, 1.0)
 correct_answer_correctness_ratio.append(_correct_answer_answered_correctly_ratio)
 del _batch[key]
del _batch[key_2]
correct_answer_count_scaled = tf.stack(correct_answer_count_scaled, axis=-1)
 correct_answer_correctness_ratio = tf.stack(correct_answer_correctness_ratio, axis=-1)
 _batch['correct_answer_count_scaled'] = correct_answer_count_scaled
_batch['correct_answer_correctness_ratio'] = correct_answer_correctness_ratio
_correct_answer_count_scaled = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
correct_answer_count_scaled
],
axis=-1 
)
 _correct_answer_correctness_ratio = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
correct_answer_correctness_ratio
],
axis=-1 
)
 _indices = tf.math.maximum(_batch['correct_answer_id'] - ANSWER_LECTURE_ID + 1, 0)
 current_correct_answer_count_scaled = tf.gather(params=_correct_answer_count_scaled, indices=_indices, batch_dims=2)
 current_correct_answer_correctness_ratio = tf.gather(params=_correct_answer_correctness_ratio, indices=_indices, batch_dims=2)
 _batch['current_correct_answer_count_scaled'] = current_correct_answer_count_scaled
_batch['current_correct_answer_correctness_ratio'] = current_correct_answer_correctness_ratio
_batch['current_question_count_scaled'] = tf.cast(_batch['n_prev_seen'], dtype=tf.float32)/ scaling_factor
current_question_count = tf.cast(tf.math.maximum(_batch['n_prev_seen'], 1), dtype=tf.float32)
 _batch['current_question_correctness_ratio'] = tf.cast(_batch['n_prev_correctness'], dtype=tf.float32)/ current_question_count
return _batch<categorify>
print('Fração de NaN em df_train: {:.4f} %'.format(100*len(df_train[df_train['diff'].isnull() ])/len(df_train)))
 print('Fração de NaN em df_test: {:.4f} %'.format(100*len(df_test[df_test['diff'].isnull() ])/len(df_test)) )
def add_aggregated_historical_information(parsed):

 question_mask = tf.cast(parsed['content_type_id'] == 0, dtype=tf.int32)
 lecture_mask = tf.cast(parsed['content_type_id'] == 1, dtype=tf.int32)
 correction_mask = tf.cast(parsed['answered_correctly'] == 1, dtype=tf.int32)
 content_ids = parsed['content_id']
non_pred_time_mask = 1 - parsed['pred_time_mask']
non_pred_time_question_mask = question_mask * non_pred_time_mask
non_pred_time_question_correction_mask = non_pred_time_question_mask * correction_mask
parsed['n_questions_answered'] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(non_pred_time_question_mask[:-1], axis=-1)
 ],
axis=-1 
)
 parsed['n_questions_answered_correctly'] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(non_pred_time_question_correction_mask[:-1], axis=-1)
 ],
axis=-1 
)
 parsed['n_lectures_watched'] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(lecture_mask[:-1], axis=-1)
 ],
axis=-1 
)
 content_input_ids = question_mask * question_id_to_input_id_table.lookup(content_ids)+ lecture_mask * lecture_id_to_input_id_table.lookup(content_ids)
 part_ids = tf.gather(params=c_inputs_ids_to_part, indices=content_input_ids)+ 1
for part_idx in range(2, 9):
key = f'part_{part_idx}_count'
part_mask = tf.cast(part_ids == part_idx, dtype=tf.int32)
 part_mask = part_mask * non_pred_time_question_mask
part_correct_mask = part_mask * non_pred_time_question_correction_mask
parsed[key] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(part_mask[:-1], axis=-1)
 ],
axis=-1 
)
 key = f'part_{part_idx}_correctness_count'
parsed[key] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(part_correct_mask[:-1], axis=-1)
 ],
axis=-1 
)
 correct_answer_id = tf.gather(params=c_inputs_ids_to_correct_answer_id, indices=content_input_ids)
 for correct_answer_idx in range(ANSWER_0_ID, ANSWER_3_ID + 1):
key = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_count'
correct_answer_mask = tf.cast(correct_answer_id == correct_answer_idx, dtype=tf.int32)
 correct_answer_mask = correct_answer_mask * non_pred_time_question_mask
correct_answer_answered_correctly_mask = correct_answer_mask * non_pred_time_question_correction_mask
parsed[key] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(correct_answer_mask[:-1], axis=-1)
 ],
axis=-1 
)
 key = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_answered_correctly_count'
parsed[key] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(correct_answer_answered_correctly_mask[:-1], axis=-1)
 ],
axis=-1 
)
 return parsed<feature_engineering>
df_train['diff'].fillna(0, inplace = True)
 df_test['diff'].fillna(0, inplace = True)
 gc.collect() ;
df_train['diff'] = df_train['diff'].astype('uint8')
 df_test['diff'] = df_test['diff'].astype('uint8')
 Learning_Y = Learning_Y.astype('uint8' )
def add_valid_block_info(valid_raw_example, num_parallel_calls=None, deterministic=None):

 example = {}
for k in valid_raw_example:
example[k] = valid_raw_example[k]
valid_block_pos = tf.stack([valid_raw_example['valid_blocks_start_pos'], valid_raw_example['valid_blocks_end_pos']], axis=1)
 example['valid_block_pos'] = valid_block_pos
n_valid_blocks = example['n_valid_blocks']
ds_1 = tf.data.Dataset.from_tensors(example ).repeat(tf.cast(n_valid_blocks, dtype=tf.int64))
ds_2 = tf.data.Dataset.range(tf.cast(n_valid_blocks, dtype=tf.int64), output_type=tf.int32)
 ds = tf.data.Dataset.zip(( ds_1, ds_2))
ds = ds.map(lambda ex, valid_block_idx: add_valid_block_idx(ex, valid_block_idx), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 return ds
def add_valid_block_idx(valid_example, valid_block_idx):

 valid_example['valid_block_idx'] = valid_block_idx
valid_example['valid_start'] = valid_example['valid_block_pos'][valid_block_idx][0]
valid_example['valid_end'] = valid_example['valid_block_pos'][valid_block_idx][1]
return valid_example
def remove_future_valid_blocks(valid_example):

 example = {}
valid_start = valid_example['valid_start']
valid_end = valid_example['valid_end']
example['user_id'] = valid_example['user_id']
example['seq_len'] = valid_end + 1
example['prev_seq_len'] = valid_example['seq_len']
example['start'] = tf.constant(0, dtype=tf.int32)
 example['end'] = valid_end
for k in valid_example:
if k not in ['user_id', 'seq_len', 'prev_seq_len', 'start', 'end']:
if k in ['n_valid_blocks', 'valid_blocks_start_pos', 'valid_blocks_end_pos', 'valid_block_pos', 'valid_block_idx', 'valid_start', 'valid_end']:
example[k] = valid_example[k]
else:
example[k] = valid_example[k][0:valid_end + 1]
if k == 'pred_time_mask':
n_valid_interactions = valid_end - valid_example['valid_start'] + 1
example[k] = tf.concat([valid_example[k][:valid_start], tf.ones(shape=[n_valid_interactions], dtype=tf.int32)], axis=0)
 return example
def extract_ending_subseqs(raw_batch, window_size):
ending_indices = raw_batch['seq_len'] - 1
return extract_subseqs_from_raw_batch(raw_batch, ending_indices, window_size)
 def prepare_validation_dataset(valid_raw_ds, batch_size=3, window_size=5, generative=False, use_abs_pos=False, seed=None, num_parallel_calls=None, deterministic=None):
valid_ds = valid_raw_ds.flat_map(lambda valid_raw_example: add_valid_block_info(valid_raw_example, num_parallel_calls=num_parallel_calls, deterministic=deterministic))
valid_ds = valid_ds.map(lambda example: remove_future_valid_blocks(example), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 valid_ds = valid_ds.map(lambda example: add_aggregated_historical_information(example), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 batched_valid_ds = valid_ds.apply(tf.data.experimental.dense_to_ragged_batch(batch_size=batch_size, drop_remainder=(IS_KAGGLE and tpu is not None)))
 batched_valid_ds = batched_valid_ds.map(lambda raw_batch: extract_ending_subseqs(raw_batch, window_size), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 training = tf.constant(0, dtype=tf.int32)
 valid_ds = batched_valid_ds.map(lambda batch: add_input_ids_and_targets(batch, training, generative, use_abs_pos), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 return valid_ds<split>
X_train, X_cv, Y_train, Y_cv = model_selection.train_test_split(df_train, Learning_Y, train_size = 0.8)
 gc.collect() ;
print('Data splitting into training and cross validation is done.' )
if not IS_KAGGLE:
train_raw_ds = tf.data.TFRecordDataset(train_tfrec_paths, num_parallel_reads=1)
 train_raw_ds = train_raw_ds.map(parse_train_example, num_parallel_calls=1, deterministic=True)
 train_raw_ds = train_raw_ds.map(lambda example: add_aggregated_historical_information(example))
train_valid_split_indices = load_data(train_valid_split_indices_paths[0])
 train_valid_split_table = convert_split_index_dict(train_valid_split_indices)
 reduced_raw_ds = split_train_ds(train_raw_ds, train_valid_split_table, num_parallel_calls=1, deterministic=True)
 batched_raw_ds = reduced_raw_ds.apply(tf.data.experimental.dense_to_ragged_batch(batch_size=5))
batched_ds = random_subseqs_from_batched_raw_ds(batched_raw_ds, window_size=10, seed=1, num_parallel_calls=1, deterministic=True)
 train_ds = prepare_training_dataset(batched_ds, use_abs_pos=False, num_parallel_calls=1, deterministic=True )<train_model>
print('Starting to fit Random Forest Model...The machine is learning...')
 start_time = time.time()
rf = RandomForestClassifier(n_estimators=13, max_depth=13, random_state=13, verbose=2, n_jobs = 4)
 cols = ['app', 'os', 'channel', 'device', 'click_hour', 'diff', 'ip_count']
rf.fit(X_train[cols], Y_train)
 print('The machine has learned.')
 print('RandomForest has fitted X_train with {:.2f} seconds'.format(time.time() - start_time))
if not IS_KAGGLE:
valid_raw_ds = tf.data.TFRecordDataset([valid_tfrec_paths], num_parallel_reads=1)
 valid_raw_ds = valid_raw_ds.map(parse_train_example_with_valid_info, num_parallel_calls=1, deterministic=True)
 for x in valid_raw_ds.take(1):
for k in x:
print(f'{k} : shape = {x[k].shape}')
 print('--------')
 print('============================================')
 valid_ds = prepare_validation_dataset(valid_raw_ds, batch_size=5, window_size=10, use_abs_pos=False)
 for x in valid_ds.take(1):
for k in x:
print(f'{k} : shape = {x[k].shape}')
 print('--------' )<normalization>
print('Starting cross-validation prediction phase...')
 start_time = time.time()
predictions = rf.predict_proba(X_cv[cols])[:,1]
print('Prediction done.Elapsed time: {:.2f} seconds'.format(time.time() - start_time))
clf_eval(Y_cv, predictions )
def get_initializer(seed):
return tf.keras.initializers.GlorotUniform(seed=seed)
 def gelu(x):

 x = tf.convert_to_tensor(x)
 cdf = 0.5 *(1.0 + tf.math.erf(x / tf.cast(tf.math.sqrt(2.0), dtype=x.dtype)))
 return x * cdf
def gelu_new(x):

 x = tf.convert_to_tensor(x)
 pi = tf.cast(math.pi, x.dtype)
 coeff = tf.cast(0.044715, x.dtype)
 cdf = 0.5 *(1.0 + tf.tanh(tf.sqrt(2.0 / pi)*(x + coeff * tf.pow(x, 3))))
return x * cdf
def gelu_fast(x):
x = tf.convert_to_tensor(x)
 coeff1 = tf.cast(7978845608, x.dtype)
 coeff2 = tf.cast(0.044715, x.dtype)
 return 0.5 * x *(1.0 + tf.tanh(x * coeff2 *(1.0 + coeff1 * x * x)))
 ACT2FN = {
"gelu": tf.keras.layers.Activation(gelu),
"relu": tf.keras.activations.relu,
"swish": tf.keras.activations.swish,
"silu": tf.keras.activations.swish,
"gelu_new": tf.keras.layers.Activation(gelu_new),
"tanh": tf.keras.activations.tanh,
"gelu_fast": tf.keras.layers.Activation(gelu_fast),
}
def get_tf_activation(activation_string):
if activation_string in ACT2FN:
return ACT2FN[activation_string]
else:
raise KeyError("function {} not found in ACT2FN mapping {}".format(activation_string, list(ACT2FN.keys())) )<init_hyperparams>
rus = RandomUnderSampler(return_indices=True)
 X_resampled, Y_resampled, idx_resampled = rus.fit_sample(X = X_train, y = Y_train )
class EdFormerConfig(PretrainedConfig):
def __init__(
 self,
model_type,
model_desc,
model_size='none',
content_vocab_size=CONTENT_VOCAB_SIZE,
response_vocab_size=RESPONSE_VOCAB_SIZE,
tag_vocab_size=TAG_VOCAB_SIZE,
part_vocab_size=PART_VOCAB_SIZE,
prior_explanation_vocab_size=PRIOR_EXPLANATION_VOCAB_SIZE,
max_position_embeddings=WINDOW_SIZE + 1,
sinusoidal_pos_embds=False,
n_layers=4,
n_heads=8,
dim=512,
hidden_dim=4 * 512,
activation=ACTIVATION,
dropout=0.1,
attention_dropout=0.1,
seq2seq_dropout=0.1,
initializer_range=0.02,
seed=SEED,
pad_token_id=PAD_ID,
use_user_answer=USE_USER_ANSWER,
use_user_answer_loss=USE_USER_ANSWER_LOSS,
use_correct_answer_for_encoder=USE_CORRECT_ANSWER_FOR_ENCODER,
use_correct_answer_for_decoder=USE_CORRECT_ANSWER_FOR_DECODER,
use_abs_pos=USE_ABS_POS,
use_task_container_pos=USE_TASK_CONTAINER_POS,
share_position_embeddings=SHARE_POS_EMBEDDING,
use_tags=USE_TAGS,
use_part=USE_PART,
use_prior_explanation=USE_PRIOR_EXPLANATION,
use_prior_question_elapsed_time_input=USE_PRIOR_QUESTION_ELAPSED_TIME_INPUT,
use_lag_time=USE_LAG_TIME,
use_lag_time_for_encoder=USE_LAG_TIME_FOR_ENCODER,
use_user_level_aggregated_historical_info=USE_USER_LEVEL_AGGREGATED_HISTORICAL_INFO,
use_part_aggregated_historical_info=USE_PART_AGGREGATED_HISTORICAL_INFO,
use_correct_answer_aggregated_historical_info=USE_CORRECT_ANSWER_AGGREGATED_HISTORICAL_INFO,
use_question_level_aggregated_historical_info=USE_QUESTION_LEVEL_AGGREGATED_HISTORICAL_INFO,
allow_bundle_atten=ALLOW_BUNDLE_ATTEN,
generative=GENERATIVE,
use_pre_classifier=USE_PRE_CLASSIFIER,
use_softmax=USE_SOFTMAX,
**kwargs 
):
super().__init__(**kwargs, pad_token_id=pad_token_id)
 self.model_type = model_type
self.model_size = model_size
self.model_desc = model_desc
self.content_vocab_size = content_vocab_size
self.response_vocab_size = response_vocab_size
self.tag_vocab_size = tag_vocab_size
self.part_vocab_size = part_vocab_size
self.prior_explanation_vocab_size = prior_explanation_vocab_size
self.use_user_answer = use_user_answer
self.use_correct_answer_for_encoder = use_correct_answer_for_encoder
self.use_correct_answer_for_decoder = use_correct_answer_for_decoder
self.use_user_answer_loss = use_user_answer_loss
self.use_abs_pos = use_abs_pos
self.use_task_container_pos = use_task_container_pos
self.share_position_embeddings = share_position_embeddings
self.use_tags = use_tags
self.use_part = use_part
self.use_prior_explanation = use_prior_explanation
self.allow_bundle_atten = allow_bundle_atten
self.generative = generative
self.use_prior_question_elapsed_time_input = use_prior_question_elapsed_time_input
self.use_part_aggregated_historical_info = use_part_aggregated_historical_info
self.use_correct_answer_aggregated_historical_info=use_correct_answer_aggregated_historical_info
self.use_question_level_aggregated_historical_info=use_question_level_aggregated_historical_info
self.use_lag_time = use_lag_time
self.use_lag_time_for_encoder = use_lag_time_for_encoder
self.use_user_level_aggregated_historical_info = use_user_level_aggregated_historical_info
self.use_pre_classifier = use_pre_classifier
self.use_softmax = use_softmax
self.max_position_embeddings = max_position_embeddings
self.sinusoidal_pos_embds = sinusoidal_pos_embds
self.n_layers = n_layers
self.n_heads = n_heads
self.dim = dim
self.hidden_dim = hidden_dim
self.activation = activation
self.dropout = dropout
self.attention_dropout = attention_dropout
self.seq2seq_dropout = seq2seq_dropout
self.initializer_range = initializer_range
self.seed = seed
def toJSON(self):
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=False, indent=4)
 def vocab_size(self, input_name):
if input_name == 'content':
return self.content_vocab_size
elif input_name == 'response':
return self.response_vocab_size
elif input_name == 'tag':
return self.tag_vocab_size
elif input_name == 'part':
return self.part_vocab_size
elif input_name == 'prior_explanation':
return self.prior_explanation_vocab_size
else:
raise ValueError('input name not used for model')
 @property
def hidden_size(self):
return self.dim
@property
def num_attention_heads(self):
return self.n_heads
@property
def num_hidden_layers(self):
return self.n_layers
class TFSharedEmbeddings(tf.keras.layers.Layer):
def __init__(self, vocab_size, hidden_size, seed=None, **kwargs):
super().__init__(**kwargs)
 self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.seed = seed
def build(self, input_shape):
self.weight = self.add_weight(
 "weight", shape=[self.vocab_size, self.hidden_size], initializer=get_initializer(seed=self.seed)
)
 super().build(input_shape)
 def call(self, inputs: tf.Tensor, mode: str = "embedding")-> tf.Tensor:
if mode == "embedding":
return self._embedding(inputs)
 elif mode == "linear":
return self._linear(inputs)
 else:
raise ValueError("mode {} is not valid.".format(mode))
def _embedding(self, input_ids):

 return tf.gather(self.weight, input_ids)
 def _linear(self, inputs):
first_dims = shape_list(inputs)[:-1]
x = tf.reshape(inputs, [-1, self.hidden_size])
 logits = tf.matmul(x, self.weight, transpose_b=True)
 return tf.reshape(logits, first_dims + [self.vocab_size])
 class TFEmbeddings(tf.keras.layers.Layer):
def __init__(
 self, config, input_name,
position_embeddings_layer=None,
**kwargs 
):
super().__init__(**kwargs)
 self.pad_id = config.pad_token_id
self.input_name = input_name
self.vocab_size = config.vocab_size(input_name)
 self.use_user_answer = config.use_user_answer
self.use_correct_answer_for_encoder = config.use_correct_answer_for_encoder
self.use_correct_answer_for_decoder = config.use_correct_answer_for_decoder
self.use_abs_pos = config.use_abs_pos
self.use_task_container_pos = config.use_task_container_pos
self.use_tags = config.use_tags
self.use_part = config.use_part
self.use_prior_explanation = config.use_prior_explanation
self.use_prior_question_elapsed_time_input = config.use_prior_question_elapsed_time_input
self.use_lag_time = config.use_lag_time
self.use_lag_time_for_encoder = config.use_lag_time_for_encoder
self.use_user_level_aggregated_historical_info = config.use_user_level_aggregated_historical_info
self.use_part_aggregated_historical_info = config.use_part_aggregated_historical_info
self.use_correct_answer_aggregated_historical_info = config.use_correct_answer_aggregated_historical_info
self.use_question_level_aggregated_historical_info = config.use_question_level_aggregated_historical_info
self.dim = config.dim
self.seed = config.seed
assert config.activation in ["relu", "gelu"], "activation({})must be in ['relu', 'gelu']".format(
 config.activation 
)
 self.activation = get_tf_activation(config.activation)
 self.word_embeddings = TFSharedEmbeddings(
 self.vocab_size, config.dim, seed=config.seed, name="word_embeddings" 
)
 self.word_embeddings_2 = TFSharedEmbeddings(
 ANSWER_VOCAB_SIZE, config.dim, seed=config.seed, name="word_embeddings_2" 
)
 if position_embeddings_layer is None:
self.position_embeddings = tf.keras.layers.Embedding(
 config.max_position_embeddings,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="position_embeddings", 
)
 else:
self.position_embeddings = position_embeddings_layer
self.correct_answer_embeddings = tf.keras.layers.Embedding(
 ANSWER_VOCAB_SIZE,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="correct_answer_embeddings", 
)
 self.tag_embeddings = tf.keras.layers.Embedding(
 config.tag_vocab_size,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="tag_embeddings", 
)
 self.part_embeddings = tf.keras.layers.Embedding(
 config.part_vocab_size,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="part_embeddings", 
)
 self.prior_explanation_embeddings = tf.keras.layers.Embedding(
 config.prior_explanation_vocab_size,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="prior_explanation_embeddings", 
)
 self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="LayerNorm")
 self.dropout = tf.keras.layers.Dropout(0.05)
 def build(self, input_shape):

 with tf.name_scope("word_embeddings"):
self.word_embeddings = self.add_weight(
 "weight", shape=[self.vocab_size, self.dim], initializer=get_initializer(self.seed)
)
 if self.input_name == 'response':
with tf.name_scope("word_embeddings_2"):
self.word_embeddings_2 = self.add_weight(
 "weight", shape=[ANSWER_VOCAB_SIZE, self.dim], initializer=get_initializer(self.seed)
)
 super().build(input_shape)
 def call(
 self,
input_ids=None, input_ids_2=None, position_ids=None,
tag_ids=None, part_ids=None,
prior_explanation_ids=None,
correct_answer_ids=None,
dense_embeddings_encoder=None,
dense_embeddings_decoder=None,
mode="embedding", training=False 
):
if mode == "embedding":
return self._embedding(
 input_ids, input_ids_2, position_ids,
tag_ids, part_ids,
prior_explanation_ids,
correct_answer_ids,
dense_embeddings_encoder,
dense_embeddings_decoder,
training=training 
)
 elif mode == "linear":
return self._linear(input_ids)
 else:
raise ValueError("mode {} is not valid.".format(mode))
def _embedding(
 self,
input_ids, input_ids_2, position_ids,
tag_ids, part_ids,
prior_explanation_ids,
correct_answer_ids,
dense_embeddings_encoder,
dense_embeddings_decoder,
training=False 
):
seq_length = shape_list(input_ids)[1]
inputs_embeds = tf.gather(self.word_embeddings, input_ids)
 position_embeddings = tf.cast(
 self.position_embeddings(position_ids), inputs_embeds.dtype 
)
 if self.input_name == 'content':
tag_embeddings = tf.cast(
 self.tag_embeddings(tag_ids), inputs_embeds.dtype 
)
 tag_mask = tf.cast(tag_ids != self.pad_id, dtype=tf.int32)
 tag_embeddings = tag_embeddings * tf.cast(tag_mask, dtype=inputs_embeds.dtype)[:, :, :, tf.newaxis]
nb_tags = tf.math.reduce_sum(tag_mask, axis=2)
 nb_tags = tf.cast(nb_tags, dtype=inputs_embeds.dtype)
 nb_tags = tf.math.maximum(nb_tags, tf.cast(1.0, dtype=inputs_embeds.dtype))
tag_embeddings = tf.math.reduce_sum(tag_embeddings, axis=2)/ nb_tags[:, :, tf.newaxis]
part_embeddings = tf.cast(
 self.part_embeddings(part_ids), inputs_embeds.dtype 
)
 correct_answer_embeddings = tf.cast(self.correct_answer_embeddings(correct_answer_ids), inputs_embeds.dtype)
 concated_embeddings = tf.concat(
 [
inputs_embeds[tf.newaxis, :, :, :],
position_embeddings[tf.newaxis, :, :, :],
tag_embeddings[tf.newaxis, :, :, :],
part_embeddings[tf.newaxis, :, :, :],
correct_answer_embeddings[tf.newaxis, :, :, :],
dense_embeddings_encoder
],
axis=0 
)
 if self.input_name == 'response':
inputs_embeds_2 = tf.gather(self.word_embeddings_2, input_ids_2)
 prior_explanation_embeddings = tf.cast(
 self.prior_explanation_embeddings(prior_explanation_ids), inputs_embeds.dtype 
)
 concated_embeddings = tf.concat(
 [
inputs_embeds[tf.newaxis, :, :, :],
inputs_embeds_2[tf.newaxis, :, :, :],
position_embeddings[tf.newaxis, :, :, :],
prior_explanation_embeddings[tf.newaxis, :, :, :],
dense_embeddings_decoder,
],
axis=0 
)
 embeddings = tf.math.reduce_sum(concated_embeddings, axis=0)
 embeddings = self.LayerNorm(embeddings)
 embeddings = self.dropout(embeddings, training=training)
 return embeddings
class TFMultiHeadSelfAttention(HFTFMultiHeadSelfAttention):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.q_lin = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="q_lin" 
)
 self.k_lin = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="k_lin" 
)
 self.v_lin = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="v_lin" 
)
 self.out_lin = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="out_lin" 
)
 def call(self, query, key, value, mask, head_mask, output_attentions, training=False):

 bs, q_length, dim = shape_list(query)
 k_length = shape_list(key)[1]
dim_per_head = tf.math.divide(self.dim, self.n_heads)
 dim_per_head = tf.cast(dim_per_head, dtype=tf.int32)
 def shape(x):

 return tf.transpose(tf.reshape(x,(bs, -1, self.n_heads, dim_per_head)) , perm=(0, 2, 1, 3))
def unshape(x):

 return tf.reshape(tf.transpose(x, perm=(0, 2, 1, 3)) ,(bs, -1, self.n_heads * dim_per_head))
q = shape(self.q_lin(query))
k = shape(self.k_lin(key))
v = shape(self.v_lin(value))
q = tf.cast(q, dtype=DTYPE)
 q = tf.multiply(q, tf.math.rsqrt(tf.cast(dim_per_head, dtype=DTYPE)))
 k = tf.cast(k, dtype=q.dtype)
 scores = tf.matmul(q, k, transpose_b=True)
 mask = mask[:, tf.newaxis, :, :]
mask = tf.cast(mask, dtype=scores.dtype)
 scores = scores - 1e30 *(1.0 - mask)
 weights = tf.nn.softmax(scores, axis=-1)
 weights = self.dropout(weights, training=training)
 weights = weights * mask
if head_mask is not None:
weights = weights * head_mask
context = tf.matmul(weights, v)
 context = unshape(context)
 context = self.out_lin(context)
 if output_attentions:
return(context, weights)
 else:
return(context,)
 class TFFFN(HFTFFFN):
def __init__(self, config, **kwargs):
super(TFFFN, self ).__init__(**kwargs)
 self.lin1 = tf.keras.layers.Dense(
 config.hidden_dim,
kernel_initializer=get_initializer(config.seed),
name="lin1" 
)
 self.lin2 = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="lin2" 
)
 self.activation = get_tf_activation(config.activation)
 class TFTransformerBlock(HFTFTransformerBlock):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.attention = TFMultiHeadSelfAttention(config, name="attention")
 class TFContentBlock(TFTransformerBlock):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.attention = TFMultiHeadSelfAttention(config, name="c_attention")
 class TFResponseBlock(TFTransformerBlock):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.attention = TFMultiHeadSelfAttention(config, name="r_attention")
 self.r_c_attentioin = TFMultiHeadSelfAttention(config, name="r_c_attentioin")
 self.r_c_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="r_c_attn_layer_norm")
 def call(self, r, c_hidden, r_mask, r_c_mask, head_mask, output_attentions, training=False):
r_output = self.attention(r, r, r, r_mask, head_mask, output_attentions, training=training)
 if output_attentions:
r_output, r_weights = r_output
else:
r_output = r_output[0]
r_output = self.sa_layer_norm(r_output + r)
 r_c_output = self.r_c_attentioin(
 query=r_output,
key=c_hidden,
value=c_hidden,
mask=r_c_mask,
head_mask=head_mask,
output_attentions=output_attentions,
training=training 
)
 if output_attentions:
r_c_output, r_c_weights = r_c_output
else:
r_c_output = r_c_output[0]
r_c_output = self.r_c_attn_layer_norm(r_c_output + r_output)
 ffn_output = self.ffn(r_c_output, training=training)
 ffn_output = self.output_layer_norm(ffn_output + r_c_output)
 output =(ffn_output,)
 if output_attentions:
output =(r_weights, r_c_weights)+ output
return output
class TFCRBlock(TFTransformerBlock):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.attention = TFMultiHeadSelfAttention(config, name="cr_attention")
 class TFContentCoder(TFTransformer):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.layer = [TFContentBlock(config, name="layer_._{}".format(i)) for i in range(config.n_layers)]
class TFResponseCoder(TFTransformer):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.layer = [TFResponseBlock(config, name="layer_._{}".format(i)) for i in range(config.n_layers)]
def call(self, r_embeds, c_hidden, r_mask, r_c_mask, head_mask, output_attentions, output_hidden_states, return_dict, training=False):

 all_hidden_states =() if output_hidden_states else None
all_attentions =() if output_attentions else None
hidden_state = r_embeds
for i, layer_module in enumerate(self.layer):
if output_hidden_states:
all_hidden_states = all_hidden_states +(hidden_state,)
 layer_outputs = layer_module(hidden_state, c_hidden, r_mask, r_c_mask, head_mask[i], output_attentions, training=training)
 hidden_state = layer_outputs[-1]
if output_attentions:
assert len(layer_outputs)== 3
r_attn = layer_outputs[0]
r_c_attn = layer_outputs[1]
all_attentions = all_attentions +(( r_attn, r_c_attn),)
 else:
assert len(layer_outputs)== 1, f"Incorrect number of outputs {len(layer_outputs)} instead of 1"
if output_hidden_states:
all_hidden_states = all_hidden_states +(hidden_state,)
 if not return_dict:
return tuple(v for v in [hidden_state, all_hidden_states, all_attentions] if v is not None)
 return TFBaseModelOutput(
 last_hidden_state=hidden_state, hidden_states=all_hidden_states, attentions=all_attentions 
)
 def process_mask(mask, input_shpae):
if mask is None:
mask = tf.ones(input_shpae)
 mask = tf.cast(mask, dtype=tf.float32)
 return mask
class TFEdFormerEncoderDecoder(tf.keras.layers.Layer):
def __init__(self, config, **kwargs):
super().__init__(**kwargs)
 self.encoder = TFContentCoder(config, name="encoder")
 self.decoder = TFResponseCoder(config, name="decoder")
 def call(
 self,
c_embeds,
r_embeds,
d_embeds,
c_mask,
r_mask,
r_c_mask,
c_r_mask,
head_mask,
output_attentions,
output_hidden_states,
return_dict,
training=False, 
):
c_outputs = self.encoder(
 c_embeds, c_mask, head_mask,
output_attentions, output_hidden_states, return_dict, training 
)
 if not return_dict:
c_hidden = c_outputs[0]
else:
c_hidden = c_outputs.last_hidden_state
r_outputs = self.decoder(
 d_embeds, c_hidden, r_mask, r_c_mask, head_mask,
output_attentions, output_hidden_states, return_dict, training 
)
 if not return_dict:
r_hidden = r_outputs[0]
else:
r_hidden = r_outputs.last_hidden_state
hidden_states = r_hidden
return(hidden_states, c_outputs, r_outputs)
 class TFEdFormerMainLayer(tf.keras.layers.Layer):
config_class = EdFormerConfig
def __init__(self, config, **kwargs):
super().__init__(**kwargs)
 self.num_hidden_layers = config.num_hidden_layers
self.output_attentions = config.output_attentions
self.output_hidden_states = config.output_hidden_states
self.return_dict = config.use_return_dict
self.use_user_answer = config.use_user_answer
self.use_correct_answer_for_encoder = config.use_correct_answer_for_encoder
self.use_correct_answer_for_decoder = config.use_correct_answer_for_decoder
self.use_abs_pos = config.use_abs_pos
self.use_task_container_pos = config.use_task_container_pos
self.use_tags = config.use_tags
self.use_part = config.use_part
self.use_prior_explanation = config.use_prior_explanation
self.use_prior_question_elapsed_time_input = config.use_prior_question_elapsed_time_input
self.use_lag_time = config.use_lag_time
self.use_lag_time_for_encoder = config.use_lag_time_for_encoder
self.use_user_level_aggregated_historical_info = config.use_user_level_aggregated_historical_info
self.use_part_aggregated_historical_info = config.use_part_aggregated_historical_info
self.use_correct_answer_aggregated_historical_info = config.use_correct_answer_aggregated_historical_info
self.use_question_level_aggregated_historical_info = config.use_question_level_aggregated_historical_info
if config.share_position_embeddings:
position_embeddings = tf.keras.layers.Embedding(
 config.max_position_embeddings,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="position_embeddings", 
)
 else:
position_embeddings = None
self.prior_question_elapsed_time_embeddings = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="prior_question_elapsed_time_embeddings", 
)
 self.lag_time_embeddings = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="lag_time_embeddings", 
)
 self.key_names_for_encoder = {
'n_questions_answered_scaled',
'n_lectures_watched_scaled',
'part_count_scaled',
'current_part_count_scaled',
'correct_answer_count_scaled',
'current_correct_answer_count_scaled',
'current_question_count_scaled',
}
self.key_name_need_newaxis = {
'n_questions_answered_scaled',
'n_lectures_watched_scaled',
'current_part_count_scaled',
'current_correct_answer_count_scaled',
'current_question_count_scaled',
'prior_question_elapsed_time_input',
'lag_time',
'answered_correctly_ratio',
'current_part_correctness_ratio',
'current_correct_answer_correctness_ratio',
'current_question_correctness_ratio',
}
self.dense_layer_name_mapping = {
'n_questions_answered_scaled': ['n_questions_answered_embeddings', 'n_questions_answered_layer_norm'],
'n_lectures_watched_scaled': ['n_lectures_watched_embeddings', 'n_lectures_watched_layer_norm'],
'part_count_scaled': ['part_count_embeddings', 'part_count_layer_norm'],
'current_part_count_scaled': ['current_part_count_embeddings', 'current_part_count_layer_norm'],
'correct_answer_count_scaled': ['correct_answer_count_embeddings', 'correct_answer_count_layer_norm'],
'current_correct_answer_count_scaled': ['current_correct_answer_count_embeddings', 'current_correct_answer_count_layer_norm'],
'current_question_count_scaled': ['current_question_count_embeddings', 'current_question_count_layer_norm'],
'prior_question_elapsed_time_input': ['prior_question_elapsed_time_input_embeddings', 'prior_question_elapsed_time_input_layer_norm'],
'lag_time': ['lag_time_embeddings', 'lag_time_layer_norm'],
'answered_correctly_ratio': ['answered_correctly_ratio_embeddings', 'answered_correctly_ratio_layer_norm'],
'part_correctness_ratio': ['part_correctness_ratio_embeddings', 'part_correctness_ratio_layer_norm'],
'current_part_correctness_ratio': ['current_part_correctness_ratio_embeddings', 'current_part_correctness_ratio_layer_norm'],
'correct_answer_correctness_ratio': ['correct_answer_correctness_ratio_embeddings', 'correct_answer_correctness_ratio_layer_norm'],
'current_correct_answer_correctness_ratio': ['current_correct_answer_correctness_ratio_embeddings', 'current_correct_answer_correctness_ratio_layer_norm'],
'current_question_correctness_ratio': ['current_question_correctness_ratio_embeddings', 'current_question_correctness_ratio_layer_norm'],
}
self.dense_layer_keys = [
'n_questions_answered_scaled',
'n_lectures_watched_scaled',
'part_count_scaled',
'current_part_count_scaled',
'correct_answer_count_scaled',
'current_correct_answer_count_scaled',
'current_question_count_scaled',
'prior_question_elapsed_time_input',
'lag_time',
'answered_correctly_ratio',
'part_correctness_ratio',
'current_part_correctness_ratio',
'correct_answer_correctness_ratio',
'current_correct_answer_correctness_ratio',
'current_question_correctness_ratio'
]
self.dense_layer_mapping = dict()
for key_name, layer_names in self.dense_layer_name_mapping.items() :
setattr(
 self,
layer_names[0],
tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name=layer_names[0], 
)
)
 setattr(
 self,
layer_names[1],
tf.keras.layers.LayerNormalization(epsilon=1e-12, name=layer_names[1])
)
 self.dense_layer_mapping[key_name] = [getattr(self, layer_names[0]), getattr(self, layer_names[1])]
assert config.activation in ["relu", "gelu"], "activation({})must be in ['relu', 'gelu']".format(
 config.activation 
)
 self.activation = get_tf_activation(config.activation)
 self.dropout = tf.keras.layers.Dropout(0.05)
 self.content_embeddings = TFEmbeddings(
 config,
input_name = 'content',
position_embeddings_layer=position_embeddings,
name="content_embeddings" 
)
 self.response_embeddings = TFEmbeddings(
 config,
input_name = 'response',
position_embeddings_layer=position_embeddings,
name="response_embeddings" 
)
 self.coder = TFEdFormerEncoderDecoder(config, name='coder')
 def call(
 self,
inputs,
c_mask=None,
r_mask=None,
r_c_mask=None,
c_r_mask=None,
head_mask=None,
output_attentions=None,
output_hidden_states=None,
return_dict=None,
training=False, 
):
c_input_ids = inputs.get('c_input_ids')
 r_input_ids = inputs.get('r_input_ids')
 d_input_ids = inputs.get('d_input_ids')
 d_ans_input_ids = inputs.get('d_ans_input_ids')
 tag_ids = inputs.get('tag_ids')
 part_ids = inputs.get('part_ids')
 batch_size = tf.math.reduce_sum(tf.ones_like(c_input_ids[:, 0], dtype=tf.int32))
seq_len = tf.math.reduce_sum(tf.ones_like(c_input_ids[0, :], dtype=tf.int32))
pos_ids = inputs['pos_ids']
shifted_pos_ids = inputs['shifted_pos_ids']
prior_explanation_ids = inputs.get('prior_explanation_ids')
 correct_answer_ids = inputs.get('correct_answer_id')
 c_mask = inputs.get('c_mask', c_mask)
 r_mask = inputs.get('r_mask', r_mask)
 r_c_mask = inputs.get('r_c_mask', r_c_mask)
 c_r_mask = inputs.get('c_r_mask', c_r_mask)
 output_attentions = inputs.get('output_attentions', output_attentions)
 output_hidden_states = inputs.get('output_hidden_states', output_hidden_states)
 return_dict = inputs.get('return_dict', return_dict)
 output_attentions = output_attentions if output_attentions is not None else self.output_attentions
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.output_hidden_states
return_dict = return_dict if return_dict is not None else self.return_dict
c_mask = process_mask(c_mask, [batch_size, seq_len, seq_len])
 r_mask = process_mask(r_mask, [batch_size, seq_len, seq_len])
 r_c_mask = process_mask(r_c_mask, [batch_size, seq_len, seq_len])
 c_r_mask = process_mask(c_r_mask, [batch_size, seq_len, seq_len])
 head_mask = [None] * self.num_hidden_layers
dense_embeddings_encoder = []
dense_embeddings_decoder = []
for key_name, layers in self.dense_layer_mapping.items() :
dense_layer = layers[0]
layer_norm = layers[1]
layer_input = inputs[key_name]
if key_name in self.key_name_need_newaxis:
layer_input = layer_input[:, :, tf.newaxis]
layer_output = dense_layer(layer_input)
 layer_output = layer_norm(
 self.dropout(
 self.activation(
 layer_output 
),
training=training 
)+ layer_output 
)
 if key_name in self.key_names_for_encoder:
dense_embeddings_encoder.append(layer_output)
 layer_output = dense_layer(layer_input)
 layer_output = layer_norm(
 self.dropout(
 self.activation(
 layer_output 
),
training=training 
)+ layer_output 
)
 dense_embeddings_decoder.append(layer_output)
 else:
dense_embeddings_decoder.append(layer_output)
 dense_embeddings_encoder = tf.stack(dense_embeddings_encoder, axis=0)
 dense_embeddings_decoder= tf.stack(dense_embeddings_decoder, axis=0)
 c_embedding_output = self.content_embeddings(
 input_ids=c_input_ids, input_ids_2=d_ans_input_ids, position_ids=pos_ids,
tag_ids=tag_ids, part_ids=part_ids,
prior_explanation_ids=prior_explanation_ids,
correct_answer_ids=correct_answer_ids,
dense_embeddings_encoder=dense_embeddings_encoder,
dense_embeddings_decoder=dense_embeddings_decoder,
training=training 
)
 d_embedding_output = self.response_embeddings(
 input_ids=d_input_ids, input_ids_2=d_ans_input_ids, position_ids=shifted_pos_ids,
tag_ids=tag_ids, part_ids=part_ids,
prior_explanation_ids=prior_explanation_ids,
correct_answer_ids=correct_answer_ids,
dense_embeddings_encoder=dense_embeddings_encoder,
dense_embeddings_decoder=dense_embeddings_decoder,
training=training 
)
 r_embedding_output = d_embedding_output
outputs = self.coder(
 c_embedding_output,
r_embedding_output,
d_embedding_output,
c_mask,
r_mask,
r_c_mask,
c_r_mask,
head_mask,
output_attentions,
output_hidden_states,
return_dict,
training=training 
)
 return outputs
class TFEdFormerPreTrainedModel(TFPreTrainedModel):

 config_class = EdFormerConfig
base_model_prefix = "edformer"
class TFEdFormerModel(TFEdFormerPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.edformer = TFEdFormerMainLayer(config, name="edformer")
 def call(self, inputs, **kwargs):
outputs = self.edformer(
 inputs,
**kwargs 
)
 return outputs
class TFEdFormerAnswerPredictionModel(TFPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.use_pre_classifier = config.use_pre_classifier
self.edformer = TFEdFormerMainLayer(config, name="edformer")
 if self.use_pre_classifier:
self.pre_classifier = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="pre_classifier", 
)
 self.pre_classifier_2 = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="pre_classifier_2", 
)
 assert config.activation in ["relu", "gelu"], "activation({})must be in ['relu', 'gelu']".format(
 config.activation 
)
 self.activation = get_tf_activation(config.activation)
 self.dropout = tf.keras.layers.Dropout(config.seq2seq_dropout)
 n_targets = 1
if config.use_softmax:
n_targets = 2
self.classifier = tf.keras.layers.Dense(
 n_targets, kernel_initializer=get_initializer(config.seed), name="classifier" 
)
 self.classifier_2 = tf.keras.layers.Dense(
 4, kernel_initializer=get_initializer(config.seed), name="classifier_2" 
)
 def call(
 self,
inputs=None,
c_mask=None,
r_mask=None,
r_c_mask=None,
c_r_mask=None,
head_mask=None,
output_attentions=None,
output_hidden_states=None,
return_dict=None,
training=False, 
):
return_dict = return_dict if return_dict is not None else self.edformer.return_dict
edformer_output = self.edformer(
 inputs,
c_mask=c_mask,
r_mask=r_mask,
r_c_mask=r_c_mask,
c_r_mask=c_r_mask,
head_mask=head_mask,
output_attentions=output_attentions,
output_hidden_states=output_hidden_states,
return_dict=return_dict,
training=training, 
)
 hidden_states, c_outputs, r_outputs = edformer_output
o = hidden_states
o_2 = o
if self.use_pre_classifier:
o = self.pre_classifier(o)
 o = self.activation(o)
 o = self.dropout(o, training=training)
 o_2 = self.pre_classifier_2(o_2)
 o_2 = self.activation(o_2)
 o_2 = self.dropout(o_2, training=training)
 logits = self.classifier(o)
 answer_logits = self.classifier_2(o_2)
 return(logits, answer_logits, c_outputs, r_outputs )<define_search_space>
X_resampled = pd.DataFrame(X_resampled, columns=df_train.columns)
 Y_resampled = pd.DataFrame(Y_resampled, columns=['is_attributed'] )
if not IS_KAGGLE or True:
content_input_ids = tf.constant(1, shape=[3, 5])
 response_input_ids = tf.constant(1, shape=[3, 5])
 pos_ids = tf.constant(1, shape=[3, 5])
 shifted_pos_ids = tf.constant(1, shape=[3, 5])
 d_input_ids = tf.constant(1, shape=[3, 5])
 d_ans_input_ids = tf.constant(1, shape=[3, 5])
 tag_ids = tf.constant(0, shape=[3, 5, N_TAGS_PER_CONTENT])
 part_ids = tf.constant(1, shape=[3, 5])
 prior_explanation_ids = tf.constant(1, shape=[3, 5])
 prior_question_elapsed_time_input = tf.constant(1.0, shape=[3, 5])
 lag_time = tf.constant(1.0, shape=[3, 5])
 abs_pos_ids = tf.constant(1.0, shape=[3, 5])
 shifted_abs_pos_ids = tf.constant(1.0, shape=[3, 5])
 task_container_pos_ids = tf.constant(1.0, shape=[3, 5])
 correct_answer_id = tf.constant(1, shape=[3, 5])
 n_questions_answered_scaled = tf.constant(1.0, shape=[3, 5])
 n_lectures_watched_scaled = tf.constant(1.0, shape=[3, 5])
 answered_correctly_ratio = tf.constant(1.0, shape=[3, 5])
 part_correctness_ratio = tf.constant(1.0, shape=[3, 5, PART_VOCAB_SIZE - 2])
 part_count_scaled = tf.constant(1.0, shape=[3, 5, PART_VOCAB_SIZE - 2])
 correct_answer_count_scaled = tf.constant(1.0, shape=[3, 5, ANSWER_3_ID - ANSWER_0_ID + 1])
 correct_answer_correctness_ratio = tf.constant(1.0, shape=[3, 5, ANSWER_3_ID - ANSWER_0_ID + 1])
 current_part_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_part_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 current_correct_answer_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_correct_answer_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 current_question_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_question_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 inputs = {
'c_input_ids': content_input_ids,
'r_input_ids': response_input_ids,
'd_input_ids': d_input_ids,
'd_ans_input_ids': d_ans_input_ids,
'pos_ids': pos_ids,
'shifted_pos_ids': shifted_pos_ids,
'tag_ids': tag_ids,
'part_ids': part_ids,
'prior_explanation_ids': prior_explanation_ids,
'prior_question_elapsed_time_input': prior_question_elapsed_time_input,
'lag_time': lag_time,
'abs_pos_ids': abs_pos_ids,
'shifted_abs_pos_ids': shifted_abs_pos_ids,
'task_container_pos_ids': task_container_pos_ids,
'correct_answer_id': correct_answer_id,
'n_questions_answered_scaled': n_questions_answered_scaled,
'n_lectures_watched_scaled': n_lectures_watched_scaled,
'answered_correctly_ratio': answered_correctly_ratio,
'part_correctness_ratio': part_correctness_ratio,
'part_count_scaled': part_count_scaled,
'correct_answer_count_scaled': correct_answer_count_scaled,
'correct_answer_correctness_ratio': correct_answer_correctness_ratio,
'current_part_count_scaled': current_part_count_scaled,
'current_part_correctness_ratio': current_part_correctness_ratio,
'current_correct_answer_count_scaled': current_correct_answer_count_scaled,
'current_correct_answer_correctness_ratio': current_correct_answer_correctness_ratio,
'current_question_count_scaled': current_question_count_scaled,
'current_question_correctness_ratio': current_question_correctness_ratio,
}
model_type = 'ed'
config = EdFormerConfig(
 model_type=model_type, model_desc='dummy',
share_position_embeddings=True, use_tags=True, user_part=True,
use_prior_explanation=True, use_prior_question_elapsed_time_input=True,
use_lag_time=True,
use_lag_time_for_encoder=True,
use_user_answer=True 
)
 predictor = TFEdFormerAnswerPredictionModel(config)
 @tf.function
def foo(inputs):
logits, logits_2, c_outputs, r_outputs = predictor(inputs=inputs, output_attentions=True, output_hidden_states=True, return_dict=False)
 return logits
logits = foo(inputs)
 print(logits )<categorify>
rf_resampled = RandomForestClassifier(n_estimators=13,
max_depth=13,
random_state=13,
verbose=2,
n_jobs = 5)
 rf_resampled.fit(X_resampled, Y_resampled)
 print('The machine has learned.')
 print('RandomForest has fitted X_train with {:.2f} seconds'.format(time.time() - start_time))
print('Starting cross-validation prediction phase...')
 predictions = rf.predict_proba(X_cv)[:,1]
print('Prediction done.Elapsed time: {:.2f} seconds'.format(time.time() - start_time))
clf_eval(Y_cv, predictions )
class LinearLR(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, total_steps, lr=1e-4, end_lr=1e-6, warmup_steps=WARMUP_STEPS):
self.total_steps = tf.cast(total_steps, dtype=DTYPE)
 self.lr = lr
self.end_lr = end_lr
self.warmup_steps = tf.cast(warmup_steps, dtype=DTYPE)
 def __call__(self, step):
is_warmup = tf.cast(step < self.warmup_steps, dtype=DTYPE)
 warmup_lr = is_warmup * self.lr *(step + 1)/ self.warmup_steps
_lr = tf.math.minimum(self.lr, 1 * self.end_lr)
 decay_lr =(1 - is_warmup)*(_lr -(_lr - self.end_lr)/ tf.math.maximum(self.total_steps - self.warmup_steps, 1)*(step - self.warmup_steps + 1))
decay_lr =(1 - is_warmup)* tf.math.maximum(self.end_lr, decay_lr)
 lr = warmup_lr + decay_lr
return lr
class NoamLR(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, hidden_size, lr, end_lr, warmup_steps):
self.lr = tf.cast(lr, DTYPE)
 self.end_lr = tf.cast(end_lr, DTYPE)
 self.warmup_steps = tf.cast(warmup_steps, DTYPE)
 self.hidden_size = tf.cast(hidden_size, DTYPE)
 def __call__(self, step):
scaling = self.lr /(self.hidden_size**-0.5 * self.warmup_steps**-0.5)
 lr = scaling * self.hidden_size**-0.5 * tf.math.minimum(( step + 1)* self.warmup_steps**-1.5,(step + 1)**-0.5)
 is_warmup = tf.cast(step < self.warmup_steps, dtype=DTYPE)
 lr = lr * is_warmup +(1.0 - is_warmup)* tf.math.maximum(self.end_lr, lr)
 return lr<data_type_conversions>
print('Starting prediction phase...')
 start_time = time.time()
predictions = rf.predict_proba(df_test[cols])
 print('Prediction done.Elapsed time: {:.2f} seconds'.format(time.time() - start_time))
df_submit['is_attributed'] = predictions[:,1]
print('Submission dataset created.')
 df_submit.to_csv('timed_rf_raul.csv', index=False)
 print('Submission dataset saved correctly.' )