Kaggle competition: TalkingData AdTracking Fraud Detection Challenge
def get_input_signatures(batch_size, seq_len, valid=False):
input_signatures = {
'user_id': tf.TensorSpec(shape=[batch_size], dtype=tf.int64, name='user_id'),
'seq_len': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='seq_len'),
'prev_seq_len': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='prev_seq_len'),
'start': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='start'),
'end': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='end'),
'row_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='row_id'),
'timestamp': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='timestamp'),
'content_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='content_id'),
'content_type_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='content_type_id'),
'task_container_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='task_container_id'),
'user_answer': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='user_answer'),
'shifted_user_answer': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_user_answer'),
'answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='answered_correctly'),
'shifted_answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_answered_correctly'),
'prior_question_elapsed_time': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='prior_question_elapsed_time'),
'prior_question_elapsed_time_input': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='prior_question_elapsed_time_input'),
'prior_question_had_explanation': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='prior_question_had_explanation'),
'pred_time_mask': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='pred_time_mask'),
'abs_pos': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='abs_pos'),
'shifted_abs_pos': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_abs_pos'),
'pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='pos_ids'),
'shifted_pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_pos_ids'),
'abs_pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='abs_pos_ids'),
'shifted_abs_pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='shifted_abs_pos_ids'),
'task_container_pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='task_container_pos_ids'),
'c_input_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='c_input_ids'),
'r_input_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='r_input_ids'),
'd_input_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='d_input_ids'),
'd_ans_input_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='d_ans_input_ids'),
'tag_ids': tf.TensorSpec(shape=[batch_size, seq_len, N_TAGS_PER_CONTENT], dtype=tf.int32, name='tag_ids'),
'part_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_ids'),
'prior_explanation_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='prior_explanation_ids'),
'lag_time': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='lag_time'),
'n_questions_answered': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_questions_answered'),
'n_lectures_watched': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_lectures_watched'),
'n_questions_answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_questions_answered_correctly'),
'n_questions_answered_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='n_questions_answered_scaled'),
'n_lectures_watched_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='n_lectures_watched_scaled'),
'answered_correctly_ratio': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='answered_correctly_ratio'),
'part_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len, PART_VOCAB_SIZE - 2], dtype=DTYPE, name='part_count_scaled'),
'part_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len, PART_VOCAB_SIZE - 2], dtype=DTYPE, name='part_correctness_ratio'),
'current_part_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_correccurrent_part_count_scaledt_answer_count_scaled'),
'current_part_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_part_correctness_ratio'),
'correct_answer_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len, ANSWER_3_ID - ANSWER_0_ID + 1], dtype=DTYPE, name='correct_answer_count_scaled'),
'correct_answer_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len, ANSWER_3_ID - ANSWER_0_ID + 1], dtype=DTYPE, name='correct_answer_correctness_ratio'),
'current_correct_answer_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_correct_answer_count_scaled'),
'current_correct_answer_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_correct_answer_correctness_ratio'),
'n_prev_seen': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_prev_seen'),
'n_prev_correctness': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_prev_correctness'),
'current_question_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_question_count_scaled'),
'current_question_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_question_correctness_ratio'),
'correct_answer_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_id'),
'target': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='target'),
'answer_target': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='answer_target'),
'nb_pred_places': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='nb_pred_places'),
}
if valid:
input_signatures['n_valid_blocks'] = tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='n_valid_blocks')
 input_signatures['valid_block_idx'] = tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='valid_block_idx')
 input_signatures['valid_start'] = tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='valid_start')
 input_signatures['valid_end'] = tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='valid_end')
 return [input_signatures]<define_variables>
from sklearn.cross_validation import train_test_split
import lightgbm as lgb
import time
class TrainConfig:
def __init__(
 self,
ckpt_path=CKPT_TRAIN_PATH,
window_size=WINDOW_SIZE,
loss_weight_window_size=LOSS_WEIGHT_WINDOW_SIZE,
n_epochs=N_EPOCHS,
batch_size=BATCH_SIZE,
pred_batch_size=PRED_BATCH_SIZE,
shuffle_buf_size=SHUFFLE_BUFFER_SIZE,
seed=SEED,
deterministic=DETERMINISTIC,
num_parallel_reads=N_PARALLEL_READS,
num_parallel_calls=N_PARALLEL_CALLS,
steps_per_call=STEPS_PER_CALL,
max_n_contents_per_user_for_sampling_prob=MAX_N_CONTENTS_PER_USER_FOR_SAMPLING_PROB,
valid_fold=VALID_FOLD 
):
self.ckpt_path = ckpt_path
self.window_size = window_size
self.loss_weight_window_size = loss_weight_window_size
self.n_epochs = n_epochs
self.batch_size = batch_size
self.pred_batch_size = pred_batch_size
self.shuffle_buf_size = shuffle_buf_size
self.seed=seed
self.deterministic = deterministic
self.num_parallel_reads = num_parallel_reads
self.num_parallel_calls = num_parallel_calls
self.steps_per_call = steps_per_call
self.max_n_contents_per_user_for_sampling_prob = max_n_contents_per_user_for_sampling_prob
self.valid_fold = valid_fold
n_contents_dict = load_data(n_contents_dict_path)
 n_training_examples = 0
for k, v in n_contents_dict.items() :
n_training_examples += math.ceil(v * 1.0 / self.window_size)
 del n_contents_dict
if self.valid_fold not in [0, 1, 2, 3]:
valid_info = {}
n_valid_examples = 0
else:
valid_info = load_data(valid_info_paths[self.valid_fold])
 n_valid_examples = 0
for k, v in valid_info.items() :
n_valid_examples += v['bundle_info']['n_blocks']
del valid_info
self.n_training_examples = n_training_examples
n_training_steps_per_epoch = 1 * self.n_training_examples // self.batch_size
self.n_training_calls_per_epoch = n_training_steps_per_epoch // self.steps_per_call
self.n_training_steps_per_epoch = self.n_training_calls_per_epoch * self.steps_per_call
self.n_training_steps = self.n_epochs * self.n_training_steps_per_epoch
self.n_valid_examples = n_valid_examples
if IS_KAGGLE and tpu is not None:
self.n_valid_examples -= self.n_valid_examples % self.pred_batch_size
self.n_valid_steps = self.n_valid_examples // self.pred_batch_size + int(self.n_valid_examples % self.pred_batch_size > 0)
 self.n_steps_in_last_valid_call = self.n_valid_steps % self.steps_per_call
self.n_valid_calls = self.n_valid_steps // self.steps_per_call + int(self.n_steps_in_last_valid_call > 0)
 self.lr = None
self.end_lr = None
self.warmup_steps = None
def toJSON(self):
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=False, indent=4)
 class Train_Manager:
def __init__(self, config, train_config):
self.config = config
self.train_config = train_config
n_contents_dict = load_data(n_contents_dict_path)
 user_id_list = []
n_contents_list = []
if self.train_config.valid_fold in [0, 1, 2, 3]:
train_valid_split_indices = load_data(train_valid_split_indices_paths[self.train_config.valid_fold])
 self.valid_user_ids = list(train_valid_split_indices.keys())
 else:
train_valid_split_indices = {}
self.valid_user_ids = []
for user_id, n_contents_orig in n_contents_dict.items() :
if self.train_config.valid_fold in [0, 1, 2, 3] and user_id in train_valid_split_indices:
split_index = train_valid_split_indices[user_id]
else:
split_index = n_contents_orig
n_contents_in_train = split_index
user_id_list.append(user_id)
 if n_contents_in_train <= 32:
n_contents_in_train = 32
n_contents_list.append(n_contents_in_train)
 user_id_tensor = tf.constant(user_id_list, dtype=tf.int64)
 n_contents_tensor = tf.constant(n_contents_list, dtype=tf.int32)
 n_contents_tensor = tf.math.minimum(n_contents_tensor, self.train_config.max_n_contents_per_user_for_sampling_prob)
 sample_prob_tensor = tf.cast(n_contents_tensor, dtype=tf.float32)/ tf.cast(tf.math.reduce_sum(n_contents_tensor), dtype=tf.float32)
 sample_prob_tensor = 0.9 * sample_prob_tensor / tf.math.reduce_max(sample_prob_tensor)
 del user_id_list
del n_contents_list
del n_contents_dict
initializer = tf.lookup.KeyValueTensorInitializer(user_id_tensor, sample_prob_tensor)
 training_sample_prob_table = tf.lookup.StaticHashTable(initializer, default_value=0.0)
 self.training_sample_prob_table = training_sample_prob_table
self.train_ds = strategy.experimental_distribute_dataset(self.get_train_ds(from_valid=False))
self.train_ds_from_valid = strategy.experimental_distribute_dataset(self.get_train_ds(from_valid=True))
self.train_ds_from_valid_only_last = strategy.experimental_distribute_dataset(self.get_train_ds(from_valid=True, only_last=True))
self.valid_ds = strategy.experimental_distribute_dataset(self.get_valid_ds(debug=False))
self.valid_ds_debug = self.get_valid_ds(debug=True)
 def toJSON(self):
config = json.loads(self.config.toJSON())
 train_config = json.loads(self.train_config.toJSON())
 for k, v in train_config.items() :
config[k] = v
return json.dumps(config, sort_keys=False, ensure_ascii=False, indent=4)
 def get_train_ds(self, from_valid=False, only_last=False):
only_last = False
train_raw_ds = tf.data.TFRecordDataset(train_tfrec_paths, num_parallel_reads=self.train_config.num_parallel_reads)
 train_raw_ds = train_raw_ds.map(parse_train_example, num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 train_raw_ds = train_raw_ds.map(lambda example: add_aggregated_historical_information(example), num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 if self.train_config.valid_fold in [0, 1, 2, 3]:
train_valid_split_indices = load_data(train_valid_split_indices_paths[self.train_config.valid_fold])
 train_valid_split_table = convert_split_index_dict(train_valid_split_indices)
 reduced_raw_ds = split_train_ds(train_raw_ds, train_valid_split_table, num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 else:
reduced_raw_ds = train_raw_ds
reduced_raw_ds_augmented = reduced_raw_ds.flat_map(
 lambda x: tf.data.Dataset.from_tensors(x ).repeat(
 tf.cast(
 tf.math.ceil(tf.cast(x['seq_len'], tf.float32)/ self.train_config.window_size),
dtype=tf.int64 
)
)
)
 batched_raw_ds = reduced_raw_ds_augmented.repeat().shuffle(self.train_config.shuffle_buf_size, seed=self.train_config.seed ).apply(tf.data.experimental.dense_to_ragged_batch(self.train_config.batch_size))
batched_ds = random_subseqs_from_batched_raw_ds(batched_raw_ds, window_size=self.train_config.window_size, only_last=only_last, seed=self.train_config.seed, num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 train_ds = prepare_training_dataset(batched_ds, generative=self.config.generative, use_abs_pos=self.config.use_abs_pos, num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 train_ds = train_ds.prefetch(8)
 return train_ds
def get_valid_ds(self, debug=False):
p = valid_tfrec_paths
num_parallel_reads = self.train_config.num_parallel_reads
num_parallel_calls = self.train_config.num_parallel_calls
deterministic = self.train_config.deterministic
valid_user_id_tensor = tf.constant(self.valid_user_ids, dtype=tf.int64)
 repeat_tensor = tf.ones_like(valid_user_id_tensor, dtype=tf.int32)
 initializer = tf.lookup.KeyValueTensorInitializer(valid_user_id_tensor, repeat_tensor)
 valid_users_table = tf.lookup.StaticHashTable(
 initializer, default_value=tf.constant(0, dtype=tf.int32), name=None 
)
 valid_raw_ds = tf.data.TFRecordDataset(p, num_parallel_reads=num_parallel_reads)
 valid_raw_ds = valid_raw_ds.map(parse_train_example_with_valid_info, num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 valid_raw_ds.filter(lambda x: tf.math.equal(valid_users_table.lookup(x['user_id']), 1))
valid_ds = prepare_validation_dataset(valid_raw_ds, batch_size=self.train_config.pred_batch_size, window_size=self.train_config.window_size, generative=self.config.generative, use_abs_pos=self.config.use_abs_pos, num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 valid_ds = valid_ds.prefetch(8)
 def get_train_objs(self, lr=LR, end_lr=END_LR, warmup_steps=WARMUP_STEPS):
self.train_config.lr = lr
self.train_config.end_lr = end_lr,
self.train_config.warmup_steps = warmup_steps
with strategy.scope() :
predictor = TFEdFormerAnswerPredictionModel(self.config)
 _lr = LinearLR(total_steps=self.train_config.n_training_steps, lr=lr, end_lr=end_lr, warmup_steps=warmup_steps)
 optimizer = tf.keras.optimizers.Adam(
 _lr,
beta_1=tf.Variable(0.9),
beta_2=tf.Variable(0.999),
epsilon=tf.Variable(1e-8)
)
 optimizer.iterations
optimizer.decay = tf.Variable(0.0)
 if self.config.use_softmax:
loss_obj = tf.keras.losses.SparseCategoricalCrossentropy(
 from_logits=True, reduction=tf.keras.losses.Reduction.NONE,
name='sparse_categorical_crossentropy_for_correctness' 
)
 else:
loss_obj = tf.keras.losses.BinaryCrossentropy(
 from_logits=True, label_smoothing=0, reduction=tf.keras.losses.Reduction.NONE,
name='binary_crossentropy_for_correctness' 
)
 loss_obj_answer = tf.keras.losses.SparseCategoricalCrossentropy(
 from_logits=True, reduction=tf.keras.losses.Reduction.NONE,
name='sparse_categorical_crossentropy_for_answer' 
)
 loss_metric = tf.keras.metrics.Sum()
acc_metric = tf.keras.metrics.BinaryAccuracy()
auc_metric = tf.keras.metrics.AUC(num_thresholds=2000)
 loss_metric_answer = tf.keras.metrics.Sum()
acc_metric_answer = tf.keras.metrics.SparseCategoricalAccuracy()
metrics =(loss_metric, acc_metric, auc_metric, loss_metric_answer, acc_metric_answer)
 return(predictor, optimizer, loss_obj, loss_obj_answer, metrics)
 def train_valid(
 self, predictor, optimizer, ckpt_manager,
loss_obj, loss_obj_answer, metrics,
last_epoch=0,
from_valid=False, valid_epochs=None, only_valid=False 
):
use_user_answer_loss = tf.cast(tf.constant(self.config.use_user_answer_loss), dtype=DTYPE)
 batch_size = self.train_config.batch_size
seq_len = self.train_config.window_size
encoder_decoder = tf.constant(self.config.model_type=='ed', dtype=tf.int32)
 generative = tf.constant(self.config.generative, dtype=tf.int32)
 allow_bundle_atten = tf.constant(self.config.allow_bundle_atten, dtype=tf.int32)
 loss_metric, acc_metric, auc_metric, loss_metric_answer, acc_metric_answer = metrics
@tf.function(
 input_signature=get_input_signatures(int(batch_size / strategy.num_replicas_in_sync), seq_len, valid=False)
)
 def train_step(train_batch):
training = tf.constant(1, dtype=tf.int32)
 c_mask, r_mask, r_c_mask, c_r_mask = get_attention_masks(train_batch, training, encoder_decoder, generative, allow_bundle_atten)
 targets = train_batch['target']
answer_targets = train_batch['answer_target']
pred_mask = targets != NON_TARGET_ID
pred_indices = tf.where(pred_mask)
 selected_targets = tf.gather_nd(targets, pred_indices)
 selected_answer_targets = tf.gather_nd(answer_targets, pred_indices)
 with tf.GradientTape() as tape: 
(logits, answer_logits, c_outputs, r_outputs)= predictor(
 train_batch,
c_mask,
r_mask,
r_c_mask,
c_r_mask,
output_attentions=False, output_hidden_states=False, training=True 
)
 selected_logits = tf.gather_nd(logits, pred_indices)
 selected_answer_logits = tf.gather_nd(answer_logits, pred_indices)
 losses = loss_obj(selected_targets[:, tf.newaxis], selected_logits)
 answer_losses = loss_obj_answer(selected_answer_targets, selected_answer_logits)
 total_loss = tf.math.reduce_sum(losses)
 total_answer_loss = tf.math.reduce_sum(answer_losses)
 loss =(total_loss + total_answer_loss * use_user_answer_loss)/ tf.cast(train_batch['nb_pred_places'][0], dtype=DTYPE)
 grads = tape.gradient(loss, predictor.trainable_variables)
 optimizer.apply_gradients(zip(grads, predictor.trainable_variables))
if self.config.use_softmax:
preds = tf.math.softmax(logits)
 selected_preds = tf.math.softmax(selected_logits)
 else:
preds = tf.math.sigmoid(logits)
 selected_preds = tf.math.sigmoid(selected_logits)
 answer_preds = tf.math.softmax(answer_logits, axis=-1)
 selected_answer_preds = tf.math.softmax(selected_answer_logits, axis=-1)
 loss_metric.update_state(total_loss)
 acc_metric.update_state(selected_targets[:, tf.newaxis], selected_preds[:, -1:])
 auc_metric.update_state(selected_targets, selected_preds[:, -1])
 loss_metric_answer.update_state(total_answer_loss)
 acc_metric_answer.update_state(selected_answer_targets, selected_answer_preds)
 batch_size = None
@tf.function(
 input_signature=get_input_signatures(batch_size, seq_len, valid=True)
)
 def valid_step(valid_batch):
training = tf.constant(0, dtype=tf.int32)
 c_mask, r_mask, r_c_mask, c_r_mask = get_attention_masks(valid_batch, training, encoder_decoder, generative, allow_bundle_atten)
 logits, answer_logits, _, _ = predictor(
 valid_batch,
c_mask,
r_mask,
r_c_mask,
c_r_mask,
output_attentions=False, output_hidden_states=False, training=False 
)
 pred_time_mask = valid_batch['pred_time_mask'] * tf.cast(valid_batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 targets = valid_batch['target']
answer_targets = valid_batch['answer_target']
pred_time_mask = pred_time_mask[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:]
targets = targets[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:]
answer_targets = answer_targets[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:]
logits = logits[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:, :]
answer_logits = answer_logits[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:, :]
row_ids = valid_batch['row_id'][:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:]
user_ids = valid_batch['user_id'][:, tf.newaxis] * tf.ones_like(row_ids, dtype=tf.int64)
 targets = targets * pred_time_mask + NON_TARGET_ID *(1 - pred_time_mask)
 answer_targets = answer_targets * pred_time_mask + NON_TARGET_ID *(1 - pred_time_mask)
 pred_mask = targets != NON_TARGET_ID
pred_indices = tf.where(pred_mask)
 selected_targets = tf.gather_nd(targets, pred_indices)
 selected_logits = tf.gather_nd(logits, pred_indices)
 if self.config.use_softmax:
preds = tf.math.softmax(logits)
 selected_preds = tf.math.softmax(selected_logits)
 else:
preds = tf.math.sigmoid(logits)
 selected_preds = tf.math.sigmoid(selected_logits)
 answer_preds = tf.math.softmax(answer_logits, axis=-1)
 selected_answer_targets = tf.gather_nd(answer_targets, pred_indices)
 selected_answer_logits = tf.gather_nd(answer_logits, pred_indices)
 selected_answer_preds = tf.math.softmax(selected_answer_logits, axis=-1)
 losses = loss_obj(selected_targets[:, tf.newaxis], selected_logits)
 answer_losses = loss_obj_answer(selected_answer_targets, selected_answer_logits)
 total_loss = tf.math.reduce_sum(losses)
 total_answer_loss = tf.math.reduce_sum(answer_losses)
 loss =(total_loss + total_answer_loss * use_user_answer_loss)/ tf.cast(tf.math.reduce_mean(valid_batch['nb_pred_places']), dtype=DTYPE)
 loss_metric.update_state(total_loss)
 acc_metric.update_state(selected_targets[:, tf.newaxis], selected_preds[:, -1:])
 auc_metric.update_state(selected_targets, selected_preds[:, -1])
 loss_metric_answer.update_state(total_answer_loss)
 acc_metric_answer.update_state(selected_answer_targets, selected_answer_preds)
 return targets, preds, row_ids, user_ids, pred_mask, answer_targets, answer_preds
@tf.function
def dist_train_step(dist_train_batch):
strategy.run(train_step, args=(dist_train_batch,))
@tf.function
def dist_valid_step(dist_valid_batch):
targets, preds, row_ids, user_ids, pred_mask, answer_targets, answer_preds = strategy.run(valid_step, args=(dist_valid_batch,))
return targets, preds, row_ids, user_ids, pred_mask, answer_targets, answer_preds
@tf.function
def dist_train_multi_steps(dist_train_iter):
total_nb_pred_places = tf.constant(0.0, dtype=DTYPE)
 for _ in tf.range(self.train_config.steps_per_call):
dist_train_batch = next(dist_train_iter)
 dist_train_step(dist_train_batch)
 if type(dist_train_batch['nb_pred_places'])== tf.Tensor:
nb_pred_places = tf.math.reduce_mean(dist_train_batch['nb_pred_places'])
 else:
nb_pred_places = tf.math.reduce_mean(tf.concat(dist_train_batch['nb_pred_places'].values, axis=0))
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 return total_nb_pred_places
@tf.function
def dist_valid_multi_steps(dist_valid_iter):
total_nb_pred_places = tf.constant(0.0, dtype=DTYPE)
 t_arr_1 = tf.TensorArray(tf.int32, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_2 = tf.TensorArray(DTYPE, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_3 = tf.TensorArray(tf.bool, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_4 = tf.TensorArray(tf.int64, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_5 = tf.TensorArray(tf.int64, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_6 = tf.TensorArray(tf.int32, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_7 = tf.TensorArray(DTYPE, size=0, dynamic_size=True, clear_after_read=False)
 for step_idx in tf.range(self.train_config.steps_per_call):
dist_valid_batch = next(dist_valid_iter)
 _targets, _preds, _row_ids, _user_ids, _pred_mask, _answer_targets, _answer_preds = dist_valid_step(dist_valid_batch)
 if type(dist_valid_batch['nb_pred_places'])== tf.Tensor:
nb_pred_places = tf.math.reduce_mean(dist_valid_batch['nb_pred_places'])
 else:
_targets = tf.concat(_targets.values, axis=0)
 _preds = tf.concat(_preds.values, axis=0)
 _pred_mask = tf.concat(_pred_mask.values, axis=0)
 _row_ids = tf.concat(_row_ids.values, axis=0)
 _user_ids = tf.concat(_user_ids.values, axis=0)
 _answer_targets = tf.concat(_answer_targets.values, axis=0)
 _answer_preds = tf.concat(_answer_preds.values, axis=0)
 nb_pred_places = tf.math.reduce_mean(tf.concat(dist_valid_batch['nb_pred_places'].values, axis=0))
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 t_arr_1 = t_arr_1.write(step_idx, _targets)
 t_arr_2 = t_arr_2.write(step_idx, _preds)
 t_arr_3 = t_arr_3.write(step_idx, _pred_mask)
 t_arr_4 = t_arr_4.write(step_idx, _row_ids)
 t_arr_5 = t_arr_5.write(step_idx, _user_ids)
 t_arr_6 = t_arr_6.write(step_idx, _answer_targets)
 t_arr_7 = t_arr_7.write(step_idx, _answer_preds)
 targets = t_arr_1.concat()
preds = t_arr_2.concat()
pred_mask = t_arr_3.concat()
row_ids = t_arr_4.concat()
user_ids = t_arr_5.concat()
answer_targets = t_arr_6.concat()
answer_preds = t_arr_7.concat()
return targets, preds, row_ids, user_ids, pred_mask, total_nb_pred_places, answer_targets, answer_preds
@tf.function
def dist_valid_multi_steps_last_call(dist_valid_iter):
total_nb_pred_places = tf.constant(0.0, dtype=DTYPE)
 t_arr_1 = tf.TensorArray(tf.int32, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_2 = tf.TensorArray(DTYPE, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_3 = tf.TensorArray(tf.bool, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_4 = tf.TensorArray(tf.int64, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_5 = tf.TensorArray(tf.int64, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_6 = tf.TensorArray(tf.int32, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_7 = tf.TensorArray(DTYPE, size=0, dynamic_size=True, clear_after_read=False)
 for step_idx in tf.range(self.train_config.n_steps_in_last_valid_call):
dist_valid_batch = next(dist_valid_iter)
 _targets, _preds, _row_ids, _user_ids, _pred_mask, _answer_targets, _answer_preds = dist_valid_step(dist_valid_batch)
 if type(dist_valid_batch['nb_pred_places'])== tf.Tensor:
nb_pred_places = tf.math.reduce_mean(dist_valid_batch['nb_pred_places'])
 else:
_targets = tf.concat(_targets.values, axis=0)
 _preds = tf.concat(_preds.values, axis=0)
 _pred_mask = tf.concat(_pred_mask.values, axis=0)
 _row_ids = tf.concat(_row_ids.values, axis=0)
 _user_ids = tf.concat(_user_ids.values, axis=0)
 _answer_targets = tf.concat(_answer_targets.values, axis=0)
 _answer_preds = tf.concat(_answer_preds.values, axis=0)
 nb_pred_places = tf.math.reduce_mean(tf.concat(dist_valid_batch['nb_pred_places'].values, axis=0))
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 t_arr_1 = t_arr_1.write(step_idx, _targets)
 t_arr_2 = t_arr_2.write(step_idx, _preds)
 t_arr_3 = t_arr_3.write(step_idx, _pred_mask)
 t_arr_4 = t_arr_4.write(step_idx, _row_ids)
 t_arr_5 = t_arr_5.write(step_idx, _user_ids)
 t_arr_6 = t_arr_6.write(step_idx, _answer_targets)
 t_arr_7 = t_arr_7.write(step_idx, _answer_preds)
 targets = t_arr_1.concat()
preds = t_arr_2.concat()
pred_mask = t_arr_3.concat()
row_ids = t_arr_4.concat()
user_ids = t_arr_5.concat()
answer_targets = t_arr_6.concat()
answer_preds = t_arr_7.concat()
return targets, preds, row_ids, user_ids, pred_mask, total_nb_pred_places, answer_targets, answer_preds
n_training_steps_per_epoch = self.train_config.n_training_steps_per_epoch
n_training_steps = self.train_config.n_epochs * n_training_steps_per_epoch
if MAX_TRAIN_ITER_STEPS is not None:
n_training_steps_per_epoch = MAX_TRAIN_ITER_STEPS
n_training_steps = self.train_config.n_epochs * n_training_steps_per_epoch
self.train_config.n_training_steps_per_epoch = n_training_steps_per_epoch
self.train_config.n_training_steps = n_training_steps
with open('train_config.json', 'w', encoding='UTF-8')as fp:
json.dump(json.loads(self.toJSON()), fp, ensure_ascii=False, indent=4)
 if not IS_KAGGLE:
!gsutil cp -r './train_config.json' '{self.train_config.ckpt_path}'
n_epochs = self.train_config.n_epochs
n_training_examples = self.train_config.n_training_examples
if from_valid:
n_epochs = 3
n_training_examples = int(2380588 / self.train_config.window_size)
 n_training_steps_per_epoch = int(n_training_examples / self.train_config.batch_size)
 n_training_steps = n_epochs * n_training_steps_per_epoch
print(f'n_epochs: {n_epochs}')
 print(f'n_training_examples: {n_training_examples}')
 print(f'n_training_steps_per_epoch: {n_training_steps_per_epoch}')
 print(f'n_training_steps: {n_training_steps}')
 print(f'n_valid_examples: {self.train_config.n_valid_examples}')
 print(f'n_valid_steps: {self.train_config.n_valid_steps}')
 training_history = dict()
def train(last_epoch=0, from_valid=False, valid_epochs=None):
start_epoch = last_epoch + 1
end_epoch = self.train_config.n_epochs + 1
train_ds = self.train_ds
if from_valid:
end_epoch = start_epoch + 3
train_ds = self.train_ds_from_valid
dist_train_iter = iter(train_ds)
 for epoch in range(start_epoch, end_epoch):
training_history[epoch] = {
'loss': [],
'acc': [],
'auc': [],
'answer_loss': [],
'answer_acc': [],
'timing': []
}
n_steps = 0
n_preds = 0
n_last_preds = 0
total_nb_pred_places = 0.0
start_epoch_t = datetime.datetime.now()
for call_idx in range(self.train_config.n_training_calls_per_epoch):
start_t = datetime.datetime.now()
nb_pred_places = dist_train_multi_steps(dist_train_iter)
 n_steps += self.train_config.steps_per_call
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 avg_loss =(loss_metric.result() / total_nb_pred_places ).numpy()
acc = acc_metric.result().numpy()
auc = auc_metric.result().numpy()
avg_answer_loss =(loss_metric_answer.result() / total_nb_pred_places ).numpy()
answer_acc = acc_metric_answer.result().numpy()
if(call_idx + 1)% max(1, self.train_config.n_training_calls_per_epoch // 100)== 0:
training_history[epoch]['loss'].append(float(avg_loss))
training_history[epoch]['acc'].append(float(acc))
training_history[epoch]['auc'].append(float(auc))
training_history[epoch]['answer_loss'].append(float(avg_answer_loss))
training_history[epoch]['answer_acc'].append(float(answer_acc))
elapsed =(datetime.datetime.now() - start_t ).total_seconds()
if(call_idx + 1)% max(1, self.train_config.n_training_calls_per_epoch // 10)== 0:
print(f'epoch: {epoch} - step: {n_steps}')
 print(f'timing per step: {elapsed / self.train_config.steps_per_call}')
 print(f'loss: {avg_loss}')
 print(f'acc: {acc}')
 print(f'auc: {auc}')
 print(f'answer_loss: {avg_answer_loss}')
 print(f'answer_acc: {answer_acc}')
 print(f'lr: {optimizer._decayed_lr(DTYPE)}')
 print('-' * 80)
 end_epoch_t = datetime.datetime.now()
elapsed_epoch_t =(end_epoch_t - start_epoch_t ).total_seconds()
training_history[epoch]['timing'] = elapsed_epoch_t
training_history[epoch]['train_loss'] = float(avg_loss)
 training_history[epoch]['train_acc'] = float(acc)
 training_history[epoch]['train_auc'] = float(auc)
 training_history[epoch]['train_answer_loss'] = float(avg_answer_loss)
 training_history[epoch]['train_answer_acc'] = float(answer_acc)
 if IS_KAGGLE and tpu:
ckpt_manager.checkpoint.save(
 file_prefix=ckpt_manager.directory + 'ckpt', options=tf.train.CheckpointOptions(experimental_io_device="/job:localhost")
)
 else:
ckpt_manager.save()
print(f'epoch: {epoch} - train')
 print(f'train loss: {avg_loss}')
 print(f'train acc: {acc}')
 print(f'train auc: {auc}')
 print(f'train answer loss: {avg_answer_loss}')
 print(f'train answer acc: {answer_acc}')
 print('-' * 80)
 loss_metric.reset_states()
acc_metric.reset_states()
auc_metric.reset_states()
loss_metric_answer.reset_states()
acc_metric_answer.reset_states()
fn = f'training_history-{epoch}.json'
with open(fn, 'w', encoding='UTF-8')as fp:
json.dump(training_history, fp, ensure_ascii=False, indent=4)
 if not IS_KAGGLE:
!gsutil cp -r './{fn}' '{self.train_config.ckpt_path}'
!rm -rf './{fn}'
if valid_epochs is None:
valid_epochs = [self.train_config.n_epochs]
if epoch in valid_epochs:
valid(epoch)
 def valid(epoch):
if epoch not in training_history:
training_history[epoch] = {
'loss': [],
'acc': [],
'auc': [],
'answer_loss': [],
'answer_acc': [],
'timing': []
}
valid_user_ids = []
valid_row_ids = []
valid_targets = []
valid_preds = []
valid_answer_targets = []
valid_answer_preds = []
n_steps = 0
total_nb_pred_places = 0.0
dist_valid_iter = iter(self.valid_ds)
 start_valid = datetime.datetime.now()
n_valid_calls = self.train_config.n_valid_calls
if self.train_config.n_steps_in_last_valid_call > 0:
n_valid_calls -= 1
for call_idx in range(n_valid_calls):
start_t = datetime.datetime.now()
targets, preds, row_ids, user_ids, pred_mask, nb_pred_places, answer_targets, answer_preds = dist_valid_multi_steps(dist_valid_iter)
 n_steps += self.train_config.steps_per_call
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 pred_indices = tf.where(pred_mask)
 selected_targets = tf.gather_nd(targets, pred_indices)
 selected_preds = tf.gather_nd(preds, pred_indices)
 selected_answer_targets = tf.gather_nd(answer_targets, pred_indices)
 selected_answer_preds = tf.gather_nd(answer_preds, pred_indices)
 selected_row_ids = tf.gather_nd(row_ids, pred_indices)
 selected_user_ids = tf.gather_nd(user_ids, pred_indices)
 avg_loss =(loss_metric.result() / total_nb_pred_places ).numpy()
acc = acc_metric.result().numpy()
auc = auc_metric.result().numpy()
avg_answer_loss =(loss_metric_answer.result() / total_nb_pred_places ).numpy()
answer_acc = acc_metric_answer.result().numpy()
elapsed =(datetime.datetime.now() - start_t ).total_seconds()
if call_idx % 1 == 0:
print(f'epoch: {epoch} - valid step: {n_steps}')
 print(f'valid timing per steps: {elapsed / self.train_config.steps_per_call}')
 print(f'valid loss: {avg_loss}')
 print(f'valid acc: {acc}')
 print(f'valid auc: {auc}')
 print(f'valid answer loss: {avg_answer_loss}')
 print(f'valid answer acc: {answer_acc}')
 print('-' * 80)
 valid_row_ids.extend(selected_row_ids.numpy().tolist())
 valid_user_ids.extend(selected_user_ids.numpy().tolist())
 valid_targets.extend(selected_targets.numpy().tolist())
 valid_preds.extend(selected_preds[:, -1].numpy().tolist())
 valid_answer_targets.extend(selected_answer_targets.numpy().tolist())
 valid_answer_preds.extend(selected_answer_preds.numpy().tolist())
 if self.train_config.n_steps_in_last_valid_call > 0:
start_t = datetime.datetime.now()
targets, preds, row_ids, user_ids, pred_mask, nb_pred_places, answer_targets, answer_preds = dist_valid_multi_steps_last_call(dist_valid_iter)
 n_steps += self.train_config.n_steps_in_last_valid_call
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 pred_indices = tf.where(pred_mask)
 selected_targets = tf.gather_nd(targets, pred_indices)
 selected_preds = tf.gather_nd(preds, pred_indices)
 selected_answer_targets = tf.gather_nd(answer_targets, pred_indices)
 selected_answer_preds = tf.gather_nd(answer_preds, pred_indices)
 selected_row_ids = tf.gather_nd(row_ids, pred_indices)
 selected_user_ids = tf.gather_nd(user_ids, pred_indices)
 avg_loss =(loss_metric.result() / total_nb_pred_places ).numpy()
acc = acc_metric.result().numpy()
auc = auc_metric.result().numpy()
avg_answer_loss =(loss_metric_answer.result() / total_nb_pred_places ).numpy()
answer_acc = acc_metric_answer.result().numpy()
elapsed =(datetime.datetime.now() - start_t ).total_seconds()
valid_row_ids.extend(selected_row_ids.numpy())
 valid_user_ids.extend(selected_user_ids.numpy())
 valid_preds.extend(selected_preds[:, -1].numpy())
 valid_answer_targets.extend(selected_answer_targets.numpy())
 valid_answer_preds.extend(selected_answer_preds.numpy())
 end_valid = datetime.datetime.now()
elapsed_valid =(end_valid - start_valid ).total_seconds()
training_history[epoch]['valid_timing'] = elapsed_valid
training_history[epoch]['valid_loss'] = float(avg_loss)
 training_history[epoch]['valid_acc'] = float(acc)
 training_history[epoch]['valid_auc'] = float(auc)
 training_history[epoch]['valid_answer_loss'] = float(avg_answer_loss)
 training_history[epoch]['valid_answer_acc'] = float(answer_acc)
 print(f'epoch: {epoch} - valid')
 print(f'valid loss: {avg_loss}')
 print(f'valid acc: {acc}')
 print(f'valid auc: {auc}')
 print(f'valid answer loss: {avg_answer_loss}')
 print(f'valid answer acc: {answer_acc}')
 print('=' * 80)
 loss_metric.reset_states()
acc_metric.reset_states()
auc_metric.reset_states()
loss_metric_answer.reset_states()
acc_metric_answer.reset_states()
valid_submission_dict = {
'row_ids': valid_row_ids,
'user_ids': valid_user_ids,
'targets': valid_targets,
'preds': valid_preds
}
if self.config.use_user_answer_loss:
valid_answer_preds = np.array(valid_answer_preds)
 valid_submission_dict['answer_targets'] = valid_answer_targets
valid_submission_dict['answer_preds_0'] = valid_answer_preds[:, 0]
valid_submission_dict['answer_preds_1'] = valid_answer_preds[:, 1]
valid_submission_dict['answer_preds_2'] = valid_answer_preds[:, 2]
valid_submission_dict['answer_preds_3'] = valid_answer_preds[:, 3]
fn = f'training_history-{epoch}.json'
if only_valid:
fn = f'training_history-{epoch}-only-valid.json'
with open(fn, 'w', encoding='UTF-8')as fp:
json.dump(training_history, fp, ensure_ascii=False, indent=4)
 if not IS_KAGGLE:
!gsutil cp -r './{fn}' '{self.train_config.ckpt_path}'
!rm -rf './{fn}'
if only_valid:
valid(last_epoch)
 else:
train(last_epoch, from_valid, valid_epochs )<feature_engineering>
def lgb_modelfit_nocv(params,dtrain,dvalid,predictors,target='target',objective='binary',metrics='auc',feval=None,early_stopping_rounds=20,
num_boost_round=3000,verbose_eval=10,categorical_features=None):
lgb_params={
'boosting_type':'gbdt',
'objective':objective,
'metric':metrics,
'learning_rate':0.01,
'num_leaves':31,
'max_depth':-1,
'min_child_samples':20,
'max_bin':255,
'subsample':0.6,
'subsample_freq':0,
'colsample_bytree':0.3,
'min_child_weight':5,
'subsample_for_bin':200000,
'min_split_gain':0,
'reg_alpha':0,
'reg_lambda':0,
'nthread':8,
'verbose':0
}
lgb_params.update(params)
 print('preparing valildation datasets')
 xgtrain=lgb.Dataset(dtrain[predictors].values,label=dtrain[target].values,feature_name=predictors,
categorical_feature=categorical_features)
 xgvalid=lgb.Dataset(dvalid[predictors].values,label=dvalid[target].values,feature_name=predictors,
categorical_feature=categorical_features)
 evals_results={}
bst1=lgb.train(lgb_params,xgtrain,valid_sets=[xgtrain,xgvalid],valid_names=['trian','valid'],
evals_result=evals_results,num_boost_round=num_boost_round,early_stopping_rounds=early_stopping_rounds,
verbose_eval=10,feval=feval)
 n_estimators=bst1.best_iteration
print('Model Report')
 print('n_estimators:',n_estimators)
 print(metrics+':',evals_results['valid'][metrics][n_estimators-1])
 return bst1
if not IS_KAGGLE:
config = EdFormerConfig(model_type='ed', model_desc='dummy')
 train_config = TrainConfig()
train_manager = Train_Manager(config, train_config )<train_model>
path='.. /input/'
dtypes={
'ip' : 'uint32',
'app' : 'uint16',
'device' : 'uint16',
'os' : 'uint16',
'channel' : 'uint16',
'is_attributed' : 'uint8',
'click_id' : 'uint32'
}
print('load train...')
 train_df = pd.read_csv(path+"train.csv",skiprows=range(1,149903891), nrows=40000000, dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'is_attributed'])
 print('load test...')
 test_df = pd.read_csv(path+"test.csv", dtype=dtypes, usecols=['ip','app','device','os', 'channel', 'click_time', 'click_id'] )
if not IS_KAGGLE:
train_manager.train_ds<set_options>
len_train = len(train_df)
 train_df=train_df.append(test_df )
if not IS_KAGGLE:
train_manager.valid_ds<set_options>
del test_df
gc.collect()
print('data prep...')
 train_df['hour'] = pd.to_datetime(train_df.click_time ).dt.hour.astype('uint8')
 train_df['day'] = pd.to_datetime(train_df.click_time ).dt.day.astype('uint8')
 gc.collect()
if not IS_KAGGLE:
train_manager.valid_ds_debug<categorify>
print('group by...')
 gp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','hour'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'qty'} )
def convert_dt(_dt, pred=False, unique_question_id_train=None, unique_lecture_id_train=None):

 _dt[dt.f.prior_question_elapsed_time] = dt.float32
_dt[dt.f.prior_question_elapsed_time == None, 'prior_question_elapsed_time'] = -1.0
_dt[dt.f.prior_question_had_explanation] = dt.int8
_dt[dt.f.prior_question_had_explanation == None, 'prior_question_had_explanation'] = -1
_dt[dt.f.content_type_id] = dt.int8
if pred:
_dt['answered_correctly'] = MASK_TOKEN
_dt['user_answer'] = MASK_TOKEN
try:
del _dt['prior_group_answers_correct']
del _dt['prior_group_responses']
except:
pass
if DEBUG:
if unique_question_id_train is not None:
assert set(_dt[dt.f.content_type_id == 0, 'content_id'].to_list() [0] ).issubset(unique_question_id_train)
 if unique_lecture_id_train is not None:
assert set(_dt[dt.f.content_type_id == 1, 'content_id'].to_list() [0] ).issubset(unique_lecture_id_train)
 def convert_df_to_dt(df, test=False, unique_question_id_train=None, unique_lecture_id_train=None):

 if DEBUG:
_question_ids = set(df[df['content_type_id'] == 0]['content_id'].values.tolist())
 _lecture_ids = set(df[df['content_type_id'] == 1]['content_id'].values.tolist())
 if unique_question_id_train is not None:
assert _question_ids.issubset(unique_question_id_train)
 if unique_lecture_id_train is not None:
assert _lecture_ids.issubset(unique_lecture_id_train)
 _dt = dt.Frame(df.astype({"prior_question_had_explanation": float}))
if test:
prior_group_answers_correct = eval(_dt[0, 'prior_group_answers_correct'])
 prior_group_responses = eval(_dt[0, 'prior_group_responses'])
 if prior_group_answers_correct is None:
prior_group_answers_correct = []
if prior_group_responses is None:
prior_group_responses = []
if DEBUG:
assert type(prior_group_answers_correct)== list
assert type(prior_group_responses)== list
convert_dt(_dt, pred=test, unique_question_id_train=unique_question_id_train, unique_lecture_id_train=unique_lecture_id_train)
 if test:
return _dt, prior_group_answers_correct, prior_group_responses
else:
return _dt<categorify>
print('merge...')
 train_df = train_df.merge(gp, on=['ip','day','hour'], how='left')
 print("vars and data type: ")
 train_df.info()
def sample_sequence_lengths(upper_bound, sample_size, expected_mean, seed):
seq_lengths = tf.range(1, upper_bound+1)
 seq_len_mean = tf.math.reduce_mean(tf.cast(seq_lengths, dtype=DTYPE))
N = sample_size
mu = expected_mean
sigma = mu / 2.5
normal_dist = tfp.distributions.Normal(
 loc=mu, scale=sigma 
)
 seq_len_sample = normal_dist.sample(sample_shape=[N], seed=seed)
 avg_selected_seq_len = tf.math.reduce_mean(seq_len_sample)
 mask = tf.math.logical_and(2 * mu >= seq_len_sample, seq_len_sample >= 1)
 seq_len_sample = tf.gather_nd(seq_len_sample, indices=tf.where(mask))
seq_len_sample = tf.cast(tf.math.round(seq_len_sample), dtype=tf.int32)
 uniform_dist = tfp.distributions.Uniform(
 low=0, high=len(seq_lengths)
)
 selected_indices = tf.cast(uniform_dist.sample(sample_shape=[N - seq_len_sample.shape[0]], seed=seed), dtype=tf.int32)
 seq_len_sample_2 = tf.gather(seq_lengths, indices=selected_indices)
 seq_len_sample_2 = seq_len_sample_2
seq_len_sample = tf.concat([seq_len_sample, seq_len_sample_2], axis=0)
 return seq_len_sample<choose_model_class>
print('grouping by ip-app combination...')
 gp = train_df[['ip', 'app', 'channel']].groupby(by=['ip', 'app'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_count'})
 train_df = train_df.merge(gp, on=['ip','app'], how='left')
 del gp
gc.collect()
class User_Interaction_Manager(collections.abc.Iterator):
def __init__(self, user_id, user_valid_info, train_dt):
self.user_id = user_id
self.user_valid_info = user_valid_info
self._n_remaining_blocks = self.n_blocks
self._current_block_starting_index = 0
self._train_dt = train_dt
@property
def n_blocks(self):
return self.user_valid_info['bundle_info']['n_blocks']
@property
def n_remaining_blocks(self):
return self._n_remaining_blocks
def _get_next_block_starting_index(self):
if self._current_block_starting_index not in self.user_valid_info['bundle_info']['block_starting_index_dict']:
next_index = self._current_block_starting_index + 1
else:
next_index = self.user_valid_info['bundle_info']['block_starting_index_dict'][self._current_block_starting_index]
if DEBUG:
assert 0 <= next_index <= self.user_valid_info['seq_len']
return next_index
def _get_user_dt(self, next_block_starting_index):
row_start = self.user_valid_info['row_start']
block_row_start = row_start + self._current_block_starting_index
block_row_end = row_start + next_block_starting_index - 1
if DEBUG:
assert block_row_end <= self.user_valid_info['row_end']
user_dt = self._get_user_dt_from_row_indices(block_row_start, block_row_end)
 return user_dt
def _get_user_dt_from_row_indices(self, row_start, row_end):
user_dt = self._train_dt[row_start:(row_end + 1), :]
return user_dt
def has_next(self):
return not(self._n_remaining_blocks <= 0 or self._current_block_starting_index >= self.user_valid_info['seq_len'])
 def __next__(self):
if self._n_remaining_blocks <= 0:
raise StopIteration
if self._current_block_starting_index >= self.user_valid_info['seq_len']:
raise StopIteration
next_block_starting_index = self._get_next_block_starting_index()
user_dt = self._get_user_dt(next_block_starting_index)
 self._n_remaining_blocks -= 1
self._current_block_starting_index = next_block_starting_index
if DEBUG:
if self._n_remaining_blocks <= 0:
assert self._current_block_starting_index == self.user_valid_info['seq_len']
if self._current_block_starting_index >= self.user_valid_info['seq_len']:
assert self._n_remaining_blocks == 0
assert user_dt.nrows > 0
return user_dt
class Valid_Iter_Manager(Pred_Iter_Manager):
def __init__(self, valid_info, train_dt, unique_question_id_train=None, unique_lecture_id_train=None):
super(Valid_Iter_Manager, self ).__init__(unique_question_id_train, unique_lecture_id_train)
 self._valid_info = valid_info
self._n_users = len(set(self._valid_info.keys()))
_n_blocks = 0
for user_id, user_valid_info in self._valid_info.items() :
_n_blocks += user_valid_info['bundle_info']['n_blocks']
self._n_blocks = _n_blocks
self._remaining_users = set(self._valid_info.keys())
 self._n_remaining_blocks = self._n_blocks
self._current_answered_correctly = None
self._current_user_answer = None
self._user_manager_dict = dict()
for user_id, user_valid_info in self._valid_info.items() :
self._user_manager_dict[user_id] = User_Interaction_Manager(user_id, user_valid_info, train_dt)
 self.train_dt = train_dt
self._split_indices = {user_id: valid_info_for_user['split_index'] for user_id, valid_info_for_user in self._valid_info.items() }
@classmethod
def create(cls, valid_info_path, train_dt_path_or_obj, unique_question_id_train_path=None, unique_lecture_id_train_path=None):
args = []
if valid_info_path is None:
valid_info = {}
else:
valid_info = load_data(valid_info_path)
 args.append(valid_info)
 for p in [unique_question_id_train_path, unique_lecture_id_train_path]:
if p is not None:
args.append(load_data(p))
if type(train_dt_path_or_obj)== str:
train_dt_path_or_obj = load_data(train_dt_path_or_obj)
 args = args[:1] + [train_dt_path_or_obj] + args[1:]
return cls(*args)
 def get_split_index(self, user_id):
if user_id not in self._split_indices:
return -1
if DEBUG:
assert 0 <= self._split_indices[user_id] < self._valid_info[user_id]['orig_seq_len']
return self._split_indices[user_id]
def iter_dataset(self, max_steps=None):
print('entering iter_dataset() ')
 expected_mean_blocks_per_batch = 30.0
expected_n_batches = int(self._n_blocks / expected_mean_blocks_per_batch)+ 1
sampled_block_size = [30] * 100000
sample_mean_blocks_per_batch = 30.0
if DEBUG:
assert len(sampled_block_size)>= expected_n_batches
print(f'expected_mean_blocks_per_batch_in_pred: {expected_mean_blocks_per_batch}')
 print(f'sample_mean_blocks_per_batch_in_pred: {sample_mean_blocks_per_batch}')
 print(f'expected_n_batches_in_pred: {expected_n_batches}')
 total_blocks_yield = 0
print('start looping...')
 while True:
if max_steps is not None and self.current_batch_no >= max_steps:
break
try:
n_blocks_to_select = 30.0
if self.current_batch_no < len(sampled_block_size):
n_blocks_to_select = sampled_block_size[self.current_batch_no]
valid_df, valid_dt, prior_group_answers_correct, prior_group_responses, n_blocks_in_batch = self._step(n_blocks_to_select)
 if DEBUG:
assert n_blocks_in_batch > 0
total_blocks_yield += n_blocks_in_batch
if total_blocks_yield > self._n_blocks:
raise ValueError('total_blocks_yield > self._n_blocks! The code has some bugs to fix!')
 if DEBUG:
assert total_blocks_yield == self._n_blocks - self._n_remaining_blocks
self.current_batch_no += 1
if self.current_batch_no % PRINTING_STEPS == 0:
print(f'current_batch_no: {self.current_batch_no}')
 print(f'n_blocks processed in current batch: {n_blocks_in_batch}')
 print(f'n_blocks processed: {total_blocks_yield}')
 print(f'averaged n_blocks processed per batch: {total_blocks_yield / self.current_batch_no}')
 print(f'n_users_completed: {self._n_users - len(self._remaining_users)}')
 yield valid_df, valid_dt, prior_group_answers_correct, prior_group_responses
except StopIteration:
break
if DEBUG:
assert len(self._user_manager_dict)== 0
assert len(self._remaining_users)== 0
assert self._n_remaining_blocks == 0
print('finished looping.')
 def _step(self, n_users):
return self._blocks_from_random_users(n_users)
 def _blocks_from_random_users(self, n_users):
if DEBUG:
assert len(self._remaining_users)>= 0
if len(self._remaining_users)== 0:
raise StopIteration
n_users_to_select = min(n_users, len(self._remaining_users))
selected_users = random.sample(self._remaining_users, k=n_users_to_select)
 if DEBUG:
assert len(selected_users)== n_users_to_select
user_dts = []
for user_id in selected_users:
if self._user_manager_dict[user_id].has_next() :
user_dts.append(next(self._user_manager_dict[user_id]))
if not self._user_manager_dict[user_id].has_next() :
del self._user_manager_dict[user_id]
self._remaining_users.remove(user_id)
 valid_dt = dt.rbind(user_dts)
 n_blocks_selected = n_users_to_select
self._n_remaining_blocks -= n_blocks_selected
prior_group_answers_correct = self._current_answered_correctly
prior_group_responses = self._current_user_answer
if prior_group_answers_correct is None:
prior_group_answers_correct = []
if prior_group_responses is None:
prior_group_responses = []
if DEBUG:
assert type(prior_group_answers_correct)== list
assert type(prior_group_responses)== list
self._current_answered_correctly = valid_dt[:, 'answered_correctly'].to_list() [0]
self._current_user_answer = valid_dt[:, 'user_answer'].to_list() [0]
convert_dt(valid_dt, pred=True, unique_question_id_train=self._unique_question_id_train, unique_lecture_id_train=self._unique_lecture_id_train)
 valid_df = valid_dt.to_pandas()
return valid_df, valid_dt, prior_group_answers_correct, prior_group_responses, n_blocks_selected<categorify>
print('grouping by ip-app-os combination...')
 gp = train_df[['ip','app', 'os', 'channel']].groupby(by=['ip', 'app', 'os'])[['channel']].count().reset_index().rename(index=str, columns={'channel': 'ip_app_os_count'})
 train_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')
 del gp
gc.collect()
class Test_Iter_Manager(Pred_Iter_Manager):
def __init__(self, unique_question_id_train=None, unique_lecture_id_train=None):
super(Test_Iter_Manager, self ).__init__(unique_question_id_train, unique_lecture_id_train)
 self.env = riiideducation.make_env()
self._iter_test = iter(self.env.iter_test())
 @classmethod
def create(cls, unique_question_id_train_path=None, unique_lecture_id_train_path=None):
args = []
for p in [unique_question_id_train_path, unique_lecture_id_train_path]:
if p is not None:
args.append(load_data(p))
return cls(*args)
 def _has_next(self):
return self._has_next
def iter_dataset(self, max_steps=None):
print('entering iter_dataset() ')
 total_blocks_yield = 0
print('start looping...')
 while True:
try:
test_df, test_dt, prior_group_answers_correct, prior_group_responses, n_blocks_in_batch = self._step(n_users=None)
 assert n_blocks_in_batch > 0
total_blocks_yield += n_blocks_in_batch
self.current_batch_no += 1
if self.current_batch_no % PRINTING_STEPS == 0:
print(f'current_batch_no: {self.current_batch_no}')
 print(f'n_blocks processed in current batch: {n_blocks_in_batch}')
 print(f'n_blocks processed: {total_blocks_yield}')
 print(f'averaged n_blocks processed per batch: {total_blocks_yield / self.current_batch_no}')
 yield test_df, test_dt, prior_group_answers_correct, prior_group_responses
except StopIteration:
break
print('finished looping.')
 def _step(self, n_users=None):
test_df, _ = next(self._iter_test)
 test_dt, prior_group_answers_correct, prior_group_responses = convert_df_to_dt(
 test_df, test=True,
unique_question_id_train=self._unique_question_id_train,
unique_lecture_id_train=self._unique_lecture_id_train 
)
 n_blocks = len(set(test_dt[:, 'user_id'].to_list() [0]))
return test_df, test_dt, prior_group_answers_correct, prior_group_responses, n_blocks<prepare_x_and_y>
print('grouping by : ip_day_chl_var_hour')
 gp = train_df[['ip','day','hour','channel']].groupby(by=['ip','day','channel'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_tchan_count'})
 train_df = train_df.merge(gp, on=['ip','day','channel'], how='left')
 del gp
gc.collect()
def get_user_dt(user_id, _dt, user_id_to_row_id, split_index=-1):

(row_id_start, row_id_end)= user_id_to_row_id[user_id]
if split_index > -1:
row_id_end = row_id_start + split_index - 1
user_dt = _dt[row_id_start:row_id_end + 1, :]
return user_dt
attrs = [
'user_id',
'row_id',
'timestamp',
'content_id',
'content_type_id',
'task_container_id',
'user_answer',
'answered_correctly',
'prior_question_elapsed_time',
'prior_question_had_explanation'
]
extended_attrs = ['user_id'] + ['seq_len', 'prev_seq_len', 'start', 'end'] + \
[k for k in attrs if k not in ['user_id']] + \
['pred_time_mask', 'abs_pos', 'shifted_abs_pos', 'shifted_answered_correctly', 'shifted_user_answer'] + \
['lag_time']
aggregated_attrs_less = [
'n_questions_answered',
'n_questions_answered_correctly',
'n_lectures_watched'
]
for part_idx in range(2, 9):
aggregated_attrs_less.append(f'part_{part_idx}_count')
 aggregated_attrs_less.append(f'part_{part_idx}_correctness_count')
 for correct_answer_idx in range(ANSWER_0_ID, ANSWER_3_ID + 1):
	aggregated_attrs_less.append(f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_count')
 	aggregated_attrs_less.append(f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_answered_correctly_count')
 aggregated_attrs = aggregated_attrs_less + ['n_prev_seen', 'n_prev_correctness']
attr_dtypes = {}
for k in extended_attrs:
if k in ['row_id', 'user_id', 'timestamp', 'lag_time']:
attr_dtypes[k] = tf.int64
elif k in ['prior_question_elapsed_time']:
attr_dtypes[k] = DTYPE
else:
attr_dtypes[k] = tf.int32
for k in aggregated_attrs:
attr_dtypes[k] = tf.int32
class User_Record:

 def __init__(self, user_dt=None, user_id=None):

 user_ids = user_dt[:, 'user_id'].to_list() [0] if user_dt is not None else []
self.user_id = user_ids[0] if user_dt is not None else user_id
self.row_id = user_dt[:, 'row_id'].to_list() [0] if user_dt is not None else []
self.timestamp = user_dt[:, 'timestamp'].to_list() [0] if user_dt is not None else []
self.content_id = user_dt[:, 'content_id'].to_list() [0] if user_dt is not None else []
self.content_type_id = user_dt[:, 'content_type_id'].to_list() [0] if user_dt is not None else []
self.task_container_id = user_dt[:, 'task_container_id'].to_list() [0] if user_dt is not None else []
self.user_answer = user_dt[:, 'user_answer'].to_list() [0] if user_dt is not None else []
self.answered_correctly = user_dt[:, 'answered_correctly'].to_list() [0] if user_dt is not None else []
self.prior_question_elapsed_time = user_dt[:, 'prior_question_elapsed_time'].to_list() [0] if user_dt is not None else []
self.prior_question_had_explanation = user_dt[:, 'prior_question_had_explanation'].to_list() [0] if user_dt is not None else []
for aggregated_feature_idx, aggregated_feature_name in enumerate(aggregated_attrs):
setattr(self, aggregated_feature_name, [])
 def extend(self, other):

 for k in attrs + aggregated_attrs:
if k != 'user_id':
getattr(self, k ).extend(getattr(other, k))
def update_answer_results(self, prior_correctnesses, prior_answers, prior_user_info_for_post_update, question_history_at_training_end):


n_pred_time_steps = prior_user_info_for_post_update['n_pred_time_steps']
aggregated_info_to_update = prior_user_info_for_post_update['aggregated_info_to_update']
self.answered_correctly = self.answered_correctly[:-len(prior_correctnesses)] + prior_correctnesses
self.user_answer = self.user_answer[:-len(prior_answers)] + prior_answers
for idx in range(n_pred_time_steps):
index_in_pred_record = -(n_pred_time_steps - idx)
 if prior_correctnesses[idx] == 1:
current_question_id_str = str(self.content_id[index_in_pred_record])
 question_history_at_training_end[str(self.user_id)][current_question_id_str][1] += 1
if idx > 0:
for k in aggregated_attrs_less:
aggregated_info_to_update[k][idx] = aggregated_info_to_update[k][idx-1]
prev_correctness = prior_correctnesses[idx - 1]
if prev_correctness == 1:
aggregated_info_to_update['n_questions_answered_correctly'][idx] = aggregated_info_to_update['n_questions_answered_correctly'][idx-1] + 1
prev_part_key = aggregated_info_to_update['prev_part_key'][idx]
prev_correct_answer_key = aggregated_info_to_update['prev_correct_answer_key'][idx]
aggregated_info_to_update[prev_part_key][idx] = aggregated_info_to_update[prev_part_key][idx-1] + 1
aggregated_info_to_update[prev_correct_answer_key][idx] = aggregated_info_to_update[prev_correct_answer_key][idx-1] + 1
for k in aggregated_attrs_less:
if k == 'n_questions_answered_correctly' or 'correctness_count' in k or 'answered_correctly_count' in k:
setattr(self, k, getattr(self, k)[:-n_pred_time_steps] + aggregated_info_to_update[k])
 def toJSON(self):
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=False, indent=4)
 def __str__(self):
return self.toJSON()
def __len__(self):
return len(self.row_id)
 class Record_Buffer:

 def __init__(self, record_dict=None):

 if record_dict is None:
self.buffer = {}
else:
self.buffer = record_dict
def __contains__(self, x):
return x in self.buffer
def __getitem__(self, x):
if x not in self.buffer:
raise KeyError(str(x))
return self.buffer[x]
def __len__(self):
return len(self.buffer)
 def __del__(self):
del self.buffer
def update(self, record):

 if record.user_id not in self.buffer:
self.buffer[record.user_id] = User_Record(user_id=record.user_id)
 self.buffer[record.user_id].extend(record)
 def update_answer_results(self, user_id, prior_correctnesses, prior_answers, prior_user_info_for_post_update, question_history_at_training_end):

 record = self.buffer[user_id]
record.update_answer_results(prior_correctnesses, prior_answers, prior_user_info_for_post_update, question_history_at_training_end)
 def toJSON(self):
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=False, indent=4)
 def __str__(self):
return self.toJSON()
def combine_user_record(record_1, record_2, check=False):

 record = User_Record(user_id=record_1.user_id)
 record.extend(record_1)
 record.extend(record_2)
 return record
class Pred_Manager:

 def __init__(
 self, config, train_config, pred_iter_manager, train_dt, user_id_to_row_id_train,
question_history_at_training_end, single_question_history_at_training_end_optimized,
max_train_buffer_size=30000, probe=False 
):
self.config = config
self.train_config = train_config
self.pred_iter_manager = pred_iter_manager
self.train_dt = train_dt
self.user_id_to_row_id_train = user_id_to_row_id_train
self.question_history_at_training_end = question_history_at_training_end
self.single_question_history_at_training_end_optimized = single_question_history_at_training_end_optimized
self.train_record_buffer = Record_Buffer()
self.pred_record_buffer = Record_Buffer()
self.current_batch_row_ids = None
self.current_batch_users = None
self.max_train_buffer_size = max_train_buffer_size
self.users_in_pred = set()
self.common_users_in_train_and_pred = set()
self.new_users_in_pred = set()
self.n_blocks_in_pred = 0
self.n_blocks_from_common_users_in_train_and_pred = 0
self.n_blocks_from_new_users_in_pred = 0
self.avg_n_blocks_in_pred = 0
self.avg_n_blocks_from_common_users_in_train_and_pred = 0
self.avg_n_blocks_from_new_users_in_pred = 0
self.users_in_train = set(user_id_to_row_id_train.keys())
 self.users_trained = set()
for user_id in self.users_in_train:
split_index = self.pred_iter_manager.get_split_index(user_id)
 if split_index == -1 or split_index > 0:
self.users_trained.add(user_id)
 self.probe = probe
self.user_performance_hdf5_fp = h5py.File(user_performance_hdf5_path, "r")
 self.user_performance_hdf5 = self.user_performance_hdf5_fp['user_performance']
@classmethod
def create(cls, config, train_config, pred_iter_manager_class,
train_dt_path_or_obj,
user_id_to_row_id_train_path,
unique_question_id_train_path,
unique_lecture_id_train_path,
question_history_at_training_end_path,
single_question_history_at_training_end_optimized_path,
valid_info_path=None,
max_train_buffer_size=30000,
probe=False,
debug=False 
):
if pred_iter_manager_class == Valid_Iter_Manager:
data_paths = [valid_info_path, train_dt_path_or_obj]
elif pred_iter_manager_class == Test_Iter_Manager:
data_paths = []
assert type(unique_question_id_train_path)== str and type(unique_lecture_id_train_path)== str
data_paths.extend([unique_question_id_train_path, unique_lecture_id_train_path])
 pred_iter_manager = pred_iter_manager_class.create(*data_paths)
 user_id_to_row_id_train = load_data(user_id_to_row_id_train_path)
 start_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
print(f'start_memeory before loading question_history_at_training_end.json: {start_memeory}')
 question_history_at_training_end = load_data(question_history_at_training_end_path)
 end_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
print(f'end_memeory after loading question_history_at_training_end.json: {end_memeory}')
 used_memory = start_memeory - end_memeory
print(f'used_memory for loading question_history_at_training_end.json: {used_memory}')
 print('----------------------------------------')
 start_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
print(f'start_memeory before loading single_question_history_at_training_end_optimized.json: {start_memeory}')
 single_question_history_at_training_end_optimized = load_data(single_question_history_at_training_end_optimized_path)
 end_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
print(f'end_memeory after loading single_question_history_at_training_end_optimized.json: {end_memeory}')
 used_memory = start_memeory - end_memeory
print(f'used_memory for loading single_question_history_at_training_end_optimized.json: {used_memory}')
 print('----------------------------------------')
 if pred_iter_manager_class == Valid_Iter_Manager:
train_dt = pred_iter_manager.train_dt
elif type(train_dt_path_or_obj)!= str:
train_dt = train_dt_path_or_obj
else:
train_dt = load_data(train_dt_path_or_obj)
 args = [
config, train_config, pred_iter_manager, train_dt, user_id_to_row_id_train,
question_history_at_training_end, single_question_history_at_training_end_optimized,
max_train_buffer_size, probe
]
return cls(*args)
 def reset_train_record_buffer(self):

 user_ids = list(self.train_record_buffer.buffer.keys())
 for user_id in user_ids:
del self.train_record_buffer.buffer[user_id]
del self.train_record_buffer
self.train_record_buffer = Record_Buffer()
def update_batch_users(self, user_ids):

 self.current_batch_users = user_ids
def update_answer_results(self, prev_batch_users, prior_group_answers_correct, prior_group_responses, prior_info_for_post_update):

 d1 = defaultdict(list)
 d2 = defaultdict(list)
 for user_id, prior_ans_correct, prior_ans in zip(prev_batch_users, prior_group_answers_correct, prior_group_responses):
d1[user_id].append(prior_ans_correct)
 d2[user_id].append(prior_ans)
 for user_id in d1:
self.pred_record_buffer.update_answer_results(
 user_id,
d1[user_id], d2[user_id],
prior_info_for_post_update[user_id],
self.question_history_at_training_end 
)
 def update_aggregated_info_buffer(self, user_id, pred_record, record_to_watch, index_to_watch, n_pred_time_steps, pred_time_step_index, buffer):
user_id_str = str(user_id)
 current_index_in_pred_record = -(n_pred_time_steps - pred_time_step_index)
 n_prev_seen, n_prev_correctness = None, None
if pred_record.content_type_id[current_index_in_pred_record] == 0:
current_question_id = pred_record.content_id[current_index_in_pred_record]
current_question_id_str = str(current_question_id)
 if user_id_str not in self.question_history_at_training_end:
self.question_history_at_training_end[user_id_str] = dict()
if current_question_id_str not in self.question_history_at_training_end[user_id_str]:
self.question_history_at_training_end[user_id_str][current_question_id_str] = [None, None]
else:
n_prev_seen, n_prev_correctness = self.question_history_at_training_end[user_id_str][current_question_id_str]
if n_prev_seen is None:
if user_id_str in self.single_question_history_at_training_end_optimized:
if type(self.single_question_history_at_training_end_optimized[user_id_str]['correct'])== list:
self.single_question_history_at_training_end_optimized[user_id_str]['correct'] = set(self.single_question_history_at_training_end_optimized[user_id_str]['correct'])
 if type(self.single_question_history_at_training_end_optimized[user_id_str]['incorrect'])== list:
self.single_question_history_at_training_end_optimized[user_id_str]['incorrect'] = set(self.single_question_history_at_training_end_optimized[user_id_str]['incorrect'])
 if current_question_id_str in self.single_question_history_at_training_end_optimized[user_id_str]['correct']:
n_prev_seen, n_prev_correctness = [1, 1]
self.single_question_history_at_training_end_optimized[user_id_str]['correct'].remove(current_question_id_str)
 elif current_question_id_str in self.single_question_history_at_training_end_optimized[user_id_str]['incorrect']:
n_prev_seen, n_prev_correctness = [1, 0]
self.single_question_history_at_training_end_optimized[user_id_str]['incorrect'].remove(current_question_id_str)
 if n_prev_seen is None:
n_prev_seen, n_prev_correctness = [0, 0]
self.question_history_at_training_end[user_id_str][current_question_id_str][0] = n_prev_seen + 1
self.question_history_at_training_end[user_id_str][current_question_id_str][1] = n_prev_correctness
if n_prev_seen is None:
n_prev_seen, n_prev_correctness = [0, 0]
buffer['n_prev_seen'].append(n_prev_seen)
 buffer['n_prev_correctness'].append(n_prev_correctness)
 if record_to_watch is None:
for k in aggregated_attrs_less:
buffer[k].append(0)
 for k in ['prev_part_key', 'prev_correct_answer_key']:
buffer[k].append(None)
 return
if pred_time_step_index == 0:
for k in aggregated_attrs_less:
buffer[k].append(getattr(record_to_watch, k)[-1])
 else:
for k in aggregated_attrs_less:
buffer[k].append(buffer[k][-1])
 for k in ['prev_part_key', 'prev_correct_answer_key']:
buffer[k].append(None)
 content_id = record_to_watch.content_id[index_to_watch]
content_type_id = record_to_watch.content_type_id[index_to_watch]
if content_type_id == 0:
content_input_id = question_id_to_input_id_dict[content_id]
part_id = c_inputs_ids_to_part_dict[content_input_id] + 1
correct_answer = c_inputs_ids_to_correct_answer_id_dict[content_input_id] - ANSWER_0_ID
question_id = str(content_id)
 answered_correctly =(record_to_watch.answered_correctly[index_to_watch] == 1)
 buffer['prev_part_key'][-1] = f'part_{part_id}_correctness_count'
buffer['prev_correct_answer_key'][-1] = f'correct_answer_{correct_answer}_answered_correctly_count'
is_lecture = int(content_type_id == 1)
 buffer['n_lectures_watched'][-1] += is_lecture
if is_lecture == 0:
buffer['n_questions_answered'][-1] += 1
k = f'part_{part_id}_count'
buffer[k][-1] += 1
k = f'correct_answer_{correct_answer}_count'
buffer[k][-1] += 1
if answered_correctly is True:
for k in [
f'n_questions_answered_correctly',
f'part_{part_id}_correctness_count',
f'correct_answer_{correct_answer}_answered_correctly_count'
]:
buffer[k][-1] += 1
def get_aggregated_info_to_update(self, user_id, train_record, pred_record, n_pred_time_steps):
if len(pred_record)> n_pred_time_steps:
record_to_watch = pred_record
index_to_watch = -(n_pred_time_steps + 1)
 elif len(train_record)> 0:
record_to_watch = train_record
index_to_watch = -1
else:
record_to_watch = None
index_to_watch = None
aggregated_info_buffer = {
k: [] for k in aggregated_attrs + ['prev_part_key', 'prev_correct_answer_key']
}
for pred_time_step_index in range(n_pred_time_steps):
if pred_time_step_index >= 1:
record_to_watch = pred_record
index_to_watch = -(n_pred_time_steps - pred_time_step_index)- 1
self.update_aggregated_info_buffer(user_id, pred_record, record_to_watch, index_to_watch, n_pred_time_steps, pred_time_step_index, aggregated_info_buffer)
 return aggregated_info_buffer
def update(self, pred_dt, prior_group_answers_correct, prior_group_responses, prior_info_for_post_update):

 user_ids = pred_dt['user_id'].to_list() [0]
prev_batch_users = self.current_batch_users
if prev_batch_users is not None:
self.update_answer_results(prev_batch_users, prior_group_answers_correct, prior_group_responses, prior_info_for_post_update)
 self.update_batch_users(user_ids)
 if DEBUG:
row_ids = pred_dt['row_id'].to_list() [0]
prev_batch_row_ids = self.current_batch_row_ids
if isinstance(self.pred_iter_manager, Test_Iter_Manager):
debug_pred_batch_row_ids(row_ids, prev_batch_row_ids)
 self.current_batch_row_ids = row_ids
record_batch = [User_Record(user_dt=pred_dt[idx, :])for idx in range(pred_dt.nrows)]
if DEBUG:
debug_record_batch(record_batch)
 for record in record_batch:
self.pred_record_buffer.update(record)
 self.users_in_pred.update(set(user_ids))
n_new_users_in_batch = 0
for x in set(user_ids):
if x in self.users_trained:
self.common_users_in_train_and_pred.add(x)
 else:
self.new_users_in_pred.add(x)
 n_new_users_in_batch += 1
nb_common_users = len(self.common_users_in_train_and_pred)
 nb_new_users = len(self.new_users_in_pred)
 common_users_in_batch = set(user_ids ).intersection(self.users_trained)
 new_users_in_batch = set(user_ids ).difference(self.users_trained)
 self.n_blocks_in_pred += len(set(user_ids))
self.n_blocks_from_common_users_in_train_and_pred += len(common_users_in_batch)
 self.n_blocks_from_new_users_in_pred += len(new_users_in_batch)
 self.avg_n_blocks_in_pred = self.n_blocks_in_pred / len(self.users_in_pred)
 if nb_common_users > 0:
self.avg_n_blocks_from_common_users_in_train_and_pred = self.n_blocks_from_common_users_in_train_and_pred / nb_common_users
if nb_new_users > 0:
self.avg_n_blocks_from_new_users_in_pred = self.n_blocks_from_new_users_in_pred / nb_new_users
_pred_record_batch = []
n_pred_time_steps_dict = dict()
for x in user_ids:
if x not in n_pred_time_steps_dict:
_pred_record_batch.append(self.pred_record_buffer[x])
 n_pred_time_steps_dict[x] = 1
else:
n_pred_time_steps_dict[x] += 1
continue
user_ids_without_duplication = [x.user_id for x in _pred_record_batch]
n_pred_time_steps_batch = [n_pred_time_steps_dict[x.user_id] for x in _pred_record_batch]
train_record_batch = [self.get_training_record(x)for x in user_ids_without_duplication]
check = isinstance(self.pred_iter_manager, Valid_Iter_Manager)
 info_for_post_update = {}
pred_record_batch = []
for user_id, train_record, _pred_record, n_pred_time_steps in zip(
 user_ids_without_duplication,
train_record_batch,
_pred_record_batch,
n_pred_time_steps_batch 
):
aggregated_info_to_update = self.get_aggregated_info_to_update(user_id, train_record, _pred_record, n_pred_time_steps)
 pred_record = combine_user_record(train_record, _pred_record, check=check)
 pred_record_batch.append(pred_record)
 info_for_post_update[user_id] = {
'user_id': user_id,
'pred_record': _pred_record,
'n_pred_time_steps': n_pred_time_steps,
'aggregated_info_to_update': aggregated_info_to_update
}
for k in aggregated_attrs:
if k in ['n_lectures_watched', 'n_prev_seen', 'n_prev_correctness']:
getattr(pred_record, k ).extend(aggregated_info_to_update[k])
 else:
getattr(pred_record, k ).extend([aggregated_info_to_update[k][0]] * n_pred_time_steps)
 getattr(_pred_record, k ).extend(aggregated_info_to_update[k])
 if len(self.train_record_buffer)>= 3000:
self.reset_train_record_buffer()
gc.collect()
return pred_record_batch, n_pred_time_steps_batch, info_for_post_update
def get_training_record(self, user_id):

 if user_id in self.train_record_buffer:
return self.train_record_buffer[user_id]
if user_id not in self.user_id_to_row_id_train:
assert isinstance(self.pred_iter_manager, Test_Iter_Manager)
 else:
split_index = self.pred_iter_manager.get_split_index(user_id)
 user_dt = get_user_dt(user_id, self.train_dt, self.user_id_to_row_id_train, split_index)
 if isinstance(self.pred_iter_manager, Test_Iter_Manager):
assert user_dt.nrows > 0
if user_dt.nrows > 0:
convert_dt(user_dt, pred=False)
 user_record = User_Record(user_dt=user_dt)
 self.train_record_buffer.update(user_record)
 row_ids = self.train_record_buffer[user_id].row_id
row_start, row_end = row_ids[0], row_ids[-1]
seq_len = row_end - row_start + 1
user_performance = self.user_performance_hdf5[row_start:row_end + 1, :]
for feature_idx, k in enumerate(aggregated_attrs):
feature_performance = user_performance[:, feature_idx].tolist()
getattr(self.train_record_buffer[user_id], k ).extend(feature_performance)
 return self.train_record_buffer[user_id]
user_record = User_Record(user_id=user_id)
 self.train_record_buffer.update(user_record)
 return self.train_record_buffer[user_id]
def convert_to_pred_batch(self, pred_record_batch, n_pred_time_steps_batch):
batch_record = self.convert_record_batch(pred_record_batch, n_pred_time_steps_batch)
 batch = Pred_Manager.batch_record_to_tensor(batch_record)
 training = tf.constant(0, dtype=tf.int32)
 return batch
def record_to_dict(self, record, n_pred_time_steps):
record_as_dict = {}
orig_seq_len = len(record.row_id)
 used_seq_len = orig_seq_len
pad_seq = []
if orig_seq_len < self.train_config.window_size:
pad_seq = [PAD_TOKEN] *(self.train_config.window_size - orig_seq_len)
 if orig_seq_len > self.train_config.window_size:
used_seq_len = self.train_config.window_size
record_as_dict['user_id'] = record.user_id
record_as_dict['seq_len'] = used_seq_len
record_as_dict['prev_seq_len'] = orig_seq_len
record_as_dict['start'] = max(orig_seq_len - self.train_config.window_size, 0)
 record_as_dict['end'] = orig_seq_len - 1
answered_correctly = getattr(record, 'answered_correctly')
 user_answer = getattr(record, 'user_answer')
 for k in attrs + aggregated_attrs:
if k not in ['user_id']:
record_as_dict[k] =(pad_seq + getattr(record, k)) [- self.train_config.window_size:]
shifted_answered_correctly = [START_TOKEN] + answered_correctly[:-1]
record_as_dict['shifted_answered_correctly'] =(pad_seq + shifted_answered_correctly)[- self.train_config.window_size:]
shifted_user_answer = [START_TOKEN] + user_answer[:-1]
record_as_dict['shifted_user_answer'] =(pad_seq + shifted_user_answer)[- self.train_config.window_size:]
pred_time_mask = [0] * orig_seq_len
pred_time_mask =(pad_seq + pred_time_mask)[- self.train_config.window_size:]
pred_time_mask = pred_time_mask[:self.train_config.window_size - n_pred_time_steps] + [1] * n_pred_time_steps
record_as_dict['pred_time_mask'] = pred_time_mask
abs_pos = list(range(orig_seq_len))
shifted_abs_pos = [START_TOKEN] + abs_pos[:-1]
record_as_dict['abs_pos'] =(pad_seq + abs_pos)[- self.train_config.window_size:]
record_as_dict['shifted_abs_pos'] =(pad_seq + shifted_abs_pos)[- self.train_config.window_size:]
timestamps = getattr(record, 'timestamp')
 task_container_ids = getattr(record, 'task_container_id')
 nb_actual_elts = min(self.train_config.window_size, len(timestamps))
_start_index = max(0, len(timestamps)-(self.train_config.window_size + MAX_PRED_TIME_QUESTION_BUNDLE_LEN))
timestamps = timestamps[_start_index:]
task_container_ids = task_container_ids[_start_index:]
nb_elt_to_add =(self.train_config.window_size + MAX_PRED_TIME_QUESTION_BUNDLE_LEN)- len(timestamps)
 if nb_elt_to_add > 0:
timestamps = [0] * nb_elt_to_add + timestamps
task_container_ids = [PAD_TOKEN] * nb_elt_to_add + task_container_ids
lag_time = []
current_lag_time = 0
for _t, _prev_t, _bundle, _prev_bundle in zip(timestamps[1:], timestamps[:-1], task_container_ids[1:], task_container_ids[:-1]):
_diff = _t - _prev_t
if _diff == 0 and _bundle == _prev_bundle:
_diff = current_lag_time
elif _diff == 0 and _bundle != _prev_bundle:
pass
else:
current_lag_time = _diff
lag_time.append(_diff)
 lag_time = lag_time[-nb_actual_elts:]
timestamps = timestamps[1:][-nb_actual_elts:]
_mask = tf.cast(tf.math.logical_and(tf.constant(lag_time, dtype=tf.int64)== 0, tf.constant(timestamps, dtype=tf.int64)> 0), dtype=tf.int64)
 lag_time = tf.constant(lag_time, dtype=tf.int64)*(1 - _mask)+ tf.constant(1000, dtype=tf.int64)* _mask
lag_time = lag_time.numpy().tolist()
lag_time =(pad_seq + lag_time)[- self.train_config.window_size:]
record_as_dict['lag_time'] = lag_time
return record_as_dict
def convert_record_batch(self, record_batch, n_pred_time_steps_batch):

 records_as_dicts = [Pred_Manager.record_to_dict(self, record, n_pred_time_steps)for record, n_pred_time_steps in zip(record_batch, n_pred_time_steps_batch)]
batch_record = {k: [x[k] for x in records_as_dicts] for k in extended_attrs + aggregated_attrs}
return batch_record
@staticmethod
def batch_record_to_tensor(batch_record):

 t = {
k: tf.constant(batch_record[k], dtype=attr_dtypes[k])for k in extended_attrs + aggregated_attrs
}
return t<define_variables>
print('grouping by : ip_app_os_var_hour')
 gp = train_df[['ip','app', 'os', 'hour']].groupby(by=['ip', 'app', 'os'])[['hour']].var().reset_index().rename(index=str, columns={'hour': 'ip_app_os_var'})
 train_df = train_df.merge(gp, on=['ip','app', 'os'], how='left')
 del gp
gc.collect()
def debug_pred_batch_row_ids(row_ids, prev_batch_row_ids):

 assert len(set(row_ids)) == len(row_ids)
 assert row_ids == sorted(row_ids)
 if prev_batch_row_ids is not None:
assert row_ids[0] > prev_batch_row_ids[-1]
def debug_record_batch(record_batch):

 _tmp = defaultdict(list)
 for record in record_batch:
assert len(record.row_id)== 1
_tmp[record.user_id].append(record)
 for user_id, records in _tmp.items() :
task_container_ids_for_questions = [x.task_container_id[0] for x in records if x.content_type_id[0] == 0]
if len(task_container_ids_for_questions)> 0:
assert len(set(task_container_ids_for_questions)) == 1
row_ids_for_questions = [x.row_id[0] for x in records if x.content_type_id[0] == 0]
records_in_between = [x for x in records if row_ids_for_questions[0] <= x.row_id[0] <= row_ids_for_questions[-1]]
row_ids_in_between = [x.row_id[0] for x in records_in_between]
assert row_ids_for_questions == row_ids_in_between
assert row_ids_for_questions == [x.row_id[0] for x in records][-len(row_ids_for_questions):]<categorify>
print('grouping by : ip_app_channel_var_day')
 gp = train_df[['ip','app', 'channel', 'day']].groupby(by=['ip', 'app', 'channel'])[['day']].var().reset_index().rename(index=str, columns={'day': 'ip_app_channel_var_day'})
 train_df = train_df.merge(gp, on=['ip','app', 'channel'], how='left')
 del gp
gc.collect()
max_buffer_size = 20
buffer = []
def run_pred(pred_manager, predictor, max_steps=None, ckpt_no=None):
batch_size = None
seq_len = pred_manager.train_config.window_size
encoder_decoder = tf.constant(pred_manager.config.model_type=='ed', dtype=tf.int32)
 training = tf.constant(0, dtype=tf.int32)
 generative = tf.constant(pred_manager.config.generative, dtype=tf.int32)
 allow_bundle_atten = tf.constant(pred_manager.config.allow_bundle_atten, dtype=tf.int32)
 valid_user_ids = []
valid_row_ids = []
valid_targets = []
valid_preds = []
if pred_manager.config.use_softmax:
loss_obj = tf.keras.losses.SparseCategoricalCrossentropy(
 from_logits=True, reduction=tf.keras.losses.Reduction.NONE,
name='sparse_categorical_crossentropy_for_correctness' 
)
 else:
loss_obj = tf.keras.losses.BinaryCrossentropy(
 from_logits=True, label_smoothing=0, reduction=tf.keras.losses.Reduction.NONE,
name='binary_crossentropy_for_correctness' 
)
 loss_metric = tf.keras.metrics.Mean()
acc_metric = tf.keras.metrics.BinaryAccuracy()
auc_metric = tf.keras.metrics.AUC(num_thresholds=2000)
 @tf.function(
 input_signature=get_input_signatures_2() 
)
 def predict(batch):
pred_batch = add_input_ids_and_targets_2(batch)
 c_mask, r_mask, r_c_mask, c_r_mask = get_attention_masks(pred_batch, training, encoder_decoder, generative, allow_bundle_atten)
 pred_time_mask = pred_batch['pred_time_mask'] * tf.cast(pred_batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 logits, answer_logits, c_outputs, r_outputs = predictor(
 pred_batch, c_mask, r_mask, r_c_mask, c_r_mask,
output_attentions=False, output_hidden_states=False, training=False 
)
 if pred_manager.config.use_softmax:
preds = tf.math.softmax(logits)
 else:
preds = tf.math.sigmoid(logits)
 pred_indices = tf.where(pred_time_mask == 1)
 selected_logits = tf.gather_nd(logits, pred_indices)
 selected_preds = tf.gather_nd(preds, pred_indices)
 selected_row_ids = tf.gather_nd(pred_batch['row_id'], pred_indices)
 return logits, preds, selected_logits, selected_preds, selected_row_ids
start_memeory_global = psutil.virtual_memory().available / 1024.0 / 1024.0
start = datetime.datetime.now()
start_global = start
pred_history = []
n_pred_batches = 0
n_interactions = 0
prior_info_for_post_update = None
if isinstance(pred_manager.pred_iter_manager, Test_Iter_Manager):
if max_steps is not None:
print(f'For `Test_Iter_Manager`, {max_steps} shoulde be `None`, and we set it for you here!')
 max_steps = None
total = 0.0
for pred_df, pred_dt, prior_group_answers_correct, prior_group_responses in pred_manager.pred_iter_manager.iter_dataset(max_steps=max_steps):
n_pred_batches += 1
try:
if DEBUG:
pass
pred_record_batch, n_pred_time_steps_batch, info_for_post_update = pred_manager.update(
 pred_dt, prior_group_answers_correct, prior_group_responses, prior_info_for_post_update 
)
 elapsed =(datetime.datetime.now() - start ).total_seconds()
prior_info_for_post_update = info_for_post_update
pred_batch = pred_manager.convert_to_pred_batch(pred_record_batch, n_pred_time_steps_batch)
 logits, preds, selected_logits, selected_preds, selected_row_ids = predict(pred_batch)
 predictions = selected_preds[:, -1].numpy().tolist()
assert len(predictions)== len(pred_df['content_type_id'])
 user_ids = pred_manager.current_batch_users
n_interactions += len(user_ids)
 if isinstance(pred_manager.pred_iter_manager, Valid_Iter_Manager):
row_ids = selected_row_ids.numpy().tolist()
targets = pred_manager.pred_iter_manager._current_answered_correctly
assert len(set([len(user_ids), len(row_ids), len(targets), len(predictions)])) == 1
targets_t = tf.constant(targets, dtype=tf.int32)
 pred_mask = targets_t != -1
pred_indices = tf.where(pred_mask)
 selected_targets_for_loss = tf.gather_nd(targets_t, pred_indices)
 selected_logits_for_loss = tf.gather_nd(selected_logits, pred_indices)
 selected_preds_for_loss = tf.gather_nd(selected_preds, pred_indices)
 losses = loss_obj(selected_targets_for_loss[:, tf.newaxis], selected_logits_for_loss)
 loss_metric.update_state(losses)
 acc_metric.update_state(selected_targets_for_loss[:, tf.newaxis], selected_preds_for_loss[:, -1:])
 auc_metric.update_state(selected_targets_for_loss, selected_preds_for_loss[:, -1])
 valid_user_ids.extend(user_ids)
 valid_row_ids.extend(row_ids)
 valid_targets.extend(targets)
 valid_preds.extend(predictions)
 elif isinstance(pred_manager.pred_iter_manager, Test_Iter_Manager):
pred_df['answered_correctly'] = predictions
pred_manager.pred_iter_manager.env.predict(pred_df.loc[pred_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
 else:
raise ValueError('`pred_manager.pred_iter_manager` must be an instance of either `Valid_Iter_Manager` or `Test_Iter_Manager`')
 if n_pred_batches % 100 == 0:
end = datetime.datetime.now()
elapsed =(end - start ).total_seconds()
elapsed_global =(end - start_global ).total_seconds()
avg_timing_per_batch = elapsed / PRINTING_STEPS
avg_timing_per_batch_global = elapsed_global / n_pred_batches
end_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
used_memory_global = start_memeory_global - end_memeory
print(f'used memory: {used_memory_global}')
 print(f'pred_manager.n_blocks_in_pred: {pred_manager.n_blocks_in_pred}')
 print(f'pred_manager.n_blocks_from_common_users_in_train_and_pred: {pred_manager.n_blocks_from_common_users_in_train_and_pred}')
 print(f'pred_manager.n_blocks_from_new_users_in_pred: {pred_manager.n_blocks_from_new_users_in_pred}')
 print(f'pred_manager.avg_n_blocks_in_pred: {pred_manager.avg_n_blocks_in_pred}')
 print(f'pred_manager.avg_n_blocks_from_common_users_in_train_and_pred: {pred_manager.avg_n_blocks_from_common_users_in_train_and_pred}')
 print(f'pred_manager.avg_n_blocks_from_new_users_in_pred: {pred_manager.avg_n_blocks_from_new_users_in_pred}')
 print(f'number of user interactions processed in current batch: {len(user_ids)}')
 print(f'total number of user interactions processed: {n_interactions}')
 print(f'average number of user interactions per batch: {n_interactions / n_pred_batches}')
 print(f'avg_timing_per_batch: {avg_timing_per_batch}')
 print(f'avg_timing_per_batch_global: {avg_timing_per_batch_global}')
 if isinstance(pred_manager.pred_iter_manager, Valid_Iter_Manager):
valid_loss = loss_metric.result().numpy()
valid_acc = acc_metric.result().numpy()
valid_auc = auc_metric.result().numpy()
print(f'valid_loss: {valid_loss}')
 print(f'valid_acc: {valid_acc}')
 print(f'valid_auc: {valid_auc}')
 start = datetime.datetime.now()
print('-' * 32)
 if n_pred_batches <= 4:
pred_history.append(str(pred_manager.pred_record_buffer))
except AssertionError as e:
print('some assertions are wrong, breaking the loop')
 raise e
if isinstance(pred_manager.pred_iter_manager, Valid_Iter_Manager):
if MAX_VALID_ITER_STEPS is not None:
assert not DEBUG
else:
print(f'pred_manager.pred_iter_manager: {pred_manager.pred_iter_manager._user_manager_dict}')
 end = datetime.datetime.now()
elapsed_global =(end - start_global ).total_seconds()
avg_timing_per_batch_global = elapsed_global / n_pred_batches
print(f'avg_timing_per_batch_global: {avg_timing_per_batch_global}')
 valid_loss = float(loss_metric.result().numpy())
 valid_acc = float(acc_metric.result().numpy())
 valid_auc = float(auc_metric.result().numpy())
 print(f'valid_loss: {valid_loss}')
 print(f'valid_acc: {valid_acc}')
 print(f'valid_auc: {valid_auc}')
 valid_results = {
'valid_loss': valid_loss,
'valid_acc': valid_acc,
'valid_auc': valid_auc
}
valid_submission = pd.DataFrame.from_dict(
 {
'row_ids': valid_row_ids,
'user_ids': valid_user_ids,
'targets': valid_targets,
'preds': valid_preds
} 
)
 if ckpt_no is None:
ckpt_no = 0
valid_submission.to_csv(f'valid_submission_ckpt_{ckpt_no}.csv', index=False)
 with open(f'valid_results_ckpt_{ckpt_no}.json', 'w', encoding='UTF-8')as fp:
json.dump(valid_results, fp, ensure_ascii=False, indent=4)
 if not IS_KAGGLE:
!gsutil cp -r './valid_results_ckpt_{ckpt_no}.json' '{CKPT_PRED_PATH}'
!gsutil cp -r './valid_submission_ckpt_{ckpt_no}.csv' '{CKPT_PRED_PATH}'
del pred_manager.pred_iter_manager
del pred_manager
gc.collect()
tf.keras.backend.clear_session()<categorify>
train_df['qty'] = train_df['qty'].astype('uint16')
 train_df['ip_app_count'] = train_df['ip_app_count'].astype('uint16')
 train_df['ip_app_os_count'] = train_df['ip_app_os_count'].astype('uint16' )
def run_dummy_inputs(predictor):
@tf.function
def foo(inputs):
r = predictor(inputs=inputs)
 return r
with strategy.scope() :
content_input_ids = tf.constant(1, shape=[3, 5])
 response_input_ids = tf.constant(1, shape=[3, 5])
 d_input_ids = tf.constant(1, shape=[3, 5])
 d_ans_input_ids = tf.constant(1, shape=[3, 5])
 pos_ids = tf.constant(1, shape=[3, 5])
 shifted_pos_ids = tf.constant(1, shape=[3, 5])
 tag_ids = tf.constant(0, shape=[3, 5, N_TAGS_PER_CONTENT])
 part_ids = tf.constant(1, shape=[3, 5])
 prior_explanation_ids = tf.constant(1, shape=[3, 5])
 prior_question_elapsed_time_input = tf.constant(1.0, shape=[3, 5])
 lag_time = tf.constant(1.0, shape=[3, 5])
 abs_pos_ids = tf.constant(1.0, shape=[3, 5])
 shifted_abs_pos_ids = tf.constant(1.0, shape=[3, 5])
 task_container_pos_ids = tf.constant(1.0, shape=[3, 5])
 correct_answer_id = tf.constant(1, shape=[3, 5])
 n_questions_answered_scaled = tf.constant(1.0, shape=[3, 5])
 n_lectures_watched_scaled = tf.constant(1.0, shape=[3, 5])
 answered_correctly_ratio = tf.constant(1.0, shape=[3, 5])
 part_correctness_ratio = tf.constant(1.0, shape=[3, 5, PART_VOCAB_SIZE - 2])
 part_count_scaled = tf.constant(1.0, shape=[3, 5, PART_VOCAB_SIZE - 2])
 correct_answer_count_scaled = tf.constant(1.0, shape=[3, 5, ANSWER_3_ID - ANSWER_0_ID + 1])
 correct_answer_correctness_ratio = tf.constant(1.0, shape=[3, 5, ANSWER_3_ID - ANSWER_0_ID + 1])
 current_part_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_part_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 current_correct_answer_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_correct_answer_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 current_question_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_question_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 inputs = {
'c_input_ids': content_input_ids,
'r_input_ids': response_input_ids,
'd_input_ids': d_input_ids,
'd_ans_input_ids': d_ans_input_ids,
'pos_ids': pos_ids,
'shifted_pos_ids': shifted_pos_ids,
'tag_ids': tag_ids,
'part_ids': part_ids,
'prior_explanation_ids': prior_explanation_ids,
'prior_question_elapsed_time_input': prior_question_elapsed_time_input,
'lag_time': lag_time,
'abs_pos_ids': abs_pos_ids,
'shifted_abs_pos_ids': shifted_abs_pos_ids,
'task_container_pos_ids': task_container_pos_ids,
'correct_answer_id': correct_answer_id,
'n_questions_answered_scaled': n_questions_answered_scaled,
'n_lectures_watched_scaled': n_lectures_watched_scaled,
'answered_correctly_ratio': answered_correctly_ratio,
'part_correctness_ratio': part_correctness_ratio,
'part_count_scaled': part_count_scaled,
'correct_answer_count_scaled': correct_answer_count_scaled,
'correct_answer_correctness_ratio': correct_answer_correctness_ratio,
'current_part_count_scaled': current_part_count_scaled,
'current_part_correctness_ratio': current_part_correctness_ratio,
'current_correct_answer_count_scaled': current_correct_answer_count_scaled,
'current_correct_answer_correctness_ratio': current_correct_answer_correctness_ratio,
'current_question_count_scaled': current_question_count_scaled,
'current_question_correctness_ratio': current_question_correctness_ratio,
}
r1 = foo(inputs)
 return r1<choose_model_class>
target = 'is_attributed'
predictors = ['app','device','os', 'channel', 'hour', 'qty',
'ip_tchan_count', 'ip_app_count',
'ip_app_os_count', 'ip_app_os_var',
'ip_app_channel_var_day']
categorical = ['app','device','os', 'channel', 'hour']
sub = pd.DataFrame()
sub['click_id'] = test_df['click_id'].astype('int')
 gc.collect()
def load_ckpt(predictor, optimizer, ckpt_path, ckpt_no=-1):
with strategy.scope() :
run_dummy_inputs(predictor)
 if optimizer is None:
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
 checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=predictor)
 ckpt_manager = tf.train.CheckpointManager(checkpoint, directory=ckpt_path, max_to_keep=None)
 last_ckpt_no = 0
if ckpt_manager.latest_checkpoint:
if len(ckpt_manager.latest_checkpoint.split('/ckpt-')) > 0:
last_ckpt_no = int(ckpt_manager.latest_checkpoint.split('/ckpt-')[-1])
 if last_ckpt_no > 0:
print(f'Latest checkpoint found.Model trained for {last_ckpt_no} epochs.')
 else:
print("No checkpoint found.")
 if ckpt_no == -1:
ckpt_no_to_load = last_ckpt_no
else:
ckpt_no_to_load = ckpt_no
assert ckpt_no_to_load <= last_ckpt_no
ckpt_file = os.path.join(ckpt_path, f'ckpt-{ckpt_no_to_load}')
 print(f'try to load {ckpt_file}')
 if ckpt_no_to_load > 0:
status = checkpoint.restore(ckpt_file)
 print(f'ckpt-{ckpt_no_to_load} is restored.')
 loaded_ckpt_no = ckpt_no_to_load
else:
print(f'no ckpt is found and restored.')
 loaded_ckpt_no = 0
return ckpt_manager, loaded_ckpt_no<define_variables>
print("Training...")
 params = {
'learning_rate': 0.1,
'num_leaves': 1400,
'max_depth': 3,
'min_child_samples': 200,
'max_bin': 100,
'subsample':.7,
'subsample_freq': 1,
'colsample_bytree': 0.7,
'min_child_weight': 0,
'scale_pos_weight':99
}
DUMMY = True
if SUBMISSION:
DUMMY = False<define_variables>
bst = lgb_modelfit_nocv(params,
train_df,
val_df,
predictors,
target,
objective='binary',
metrics='auc',
early_stopping_rounds=50,
verbose_eval=True,
num_boost_round=400,
categorical_features=categorical )
if DUMMY:
MODEL_TYPE = 'ed'
MODEL_SIZE = 'b-2'
MODEL_DESC = 'master-train-all'
ACTIVATION = 'gelu'
USE_PRE_CLASSIFIER = False
USE_SOFTMAX = False
USE_USER_ANSWER = True
USE_USER_ANSWER_LOSS = True
USE_CORRECT_ANSWER_FOR_ENCODER = True
USE_CORRECT_ANSWER_FOR_DECODER = False
USE_ABS_POS = False
USE_TASK_CONTAINER_POS = False
SHARE_POS_EMBEDDING = True
USE_TAGS = True
USE_PART = True
USE_PRIOR_EXPLANATION = True
USE_PRIOR_QUESTION_ELAPSED_TIME_INPUT = True
USE_LAG_TIME = True
USE_LAG_TIME_FOR_ENCODER=False
USE_USER_LEVEL_AGGREGATED_HISTORICAL_INFO = True
USE_PART_AGGREGATED_HISTORICAL_INFO = True
USE_CORRECT_ANSWER_AGGREGATED_HISTORICAL_INFO = True
USE_QUESTION_LEVEL_AGGREGATED_HISTORICAL_INFO = True
ALLOW_BUNDLE_ATTEN = True
GENERATIVE = False
VALID_FOLD = 1
WINDOW_SIZE = 128
LOSS_WEIGHT_WINDOW_SIZE = None
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
if not tpu:
BATCH_SIZE = 8 * BATCH_SIZE
PRED_BATCH_SIZE = 256 * strategy.num_replicas_in_sync
if tpu:
assert BATCH_SIZE // strategy.num_replicas_in_sync != WINDOW_SIZE
assert PRED_BATCH_SIZE // strategy.num_replicas_in_sync != WINDOW_SIZE
N_EPOCHS = 100
STEPS_PER_CALL = 1000
MAX_N_CONTENTS_PER_USER_FOR_SAMPLING_PROB = 512
LR = 2e-4
END_LR = 2e-5
WARMUP_STEPS = 80000
DETERMINISTIC = False
if DETERMINISTIC:
SEED = 2021
N_PARALLEL_READS = None
N_PARALLEL_CALLS = None
SHUFFLE_BUFFER_SIZE = 1
else:
SEED = None
N_PARALLEL_READS = 16
N_PARALLEL_CALLS = tf.data.experimental.AUTOTUNE
SHUFFLE_BUFFER_SIZE = 65536
MAX_TRAIN_ITER_STEPS = None
MAX_VALID_ITER_STEPS = None
PRINTING_STEPS = 1000
CKPT_DIR = f'{MODEL_TYPE}-{MODEL_SIZE}-{MODEL_DESC}/'
TRAIN = False
VALID = False
PRED = True
RESUME_TRAINING = True
N_FILES = 6
SUBMISSION = False
if IS_KAGGLE:
N_FILES = len(os.listdir('/kaggle/input/riiid-test-answer-prediction'))
SUBMISSION =(N_FILES != 6)
 else:
PRED = False
if SUBMISSION:
TRAIN = False
VALID = False
PRED = True
if not TRAIN:
RESUME_TRAINING = False
DEBUG = False
PROBE = False
CKPT_TRAIN_PATH = None
CKPT_PRED_PATH = None
if CKPT_TRAIN_PATH is None:
if IS_KAGGLE:
CKPT_TRAIN_PATH = './'
else:
CKPT_TRAIN_PATH = f'{BUCKET_DIR}/r3id-ckpts/{CKPT_DIR}'
if TRAIN and not RESUME_TRAINING:
_state = !gsutil -q stat {CKPT_TRAIN_PATH}*; echo $?
already_existed = 1 - int(_state[0])
 assert not already_existed
if CKPT_PRED_PATH is None:
if IS_KAGGLE:
CKPT_PRED_PATH = f'{BASE_DIR}/{CKPT_DIR}'
else:
CKPT_PRED_PATH = f'{BUCKET_DIR}/r3id-ckpts/{CKPT_DIR}'<define_variables>
del train_df
del val_df
gc.collect()
print("Predicting...")
 sub['is_attributed'] = bst.predict(test_df[predictors])
 print("writing...")
 sub.to_csv('sub_lgb_balanced99.csv',index=False)
 print("done...")
 print(sub.info() )
<define_variables><EOS>