Kaggle competition: Riiid Answer Correctness Prediction
print("Null count: ", housing_all_df['SaleType'].isnull().sum())
 print(housing_all_df['SaleType'].value_counts())
 housing_all_df['SaleType'].fillna(housing_all_df['SaleType'].mode().values[0], inplace=True )<save_to_csv>
max_buffer_size = 20
buffer = []
def run_pred(pred_manager, predictor, max_steps=None, ckpt_no=None):
batch_size = None
seq_len = pred_manager.train_config.window_size
encoder_decoder = tf.constant(pred_manager.config.model_type=='ed', dtype=tf.int32)
 training = tf.constant(0, dtype=tf.int32)
 generative = tf.constant(pred_manager.config.generative, dtype=tf.int32)
 allow_bundle_atten = tf.constant(pred_manager.config.allow_bundle_atten, dtype=tf.int32)
 valid_user_ids = []
valid_row_ids = []
valid_targets = []
valid_preds = []
if pred_manager.config.use_softmax:
loss_obj = tf.keras.losses.SparseCategoricalCrossentropy(
 from_logits=True, reduction=tf.keras.losses.Reduction.NONE,
name='sparse_categorical_crossentropy_for_correctness' 
)
 else:
loss_obj = tf.keras.losses.BinaryCrossentropy(
 from_logits=True, label_smoothing=0, reduction=tf.keras.losses.Reduction.NONE,
name='binary_crossentropy_for_correctness' 
)
 loss_metric = tf.keras.metrics.Mean()
acc_metric = tf.keras.metrics.BinaryAccuracy()
auc_metric = tf.keras.metrics.AUC(num_thresholds=2000)
 @tf.function(
 input_signature=get_input_signatures_2() 
)
 def predict(batch):
pred_batch = add_input_ids_and_targets_2(batch)
 c_mask, r_mask, r_c_mask, c_r_mask = get_attention_masks(pred_batch, training, encoder_decoder, generative, allow_bundle_atten)
 pred_time_mask = pred_batch['pred_time_mask'] * tf.cast(pred_batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 logits, answer_logits, c_outputs, r_outputs = predictor(
 pred_batch, c_mask, r_mask, r_c_mask, c_r_mask,
output_attentions=False, output_hidden_states=False, training=False 
)
 if pred_manager.config.use_softmax:
preds = tf.math.softmax(logits)
 else:
preds = tf.math.sigmoid(logits)
 pred_indices = tf.where(pred_time_mask == 1)
 selected_logits = tf.gather_nd(logits, pred_indices)
 selected_preds = tf.gather_nd(preds, pred_indices)
 selected_row_ids = tf.gather_nd(pred_batch['row_id'], pred_indices)
 return logits, preds, selected_logits, selected_preds, selected_row_ids
start_memeory_global = psutil.virtual_memory().available / 1024.0 / 1024.0
start = datetime.datetime.now()
start_global = start
pred_history = []
n_pred_batches = 0
n_interactions = 0
prior_info_for_post_update = None
if isinstance(pred_manager.pred_iter_manager, Test_Iter_Manager):
if max_steps is not None:
print(f'For `Test_Iter_Manager`, {max_steps} shoulde be `None`, and we set it for you here!')
 max_steps = None
total = 0.0
for pred_df, pred_dt, prior_group_answers_correct, prior_group_responses in pred_manager.pred_iter_manager.iter_dataset(max_steps=max_steps):
n_pred_batches += 1
try:
if DEBUG:
pass
pred_record_batch, n_pred_time_steps_batch, info_for_post_update = pred_manager.update(
 pred_dt, prior_group_answers_correct, prior_group_responses, prior_info_for_post_update 
)
 elapsed =(datetime.datetime.now() - start ).total_seconds()
prior_info_for_post_update = info_for_post_update
pred_batch = pred_manager.convert_to_pred_batch(pred_record_batch, n_pred_time_steps_batch)
 logits, preds, selected_logits, selected_preds, selected_row_ids = predict(pred_batch)
 predictions = selected_preds[:, -1].numpy().tolist()
assert len(predictions)== len(pred_df['content_type_id'])
 user_ids = pred_manager.current_batch_users
n_interactions += len(user_ids)
 if isinstance(pred_manager.pred_iter_manager, Valid_Iter_Manager):
row_ids = selected_row_ids.numpy().tolist()
targets = pred_manager.pred_iter_manager._current_answered_correctly
assert len(set([len(user_ids), len(row_ids), len(targets), len(predictions)])) == 1
targets_t = tf.constant(targets, dtype=tf.int32)
 pred_mask = targets_t != -1
pred_indices = tf.where(pred_mask)
 selected_targets_for_loss = tf.gather_nd(targets_t, pred_indices)
 selected_logits_for_loss = tf.gather_nd(selected_logits, pred_indices)
 selected_preds_for_loss = tf.gather_nd(selected_preds, pred_indices)
 losses = loss_obj(selected_targets_for_loss[:, tf.newaxis], selected_logits_for_loss)
 loss_metric.update_state(losses)
 acc_metric.update_state(selected_targets_for_loss[:, tf.newaxis], selected_preds_for_loss[:, -1:])
 auc_metric.update_state(selected_targets_for_loss, selected_preds_for_loss[:, -1])
 valid_user_ids.extend(user_ids)
 valid_row_ids.extend(row_ids)
 valid_targets.extend(targets)
 valid_preds.extend(predictions)
 elif isinstance(pred_manager.pred_iter_manager, Test_Iter_Manager):
pred_df['answered_correctly'] = predictions
pred_manager.pred_iter_manager.env.predict(pred_df.loc[pred_df['content_type_id'] == 0, ['row_id', 'answered_correctly']])
 else:
raise ValueError('`pred_manager.pred_iter_manager` must be an instance of either `Valid_Iter_Manager` or `Test_Iter_Manager`')
 if n_pred_batches % 100 == 0:
end = datetime.datetime.now()
elapsed =(end - start ).total_seconds()
elapsed_global =(end - start_global ).total_seconds()
avg_timing_per_batch = elapsed / PRINTING_STEPS
avg_timing_per_batch_global = elapsed_global / n_pred_batches
end_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
used_memory_global = start_memeory_global - end_memeory
print(f'used memory: {used_memory_global}')
 print(f'pred_manager.n_blocks_in_pred: {pred_manager.n_blocks_in_pred}')
 print(f'pred_manager.n_blocks_from_common_users_in_train_and_pred: {pred_manager.n_blocks_from_common_users_in_train_and_pred}')
 print(f'pred_manager.n_blocks_from_new_users_in_pred: {pred_manager.n_blocks_from_new_users_in_pred}')
 print(f'pred_manager.avg_n_blocks_in_pred: {pred_manager.avg_n_blocks_in_pred}')
 print(f'pred_manager.avg_n_blocks_from_common_users_in_train_and_pred: {pred_manager.avg_n_blocks_from_common_users_in_train_and_pred}')
 print(f'pred_manager.avg_n_blocks_from_new_users_in_pred: {pred_manager.avg_n_blocks_from_new_users_in_pred}')
 print(f'number of user interactions processed in current batch: {len(user_ids)}')
 print(f'total number of user interactions processed: {n_interactions}')
 print(f'average number of user interactions per batch: {n_interactions / n_pred_batches}')
 print(f'avg_timing_per_batch: {avg_timing_per_batch}')
 print(f'avg_timing_per_batch_global: {avg_timing_per_batch_global}')
 if isinstance(pred_manager.pred_iter_manager, Valid_Iter_Manager):
valid_loss = loss_metric.result().numpy()
valid_acc = acc_metric.result().numpy()
valid_auc = auc_metric.result().numpy()
print(f'valid_loss: {valid_loss}')
 print(f'valid_acc: {valid_acc}')
 print(f'valid_auc: {valid_auc}')
 start = datetime.datetime.now()
print('-' * 32)
 if n_pred_batches <= 4:
pred_history.append(str(pred_manager.pred_record_buffer))
except AssertionError as e:
print('some assertions are wrong, breaking the loop')
 raise e
if isinstance(pred_manager.pred_iter_manager, Valid_Iter_Manager):
if MAX_VALID_ITER_STEPS is not None:
assert not DEBUG
else:
print(f'pred_manager.pred_iter_manager: {pred_manager.pred_iter_manager._user_manager_dict}')
 end = datetime.datetime.now()
elapsed_global =(end - start_global ).total_seconds()
avg_timing_per_batch_global = elapsed_global / n_pred_batches
print(f'avg_timing_per_batch_global: {avg_timing_per_batch_global}')
 valid_loss = float(loss_metric.result().numpy())
 valid_acc = float(acc_metric.result().numpy())
 valid_auc = float(auc_metric.result().numpy())
 print(f'valid_loss: {valid_loss}')
 print(f'valid_acc: {valid_acc}')
 print(f'valid_auc: {valid_auc}')
 valid_results = {
'valid_loss': valid_loss,
'valid_acc': valid_acc,
'valid_auc': valid_auc
}
valid_submission = pd.DataFrame.from_dict(
 {
'row_ids': valid_row_ids,
'user_ids': valid_user_ids,
'targets': valid_targets,
'preds': valid_preds
} 
)
 if ckpt_no is None:
ckpt_no = 0
valid_submission.to_csv(f'valid_submission_ckpt_{ckpt_no}.csv', index=False)
 with open(f'valid_results_ckpt_{ckpt_no}.json', 'w', encoding='UTF-8')as fp:
json.dump(valid_results, fp, ensure_ascii=False, indent=4)
 if not IS_KAGGLE:
!gsutil cp -r './valid_results_ckpt_{ckpt_no}.json' '{CKPT_PRED_PATH}'
!gsutil cp -r './valid_submission_ckpt_{ckpt_no}.csv' '{CKPT_PRED_PATH}'
del pred_manager.pred_iter_manager
del pred_manager
gc.collect()
tf.keras.backend.clear_session()
high_corr_matrix = housing_corr[abs(housing_corr.loc[:,:])>= 0.5]
high_corr_matrix.to_csv('./correlations.csv' )<drop_column>
def run_dummy_inputs(predictor):
@tf.function
def foo(inputs):
r = predictor(inputs=inputs)
 return r
with strategy.scope() :
content_input_ids = tf.constant(1, shape=[3, 5])
 response_input_ids = tf.constant(1, shape=[3, 5])
 d_input_ids = tf.constant(1, shape=[3, 5])
 d_ans_input_ids = tf.constant(1, shape=[3, 5])
 pos_ids = tf.constant(1, shape=[3, 5])
 shifted_pos_ids = tf.constant(1, shape=[3, 5])
 tag_ids = tf.constant(0, shape=[3, 5, N_TAGS_PER_CONTENT])
 part_ids = tf.constant(1, shape=[3, 5])
 prior_explanation_ids = tf.constant(1, shape=[3, 5])
 prior_question_elapsed_time_input = tf.constant(1.0, shape=[3, 5])
 lag_time = tf.constant(1.0, shape=[3, 5])
 abs_pos_ids = tf.constant(1.0, shape=[3, 5])
 shifted_abs_pos_ids = tf.constant(1.0, shape=[3, 5])
 task_container_pos_ids = tf.constant(1.0, shape=[3, 5])
 correct_answer_id = tf.constant(1, shape=[3, 5])
 n_questions_answered_scaled = tf.constant(1.0, shape=[3, 5])
 n_lectures_watched_scaled = tf.constant(1.0, shape=[3, 5])
 answered_correctly_ratio = tf.constant(1.0, shape=[3, 5])
 part_correctness_ratio = tf.constant(1.0, shape=[3, 5, PART_VOCAB_SIZE - 2])
 part_count_scaled = tf.constant(1.0, shape=[3, 5, PART_VOCAB_SIZE - 2])
 correct_answer_count_scaled = tf.constant(1.0, shape=[3, 5, ANSWER_3_ID - ANSWER_0_ID + 1])
 correct_answer_correctness_ratio = tf.constant(1.0, shape=[3, 5, ANSWER_3_ID - ANSWER_0_ID + 1])
 current_part_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_part_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 current_correct_answer_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_correct_answer_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 current_question_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_question_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 inputs = {
'c_input_ids': content_input_ids,
'r_input_ids': response_input_ids,
'd_input_ids': d_input_ids,
'd_ans_input_ids': d_ans_input_ids,
'pos_ids': pos_ids,
'shifted_pos_ids': shifted_pos_ids,
'tag_ids': tag_ids,
'part_ids': part_ids,
'prior_explanation_ids': prior_explanation_ids,
'prior_question_elapsed_time_input': prior_question_elapsed_time_input,
'lag_time': lag_time,
'abs_pos_ids': abs_pos_ids,
'shifted_abs_pos_ids': shifted_abs_pos_ids,
'task_container_pos_ids': task_container_pos_ids,
'correct_answer_id': correct_answer_id,
'n_questions_answered_scaled': n_questions_answered_scaled,
'n_lectures_watched_scaled': n_lectures_watched_scaled,
'answered_correctly_ratio': answered_correctly_ratio,
'part_correctness_ratio': part_correctness_ratio,
'part_count_scaled': part_count_scaled,
'correct_answer_count_scaled': correct_answer_count_scaled,
'correct_answer_correctness_ratio': correct_answer_correctness_ratio,
'current_part_count_scaled': current_part_count_scaled,
'current_part_correctness_ratio': current_part_correctness_ratio,
'current_correct_answer_count_scaled': current_correct_answer_count_scaled,
'current_correct_answer_correctness_ratio': current_correct_answer_correctness_ratio,
'current_question_count_scaled': current_question_count_scaled,
'current_question_correctness_ratio': current_question_correctness_ratio,
}
r1 = foo(inputs)
 return r1
cols_to_drop = ['GarageYrBlt','YearRemodAdd','1stFlrSF','TotalBsmtSF','TotRmsAbvGrd','GrLivArea','GarageArea']
housing_all_df.drop(columns=cols_to_drop, inplace=True )<sort_values>
def load_ckpt(predictor, optimizer, ckpt_path, ckpt_no=-1):
with strategy.scope() :
run_dummy_inputs(predictor)
 if optimizer is None:
optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
 checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=predictor)
 ckpt_manager = tf.train.CheckpointManager(checkpoint, directory=ckpt_path, max_to_keep=None)
 last_ckpt_no = 0
if ckpt_manager.latest_checkpoint:
if len(ckpt_manager.latest_checkpoint.split('/ckpt-')) > 0:
last_ckpt_no = int(ckpt_manager.latest_checkpoint.split('/ckpt-')[-1])
 if last_ckpt_no > 0:
print(f'Latest checkpoint found.Model trained for {last_ckpt_no} epochs.')
 else:
print("No checkpoint found.")
 if ckpt_no == -1:
ckpt_no_to_load = last_ckpt_no
else:
ckpt_no_to_load = ckpt_no
assert ckpt_no_to_load <= last_ckpt_no
ckpt_file = os.path.join(ckpt_path, f'ckpt-{ckpt_no_to_load}')
 print(f'try to load {ckpt_file}')
 if ckpt_no_to_load > 0:
status = checkpoint.restore(ckpt_file)
 print(f'ckpt-{ckpt_no_to_load} is restored.')
 loaded_ckpt_no = ckpt_no_to_load
else:
print(f'no ckpt is found and restored.')
 loaded_ckpt_no = 0
return ckpt_manager, loaded_ckpt_no
train_df[train_df['OverallQual'] == 10][['GrLivArea', 'SalePrice']].sort_values('GrLivArea', ascending=False )<drop_column>
DUMMY = True
if SUBMISSION:
DUMMY = False
housing_all_df.drop(index=[523,1298], inplace=True )<feature_engineering>
if DUMMY:
MODEL_TYPE = 'ed'
MODEL_SIZE = 'b-2'
MODEL_DESC = 'master-train-all'
ACTIVATION = 'gelu'
USE_PRE_CLASSIFIER = False
USE_SOFTMAX = False
USE_USER_ANSWER = True
USE_USER_ANSWER_LOSS = True
USE_CORRECT_ANSWER_FOR_ENCODER = True
USE_CORRECT_ANSWER_FOR_DECODER = False
USE_ABS_POS = False
USE_TASK_CONTAINER_POS = False
SHARE_POS_EMBEDDING = True
USE_TAGS = True
USE_PART = True
USE_PRIOR_EXPLANATION = True
USE_PRIOR_QUESTION_ELAPSED_TIME_INPUT = True
USE_LAG_TIME = True
USE_LAG_TIME_FOR_ENCODER=False
USE_USER_LEVEL_AGGREGATED_HISTORICAL_INFO = True
USE_PART_AGGREGATED_HISTORICAL_INFO = True
USE_CORRECT_ANSWER_AGGREGATED_HISTORICAL_INFO = True
USE_QUESTION_LEVEL_AGGREGATED_HISTORICAL_INFO = True
ALLOW_BUNDLE_ATTEN = True
GENERATIVE = False
VALID_FOLD = 1
WINDOW_SIZE = 128
LOSS_WEIGHT_WINDOW_SIZE = None
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
if not tpu:
BATCH_SIZE = 8 * BATCH_SIZE
PRED_BATCH_SIZE = 256 * strategy.num_replicas_in_sync
if tpu:
assert BATCH_SIZE // strategy.num_replicas_in_sync != WINDOW_SIZE
assert PRED_BATCH_SIZE // strategy.num_replicas_in_sync != WINDOW_SIZE
N_EPOCHS = 100
STEPS_PER_CALL = 1000
MAX_N_CONTENTS_PER_USER_FOR_SAMPLING_PROB = 512
LR = 2e-4
END_LR = 2e-5
WARMUP_STEPS = 80000
DETERMINISTIC = False
if DETERMINISTIC:
SEED = 2021
N_PARALLEL_READS = None
N_PARALLEL_CALLS = None
SHUFFLE_BUFFER_SIZE = 1
else:
SEED = None
N_PARALLEL_READS = 16
N_PARALLEL_CALLS = tf.data.experimental.AUTOTUNE
SHUFFLE_BUFFER_SIZE = 65536
MAX_TRAIN_ITER_STEPS = None
MAX_VALID_ITER_STEPS = None
PRINTING_STEPS = 1000
CKPT_DIR = f'{MODEL_TYPE}-{MODEL_SIZE}-{MODEL_DESC}/'
TRAIN = False
VALID = False
PRED = True
RESUME_TRAINING = True
N_FILES = 6
SUBMISSION = False
if IS_KAGGLE:
N_FILES = len(os.listdir('/kaggle/input/riiid-test-answer-prediction'))
SUBMISSION =(N_FILES != 6)
 else:
PRED = False
if SUBMISSION:
TRAIN = False
VALID = False
PRED = True
if not TRAIN:
RESUME_TRAINING = False
DEBUG = False
PROBE = False
CKPT_TRAIN_PATH = None
CKPT_PRED_PATH = None
if CKPT_TRAIN_PATH is None:
if IS_KAGGLE:
CKPT_TRAIN_PATH = './'
else:
CKPT_TRAIN_PATH = f'{BUCKET_DIR}/r3id-ckpts/{CKPT_DIR}'
if TRAIN and not RESUME_TRAINING:
_state = !gsutil -q stat {CKPT_TRAIN_PATH}*; echo $?
already_existed = 1 - int(_state[0])
 assert not already_existed
if CKPT_PRED_PATH is None:
if IS_KAGGLE:
CKPT_PRED_PATH = f'{BASE_DIR}/{CKPT_DIR}'
else:
CKPT_PRED_PATH = f'{BUCKET_DIR}/r3id-ckpts/{CKPT_DIR}'
numerical_cols = housing_all_df.select_dtypes(include='number' ).columns.tolist()
numerical_cols = [col for col in numerical_cols if col not in ['MSSubClass','OverallQual','OverallCond','PoolQC','MoSold','SalePrice','IsRemod','IsNew','YearBuilt','YrSold']]
for col in numerical_cols:
skew_val = scipy.stats.skew(housing_all_df[col])
 if abs(skew_val)> 0.8:
housing_all_df[col] = np.log(housing_all_df[col] + 1 )<categorify>
FFN_FACTOR = 4
if MODEL_SIZE == 'baby':
n_layers = 1
n_heads = 2
dim = 4
hidden_dim = dim
elif MODEL_SIZE == 't-0':
n_layers = 2
n_heads = 4
dim = 32
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 't-1':
n_layers = 2
n_heads = 4
dim = 64
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 't-2':
n_layers = 2
n_heads = 4
dim = 128
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 't-3':
n_layers = 2
n_heads = 4
dim = 256
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 's-0':
n_layers = 4
n_heads = 4
dim = 32
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 's-1':
n_layers = 4
n_heads = 4
dim = 64
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 's-2':
n_layers = 4
n_heads = 4
dim = 128
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 's-3':
n_layers = 4
n_heads = 4
dim = 256
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 'b-0':
n_layers = 4
n_heads = 8
dim = 64
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 'b-1':
n_layers = 4
n_heads = 8
dim = 128
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 'b-2':
n_layers = 4
n_heads = 8
dim = 256
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 'b-3':
n_layers = 4
n_heads = 8
dim = 512
hidden_dim = FFN_FACTOR * dim // 2
elif MODEL_SIZE == 'b-4':
n_layers = 4
n_heads = 16
dim = 512
hidden_dim = FFN_FACTOR * dim // 2
elif MODEL_SIZE == 'm-0':
n_layers = 6
n_heads = 8
dim = 64
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 'm-1':
n_layers = 6
n_heads = 8
dim = 128
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 'm-2':
n_layers = 6
n_heads = 8
dim = 256
hidden_dim = FFN_FACTOR * dim
elif MODEL_SIZE == 'm-3':
n_layers = 6
n_heads = 8
dim = 512
hidden_dim = FFN_FACTOR * dim // 2
elif MODEL_SIZE == 'm-4':
n_layers = 6
n_heads = 16
dim = 512
hidden_dim = FFN_FACTOR * dim // 2
assert MODEL_TYPE is not None and MODEL_TYPE != ''
assert MODEL_DESC is not None and MODEL_DESC != ''
for col in ['FireplaceQu', 'ExterQual', 'ExterCond', 'BsmtQual','BsmtCond','KitchenQual','GarageQual','GarageCond','HeatingQC']:
housing_all_df[col] = housing_all_df[col].map({'None':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
 housing_all_df['BsmtExposure'] = housing_all_df['BsmtExposure'].map({'None':0,'No':1,'Mn':2,'Av':3,'Gd':4})
 for col in ['BsmtFinType1','BsmtFinType2']:
housing_all_df[col] = housing_all_df[col].map({'None':0,'Unf':1,'LwQ':2,'Rec':3,'BLQ':4,'ALQ':5,'GLQ':6})
 housing_all_df['GarageFinish'] = housing_all_df['GarageFinish'].map({'No Garage':0,'Unf':1,'RFn':2,'Fin':3} )<groupby>
max_position_embeddings = WINDOW_SIZE + 1
config = EdFormerConfig(
 model_type=MODEL_TYPE,
model_desc=MODEL_DESC,
model_size=MODEL_SIZE,
content_vocab_size=CONTENT_VOCAB_SIZE,
response_vocab_size=RESPONSE_VOCAB_SIZE,
tag_vocab_size=TAG_VOCAB_SIZE,
part_vocab_size=PART_VOCAB_SIZE,
prior_explanation_vocab_size=PRIOR_EXPLANATION_VOCAB_SIZE,
use_prior_question_elapsed_time_input=USE_PRIOR_QUESTION_ELAPSED_TIME_INPUT,
max_position_embeddings=max_position_embeddings,
sinusoidal_pos_embds=False,
n_layers=n_layers,
n_heads=n_heads,
dim=dim,
hidden_dim=hidden_dim,
activation=ACTIVATION,
dropout=0.1,
attention_dropout=0.1,
seq2seq_dropout=0.1,
initializer_range=0.02,
seed=SEED,
pad_token_id=PAD_ID,
use_user_answer=USE_USER_ANSWER,
use_user_answer_loss=USE_USER_ANSWER_LOSS,
use_correct_answer_for_encoder=USE_CORRECT_ANSWER_FOR_ENCODER,
use_correct_answer_for_decoder=USE_CORRECT_ANSWER_FOR_DECODER,
use_abs_pos=USE_ABS_POS,
use_task_container_pos=USE_TASK_CONTAINER_POS,
share_position_embeddings=SHARE_POS_EMBEDDING,
use_tags=USE_TAGS,
use_part=USE_PART,
use_prior_explanation=USE_PRIOR_EXPLANATION,
use_lag_time=USE_LAG_TIME,
use_lag_time_for_encoder=USE_LAG_TIME_FOR_ENCODER,
use_user_level_aggregated_historical_info=USE_USER_LEVEL_AGGREGATED_HISTORICAL_INFO,
use_part_aggregated_historical_info=USE_PART_AGGREGATED_HISTORICAL_INFO,
use_correct_answer_aggregated_historical_info=USE_CORRECT_ANSWER_AGGREGATED_HISTORICAL_INFO,
use_question_level_aggregated_historical_info=USE_QUESTION_LEVEL_AGGREGATED_HISTORICAL_INFO,
allow_bundle_atten=ALLOW_BUNDLE_ATTEN,
generative=GENERATIVE,
use_pre_classifier=USE_PRE_CLASSIFIER,
use_softmax=USE_SOFTMAX, 
)
 train_config = TrainConfig(
 ckpt_path=CKPT_TRAIN_PATH,
window_size=WINDOW_SIZE,
loss_weight_window_size=LOSS_WEIGHT_WINDOW_SIZE,
n_epochs=N_EPOCHS,
shuffle_buf_size=SHUFFLE_BUFFER_SIZE,
batch_size=BATCH_SIZE,
pred_batch_size=PRED_BATCH_SIZE,
seed=SEED,
deterministic=DETERMINISTIC,
num_parallel_reads=N_PARALLEL_READS,
num_parallel_calls=N_PARALLEL_CALLS,
steps_per_call=STEPS_PER_CALL,
max_n_contents_per_user_for_sampling_prob=MAX_N_CONTENTS_PER_USER_FOR_SAMPLING_PROB,
valid_fold=VALID_FOLD
)
groupd_by_neighborhood = housing_all_df[~housing_all_df['SalePrice'].isnull() ].groupby('Neighborhood')
 neighborhood_median_prices = groupd_by_neighborhood['SalePrice'].median()
neighborhood_median_prices = neighborhood_median_prices.sort_values()
neighborhood_mean_prices = groupd_by_neighborhood['SalePrice'].mean()
neighborhood_mean_prices = neighborhood_mean_prices.sort_values()<feature_engineering>
if TRAIN:
from_valid = False
only_valid = False
ckpts = [0]
valid_epochs = [
5, 10, 15, 20,
21, 22, 23, 24, 25,
26, 27, 28, 29, 30,
31, 32, 33, 34, 35,
36, 37, 38, 39, 40,
41, 42, 43, 44, 45,
46, 47, 48, 49, 50,
51, 52, 53, 54, 55,
56, 57, 58, 59, 60,
]
train_manager = Train_Manager(config, train_config)
 predictor, optimizer, loss_obj, loss_obj_answer, metrics = train_manager.get_train_objs(
 lr=LR, end_lr=END_LR, warmup_steps=WARMUP_STEPS 
)
 run_dummy_inputs(predictor)
 print('-' * 40)
 print('trainable variables:')
 for v in predictor.trainable_variables:
print(v.name)
 print('-' * 40)
 for ckpt_no in ckpts:
if RESUME_TRAINING or only_valid:
ckpt_manager, loaded_ckpt_no = load_ckpt(
 predictor, optimizer,
CKPT_TRAIN_PATH, ckpt_no=ckpt_no 
)
 else:
with strategy.scope() :
loaded_ckpt_no = 0
checkpoint = tf.train.Checkpoint(optimizer=optimizer, model=predictor)
 ckpt_path = CKPT_TRAIN_PATH
ckpt_manager = tf.train.CheckpointManager(checkpoint, directory=ckpt_path, max_to_keep=None)
 train_manager.train_valid(
 predictor, optimizer, ckpt_manager,
loss_obj, loss_obj_answer, metrics,
last_epoch=loaded_ckpt_no,
from_valid=from_valid,
valid_epochs=valid_epochs,
only_valid=only_valid
)
housing_all_df.loc[~housing_all_df['Neighborhood'].isin(['StoneBr','NridgHt','NoRidge','MeadowV','IDOTRR','BrDale']), 'Neighborhood'] = 1
housing_all_df.loc[housing_all_df['Neighborhood'].isin(['StoneBr','NridgHt','NoRidge']), 'Neighborhood'] = 2
housing_all_df.loc[housing_all_df['Neighborhood'].isin(['MeadowV','IDOTRR','BrDale']), 'Neighborhood'] = 0<data_type_conversions>
CKPT_TRAIN_PATH
housing_all_df['Neighborhood'] = housing_all_df['Neighborhood'].astype(int )<categorify>
if CKPT_TRAIN_PATH.startswith('gs://'):
!gsutil ls '{CKPT_TRAIN_PATH}'
else:
!ls -l '{CKPT_TRAIN_PATH}'
all_dummies = []
cat_cols = housing_all_df.select_dtypes(include='object' ).columns.tolist()
cat_cols.extend(['MSSubClass', 'MoSold','YrSold'])
 for col in cat_cols:
dummies = pd.get_dummies(housing_all_df[col], prefix=col, prefix_sep='_', drop_first=True)
 for dum_col in dummies.columns.tolist() :
if dummies[dum_col].sum() < 10:
print("Dummy: {0} has less than 10 records".format(dum_col))
dummies.drop(columns=[dum_col], inplace =True)
 all_dummies.extend(dummies)
 housing_all_df.drop(columns=col, inplace=True)
 housing_all_df = pd.concat([housing_all_df, dummies], axis=1 )<split>
predictor = TFEdFormerAnswerPredictionModel(config)
 if(( VALID or PRED)and not DUMMY)or SUBMISSION:
assert CKPT_PRED_PATH is not None and CKPT_PRED_PATH != './'
X = housing_train_df.drop(columns=['SalePrice', 'SalePrice_log'])
 y = housing_train_df['SalePrice_log']
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size = 0.7, test_size = 0.3, random_state = 100 )<compute_test_metric>
if VALID:
valid_results = {'ckpt_no': [], 'valid_loss': [], 'valid_acc': [], 'valid_auc': []}
ckpts_to_run = [60]
print(f'ckpts_to_run: {ckpts_to_run}')
 for ckpt_no in ckpts_to_run:
if ckpt_no > 0:
ckpt_manager, loaded_ckpt_no = load_ckpt(
 predictor=predictor, optimizer=None,
ckpt_path=CKPT_PRED_PATH, ckpt_no=ckpt_no, 
)
 assert loaded_ckpt_no > 0
pred_iter_manager_class = Valid_Iter_Manager
pred_manager = Pred_Manager.create(
 config=config,
train_config=train_config,
pred_iter_manager_class=pred_iter_manager_class,
train_dt_path_or_obj=train_dt,
user_id_to_row_id_train_path=user_id_to_row_id_train_path,
unique_question_id_train_path=unique_question_id_splitted_train_path,
unique_lecture_id_train_path=unique_lecture_id_splitted_train_path,
question_history_at_training_end_path=question_history_at_training_end_for_valid_path,
single_question_history_at_training_end_optimized_path=single_question_history_at_training_end_optimized_for_valid_path,
valid_info_path=(valid_info_paths[VALID_FOLD] if VALID_FOLD in [0, 1, 2, 3] else None),
max_train_buffer_size=30000,
probe=False,
debug=False 
)
 run_pred(pred_manager, predictor, max_steps=MAX_VALID_ITER_STEPS, ckpt_no=ckpt_no)
 with open(f'valid_results_ckpt_{ckpt_no}.json', 'r', encoding='UTF-8')as fp:
valid_results_ckpt = json.load(fp)
 valid_results['ckpt_no'].append(ckpt_no)
 valid_results['valid_loss'].append(valid_results_ckpt['valid_loss'])
 valid_results['valid_acc'].append(valid_results_ckpt['valid_acc'])
 valid_results['valid_auc'].append(valid_results_ckpt['valid_auc'])
 print('=' * 80)
 with open(f'valid_results.json', 'w', encoding='UTF-8')as fp:
json.dump(valid_results, fp, ensure_ascii=False, indent=True)
 if not IS_KAGGLE:
!gsutil cp -r './valid_results.json' '{CKPT_PRED_PATH}'
def root_mean_squared_error(actual, predicted):
return np.sqrt(metrics.mean_squared_error(actual,predicted))
custom_scorer = metrics.make_scorer(root_mean_squared_error, greater_is_better = False )<train_on_grid>
valid_df = None
if VALID:
valid_df = pd.read_csv(f'valid_submission_ckpt_{ckpts_to_run[-1]}.csv')
 pd.set_option('display.max_rows', 200)
 valid_df
def grid_search_cv(params, model, X_train, y_train, X_test, y_test):
folds = 5
model_cv = GridSearchCV(estimator = model,
param_grid = params,
scoring = custom_scorer,
cv = folds,
verbose = True,
return_train_score = True,
n_jobs=-1)
 model_cv.fit(X_train, y_train)
 print("Best score: ", model_cv.best_score_)
 print("Best params: ", model_cv.best_params_)
 y_test_pred = model_cv.best_estimator_.predict(X_test)
 print("Test score: ", root_mean_squared_error(y_test, y_test_pred))<train_on_grid>
valid_results = {}
if VALID:
with open('valid_results.json', 'r', encoding='UTF-8')as fp:
valid_results = json.load(fp)
 valid_results
params = {'model__alpha' : [0.0001, 0.001,0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.2, 0.5, 1, 2,
5, 10, 50]}
lasso = Pipeline([('scaler',RobustScaler()),('model',Lasso(random_state = 100, normalize=True)) ])
 grid_search_cv(params, lasso, X_train, y_train, X_test, y_test )<compute_train_metric>
pred_start = datetime.datetime.now()
elapsed =(pred_start - kernel_start ).total_seconds()
print(f'time before prediction: {elapsed} seconds')
 if PRED:
ckpt_no = 60
ckpt_manager, loaded_ckpt_no = load_ckpt(
 predictor=predictor, optimizer=None,
ckpt_path=CKPT_PRED_PATH, ckpt_no=ckpt_no, 
)
 if SUBMISSION:
assert loaded_ckpt_no > 0
pred_iter_manager_class = Test_Iter_Manager
pred_manager = Pred_Manager.create(
 config=config,
train_config=train_config,
pred_iter_manager_class=pred_iter_manager_class,
train_dt_path_or_obj=train_dt,
user_id_to_row_id_train_path=user_id_to_row_id_train_path,
unique_question_id_train_path=unique_question_id_train_path,
unique_lecture_id_train_path=unique_lecture_id_train_path,
question_history_at_training_end_path=question_history_at_training_end_path,
single_question_history_at_training_end_optimized_path=single_question_history_at_training_end_optimized_path,
valid_info_path=None,
max_train_buffer_size=30000,
probe=PROBE,
debug=False 
)
 run_pred(pred_manager, predictor, max_steps=None )