Kaggle competition: Riiid Answer Correctness Prediction
<SOS> metric: AUC Kaggle data source: riiid-answer-correctness-prediction<import_modules>
kernel_start = datetime.datetime.now()
print(f'kernel_start: {kernel_start}' )
from sklearn.model_selection import GridSearchCV<load_from_csv>
!pip uninstall -y cloud-tpu-client
if not IS_KAGGLE:
!pip uninstall -y tensorflow
!pip install --upgrade tensorflow==2.3.0
!pip uninstall -y tensorflow-gcs-config
!pip install --upgrade tensorflow-gcs-config==2.3.0
elif IS_KAGGLE and not USE_GPU:
!pip uninstall -y tensorflow
!pip install --upgrade tensorflow==2.3.1
print(f'TensorFlow: {tf.__version__}')
 print(client.Client )
train_df = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_df = pd.read_csv('/kaggle/input/titanic/test.csv')
 print(train_df.shape)
 print(test_df.shape)
 train_df.head()<sort_values>
if IS_KAGGLE:
BASE_DIR = '/kaggle/input'
else:
BUCKET_DIR = 'gs://shieh-tpu/r3id'
BASE_DIR = '/content/drive/My Drive/r3id'
BASE_DIR_QUOTED = '"/content/drive/My Drive/r3id"'
if not IS_KAGGLE:
auth.authenticate_user()
project_id = 'shieh-tpu'
!gcloud config set project {project_id}
!gsutil ls {BUCKET_DIR}
drive.mount('/content/drive')
 if not os.path.isdir(BASE_DIR):
!mkdir {BASE_DIR_QUOTED}
!gsutil -m cp -r 'gs://shieh-tpu/r3id' "/content/drive/My Drive"
!ls -l '{BASE_DIR}'
train_df[['Pclass','Survived']].groupby('Pclass' ).mean().sort_values(by = 'Survived', ascending = False )<sort_values>
try:
client.Client().configure_tpu_version(tf.__version__, restart_type='ifNeeded')
 if IS_KAGGLE:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
else:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
 print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
train_df[['Sex','Survived']].groupby('Sex' ).mean().sort_values(by = 'Survived', ascending = False )<sort_values>
!pip uninstall -y datatable
!pip uninstall -y tokenizers
!pip uninstall -y transformers
if IS_KAGGLE:
!pip install '{BASE_DIR + '/r3id-packages/datatable-0.11.0-cp37-cp37m-manylinux2010_x86_64.whl'}'
!pip install --upgrade '{BASE_DIR + '/r3id-packages/tokenizers-0.9.2-cp37-cp37m-manylinux1_x86_64.whl'}'
else:
!pip install '{BASE_DIR + '/r3id-packages/datatable-0.11.0-cp36-cp36m-manylinux2010_x86_64.whl'}'
!pip install --upgrade '{BASE_DIR + '/r3id-packages/tokenizers-0.9.2-cp36-cp36m-manylinux1_x86_64.whl'}'
!pip install --upgrade '{BASE_DIR + '/r3id-packages/transformers-3.4.0-py3-none-any.whl'}'
train_df[['SibSp','Survived']].groupby('SibSp' ).mean().sort_values(by = 'Survived', ascending = False )<sort_values>
os.environ['TF_DETERMINISTIC_OPS'] = '1'
if IS_KAGGLE and USE_GPU:
gc.enable()
TFPreTrainedModel,
keras_serializable,
shape_list, 
)
 if IS_KAGGLE:
train_df[['Parch','Survived']].groupby('Parch' ).mean().sort_values(by = 'Survived', ascending = False )<categorify>
train_dt_path_or_obj = f'{BASE_DIR}/r3id-trainjay/train.jay'
unique_user_id_train_path = f'{BASE_DIR}/r3id-info/unique_user_id_train.json'
unique_question_id_train_path = f'{BASE_DIR}/r3id-info/unique_question_id_train.json'
unique_lecture_id_train_path = f'{BASE_DIR}/r3id-info/unique_lecture_id_train.json'
user_id_to_row_id_train_path = f'{BASE_DIR}/r3id-info/user_id_to_row_id_train.json'
unique_user_id_splitted_train_path = f'{BASE_DIR}/r3id-info-shieh/unique_user_id_splitted_train.json'
unique_question_id_splitted_train_path = f'{BASE_DIR}/r3id-info-shieh/unique_question_id_splitted_train.json'
unique_lecture_id_splitted_train_path = f'{BASE_DIR}/r3id-info-shieh/unique_lecture_id_splitted_train.json'
n_contents_dict_path = f'{BASE_DIR}/r3id-info-shieh/n_contents_dict.json'
question_tags_info_path = f'{BASE_DIR}/r3id-info-shieh/question_tags_info.json'
lecture_tags_info_path = f'{BASE_DIR}/r3id-info-shieh/lecture_tags_info.json'
question_part_info_path = f'{BASE_DIR}/r3id-info-shieh/question_part_info.json'
lecture_part_info_path = f'{BASE_DIR}/r3id-info-shieh/lecture_part_info.json'
correct_answer_info_path = f'{BASE_DIR}/r3id-info-shieh/correct_answer_info.json'
question_history_path = f'{BASE_DIR}/r3id-info-shieh/question_history.json'
question_history_at_training_end_path = f'{BASE_DIR}/r3id-info-shieh/question_history_at_training_end.json'
single_question_history_at_training_end_optimized_path = f'{BASE_DIR}/r3id-info-shieh/single_question_history_at_training_end.json'
user_performance_hdf5_path = f'{BASE_DIR}/user-performance/user_performance.hdf5'
question_history_for_valid_path = f'{BASE_DIR}/r3id-info-shieh/question_history_fold_1.json'
question_history_at_training_end_for_valid_path = f'{BASE_DIR}/r3id-info-shieh/question_history_at_training_end_fold_1.json'
single_question_history_at_training_end_optimized_for_valid_path = f'{BASE_DIR}/r3id-info-shieh/single_question_history_at_training_end_fold_1.json'
valid_info_paths = [
f'{BASE_DIR}/r3id-info-shieh/valid_info.json',
f'{BASE_DIR}/r3id-info-shieh/valid_info_fold_1.json',
f'{BASE_DIR}/r3id-info-shieh/valid_info_fold_2.json',
f'{BASE_DIR}/r3id-info-shieh/valid_info_fold_3.json',
]
train_valid_split_indices_paths = [
f'{BASE_DIR}/r3id-info-shieh/train_valid_split_indices.json',
f'{BASE_DIR}/r3id-info-shieh/train_valid_split_indices_fold_1.json',
f'{BASE_DIR}/r3id-info-shieh/train_valid_split_indices_fold_2.json',
f'{BASE_DIR}/r3id-info-shieh/train_valid_split_indices_fold_3.json',
]
if tpu is None:
TFREC_DIR = BASE_DIR
train_tfrec_dir = f'{BASE_DIR}/ednet-tfrecords-sequential-more'
valid_tfrec_dir = f'{BASE_DIR}/r3id-tfrecords-valid-more'
elif IS_KAGGLE:
train_tfrec_dir = KaggleDatasets().get_gcs_path('ednet-tfrecords-sequential-more')
 valid_tfrec_dir = KaggleDatasets().get_gcs_path('r3id-tfrecords-valid-more')
 print(f'train_tfrec_dir from Kaggle = {train_tfrec_dir}
')
 print(f'valid_tfrec_di from Kaggle = {valid_tfrec_dir}
')
 !gsutil ls $train_tfrec_dir
!gsutil ls $valid_tfrec_dir
else:
TFREC_DIR = BUCKET_DIR
train_tfrec_dir = f'{BUCKET_DIR}/ednet-tfrecords-sequential-more'
valid_tfrec_dir = f'{BUCKET_DIR}/r3id-tfrecords-valid-more'
train_dt = dt.fread(train_dt_path_or_obj )
labelencoder = LabelEncoder()
train_df['Sex'] = labelencoder.fit_transform(train_df['Sex'])
 test_df['Sex'] = labelencoder.fit_transform(test_df['Sex'])
 test_df.head()<sort_values>
MODEL_TYPE = 'ed'
MODEL_SIZE = 'b-2'
MODEL_DESC = 'raw'
ACTIVATION = 'relu'
USE_PRE_CLASSIFIER = True
USE_SOFTMAX = False
USE_USER_ANSWER = False
USE_USER_ANSWER_LOSS = False
USE_CORRECT_ANSWER_FOR_ENCODER = False
USE_CORRECT_ANSWER_FOR_DECODER = False
USE_ABS_POS = False
USE_TASK_CONTAINER_POS = False
SHARE_POS_EMBEDDING = True
USE_TAGS = False
USE_PART = False
USE_PRIOR_EXPLANATION = False
USE_PRIOR_QUESTION_ELAPSED_TIME_INPUT = False
USE_LAG_TIME = False
USE_LAG_TIME_FOR_ENCODER = False
USE_USER_LEVEL_AGGREGATED_HISTORICAL_INFO = False
USE_PART_AGGREGATED_HISTORICAL_INFO = False
USE_CORRECT_ANSWER_AGGREGATED_HISTORICAL_INFO = False
USE_QUESTION_LEVEL_AGGREGATED_HISTORICAL_INFO = False
ALLOW_BUNDLE_ATTEN = False
GENERATIVE = False
VALID_FOLD = 0
WINDOW_SIZE = 128
LOSS_WEIGHT_WINDOW_SIZE = None
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
if not tpu:
BATCH_SIZE = 8 * BATCH_SIZE
PRED_BATCH_SIZE = 256 * strategy.num_replicas_in_sync
if tpu:
assert BATCH_SIZE // strategy.num_replicas_in_sync != WINDOW_SIZE
assert PRED_BATCH_SIZE // strategy.num_replicas_in_sync != WINDOW_SIZE
N_EPOCHS = 2
STEPS_PER_CALL = 1000
MAX_N_CONTENTS_PER_USER_FOR_SAMPLING_PROB = 512
LR = 1e-4
END_LR = 1e-4
WARMUP_STEPS = 5000
DETERMINISTIC = False
if DETERMINISTIC:
SEED = 2021
N_PARALLEL_READS = None
N_PARALLEL_CALLS = None
SHUFFLE_BUFFER_SIZE = 1
else:
SEED = None
N_PARALLEL_READS = 16
N_PARALLEL_CALLS = tf.data.experimental.AUTOTUNE
SHUFFLE_BUFFER_SIZE = 4096
MAX_TRAIN_ITER_STEPS = None
MAX_VALID_ITER_STEPS = None
PRINTING_STEPS = 1000
CKPT_DIR = f'{MODEL_TYPE}-{MODEL_SIZE}-{MODEL_DESC}/'
train_df['Family size'] = train_df['SibSp'] + train_df['Parch'] + 1
test_df['Family size'] = test_df['SibSp'] + test_df['Parch'] + 1
train_df[['Family size','Survived']].groupby('Family size' ).mean().sort_values(by = 'Survived', ascending = False )<categorify>
DTYPE = tf.float32
tf.keras.backend.set_floatx('float32')
 TRAIN = True
VALID = False
PRED = False
RESUME_TRAINING = False
N_FILES = 6
SUBMISSION = False
if IS_KAGGLE:
N_FILES = len(os.listdir('/kaggle/input/riiid-test-answer-prediction'))
SUBMISSION =(N_FILES != 6)
 else:
PRED = False
if SUBMISSION:
TRAIN = False
VALID = False
PRED = True
if not TRAIN:
RESUME_TRAINING = False
DEBUG = True
PROBE = False
CKPT_TRAIN_PATH = None
CKPT_PRED_PATH = None
if CKPT_TRAIN_PATH is None:
if IS_KAGGLE:
CKPT_TRAIN_PATH = './'
else:
CKPT_TRAIN_PATH = f'{BUCKET_DIR}/r3id-ckpts/{CKPT_DIR}'
if TRAIN and not RESUME_TRAINING:
_state = !gsutil -q stat {CKPT_TRAIN_PATH}*; echo $?
already_existed = 1 - int(_state[0])
 assert not already_existed
if CKPT_PRED_PATH is None:
if IS_KAGGLE:
CKPT_PRED_PATH = f'{BASE_DIR}/{CKPT_DIR}'
else:
CKPT_PRED_PATH = f'{BUCKET_DIR}/r3id-ckpts/{CKPT_DIR}'
train_df['Fam_type'] = pd.cut(train_df['Family size'], [0,1,4,7,11], labels=['Solo', 'Small', 'Big', 'Very big'])
 test_df['Fam_type'] = pd.cut(test_df['Family size'], [0,1,4,7,11], labels=['Solo', 'Small', 'Big', 'Very big'] )<feature_engineering>
PAD_TOKEN = -2
START_TOKEN = -3
END_TOKEN = -4
MASK_TOKEN = -5
PAD_ID = 0
START_ID = 1
END_ID = 2
MASK_ID = 3
RESPONSE_LECTURE_TOKEN = -1
RESPONSE_FALSE_TOKEN = 0
RESPONSE_TRUE_TOKEN = 1
ANSWER_LECTURE_TOKEN = -1
ANSWER_0_TOKEN = 0
ANSWER_1_TOKEN = 1
ANSWER_2_TOKEN = 2
ANSWER_3_TOKEN = 3
RESPONSE_LECTURE_ID = 4
RESPONSE_FALSE_ID = 5
RESPONSE_TRUE_ID = 6
ANSWER_LECTURE_ID = 4
ANSWER_0_ID = 5
ANSWER_1_ID = 6
ANSWER_2_ID = 7
ANSWER_3_ID = 8
NON_TARGET_ID = -100
TAG_VOCAB_SIZE = 189
PART_VOCAB_SIZE = 9
PRIOR_EXPLANATION_VOCAB_SIZE = 3
N_TAGS_PER_CONTENT = 6
MAX_TRAIN_HISTORY_LEN = 17917
MAX_HISTORY_LEN = 20480
MAX_TASK_CONTAINER_ID = 10000
N_AGGREGATED_QUESTION_SCALING_FACTOR = 1000
combine = [train_df, test_df]
for dataset in combine:
dataset['Title'] = dataset.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(train_df['Title'], train_df['Sex'] )<feature_engineering>
MAX_PRED_TIME_QUESTION_BUNDLE_LEN = 10
for dataset in combine:
dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')
 dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')
 dataset['Title'] = dataset['Title'].replace(['Lady', 'Countess','Don', 'Sir', 'Jonkheer', 'Dona'],'Royalty')
 dataset['Title'] = dataset['Title'].replace(['Capt', 'Col','Dr','Major','Rev'],'Special')
 train_df[['Title','Survived']].groupby(['Title'], as_index=False ).mean()<prepare_x_and_y>
def convert_valid_info(valid_info):
data = {}
user_ids = list(valid_info.keys())
 for user_id in user_ids:
user_valid_info = valid_info[user_id]
user_valid_info['bundle_info']['block_starting_index_dict'] = {int(k): v for k, v in user_valid_info['bundle_info']['block_starting_index_dict'].items() }
user_valid_info['bundle_info']['bundle_starting_index_dict'] = {int(k): v for k, v in user_valid_info['bundle_info']['bundle_starting_index_dict'].items() }
data[int(user_id)] = user_valid_info
del valid_info[user_id]
return data
def load_data(path):
if path.endswith('.json'):
with open(path, 'r', encoding='UTF-8')as fp:
data = json.load(fp)
 if path.endswith('valid_info.json')or 'valid_info_fold_' in path:
data = convert_valid_info(data)
 elif path.endswith('n_contents_dict.json'):
data = {int(k): v for k, v in data.items() }
elif path.endswith('user_id_to_row_id_train.json')or 'train_valid_split_indices' in path:
data = {int(k): v for k, v in data.items() }
elif path.endswith('question_tags_info.json')or path.endswith('lecture_tags_info.json'):
data = {int(k): v for k, v in data.items() }
elif path.endswith('question_part_info.json')or path.endswith('lecture_part_info.json'):
data = {int(k): v for k, v in data.items() }
elif path.endswith('correct_answer_info.json'):
data = {int(k): v for k, v in data.items() }
elif path.endswith('question_history.json'):
pass
elif path.endswith('question_history_at_training_end.json'):
pass
elif path.endswith('single_question_history_at_training_end.json'):
pass
return data
elif path.endswith('train.jay'):
train_dt = dt.fread(train_dt_path_or_obj)
 return train_dt
else:
raise ValueError('The path provided is not the one needed to create an instance of `Valid_Manager`!' )
y = train_df['Survived']
features = ['Pclass','Sex','Fam_type','Fare','Age Bin','Embarked']
X = train_df[features]
X.head()<categorify>
questioin_df = pd.read_csv(f'{BASE_DIR}/riiid-test-answer-prediction/questions.csv')
 lecture_df = pd.read_csv(f'{BASE_DIR}/riiid-test-answer-prediction/lectures.csv')
 question_ids = questioin_df['question_id'].tolist()
lecture_ids = lecture_df['lecture_id'].tolist()
question_ids = sorted(question_ids)
 lecture_ids = sorted(lecture_ids)
 assert question_ids == sorted(question_ids)
 assert lecture_ids == sorted(lecture_ids)
 special_vocab = [PAD_TOKEN, START_TOKEN, END_TOKEN, MASK_TOKEN]
question_vocab = question_ids
lecture_vocab = lecture_ids
content_vocab = special_vocab + question_vocab + lecture_vocab
input_ids = tf.range(len(special_vocab), len(special_vocab)+ len(question_vocab))
initializer = tf.lookup.KeyValueTensorInitializer(tf.constant(question_ids, dtype=tf.int32), input_ids)
 question_id_to_input_id_table = tf.lookup.StaticHashTable(
 initializer, default_value=tf.int32.limits[1], name=None 
)
 input_ids = tf.range(len(special_vocab)+ len(question_vocab), len(content_vocab))
initializer = tf.lookup.KeyValueTensorInitializer(tf.constant(lecture_ids, dtype=tf.int32), input_ids)
 lecture_id_to_input_id_table = tf.lookup.StaticHashTable(
 initializer, default_value=tf.int32.limits[1], name=None 
)
 response_vocab = [PAD_TOKEN, START_TOKEN, END_TOKEN, MASK_TOKEN, RESPONSE_LECTURE_TOKEN, RESPONSE_TRUE_TOKEN, RESPONSE_FALSE_TOKEN]
answer_vocab = [
PAD_TOKEN, START_TOKEN, END_TOKEN, MASK_TOKEN,
ANSWER_LECTURE_TOKEN,
ANSWER_0_TOKEN, ANSWER_1_TOKEN, ANSWER_2_TOKEN, ANSWER_3_TOKEN
]
CONTENT_VOCAB_SIZE = len(content_vocab)
 RESPONSE_VOCAB_SIZE = len(response_vocab)
 ANSWER_VOCAB_SIZE = len(answer_vocab)
 question_tags_info = load_data(question_tags_info_path)
 lecture_tags_info = load_data(lecture_tags_info_path)
 correct_answer_info = load_data(correct_answer_info_path)
 question_part_info = load_data(question_part_info_path)
 lecture_part_info = load_data(lecture_part_info_path)
 tags_database = []
part_database = []
correct_answer_table = []
for idx in range(len(content_vocab)) :
if idx < len(special_vocab):
tags_database.append([-1] * N_TAGS_PER_CONTENT)
 part_database.append(-1)
 correct_answer_table.append(idx)
 elif idx < len(special_vocab)+ len(question_vocab):
index_in_questions_ids = idx - len(special_vocab)
 question_id = question_ids[index_in_questions_ids]
tags_database.append(question_tags_info[question_id])
 part_database.append(question_part_info[question_id])
 correct_answer_table.append(correct_answer_info[question_id] + ANSWER_0_ID)
 elif idx < len(content_vocab):
index_in_lecture_ids = idx -(len(special_vocab)+ len(question_vocab))
lecture_id = lecture_ids[index_in_lecture_ids]
tags_database.append(lecture_tags_info[lecture_id])
 part_database.append(lecture_part_info[lecture_id])
 correct_answer_table.append(ANSWER_LECTURE_ID)
 c_inputs_ids_to_tags = tf.constant(tags_database, dtype=tf.int32)
 c_inputs_ids_to_part = tf.constant(part_database, dtype=tf.int32)
 c_inputs_ids_to_correct_answer_id = tf.constant(correct_answer_table, dtype=tf.int32)
 question_id_to_input_id_dict = {
k: v for k, v in zip(question_ids, range(len(special_vocab), len(special_vocab)+ len(question_vocab)))
 }
c_inputs_ids_to_part_dict = {k: v for k, v in enumerate(part_database)}
c_inputs_ids_to_correct_answer_id_dict = {k: v for k, v in enumerate(correct_answer_table)}
numerical_col = ['Fare']
categorical_col = ['Pclass','Sex','Fam_type','Age Bin','Embarked']
num_trans = SimpleImputer(strategy = 'median')
 cat_trans = Pipeline(steps = [ 
('imputer',SimpleImputer(strategy = 'most_frequent')) , 
('onehot',OneHotEncoder())
 ])
 preprocessor = ColumnTransformer(
 transformers = [ 
('num',num_trans,numerical_col), 
('cat',cat_trans,categorical_col)
 ])
 titanic_pipeline = Pipeline(steps=[('preprocessor', preprocessor), 
('model', RandomForestClassifier(random_state=0 
))
])
 param_grid = {
'model__max_depth': [2, 3, 4, 5],
'model__min_samples_leaf': [3, 4, 5],
'model__min_samples_split': [6, 8, 10, 12],
'model__n_estimators': [100, 200, 300, 500]
}
search = GridSearchCV(titanic_pipeline, param_grid, n_jobs=-1)
 search.fit(X, y)
 print(search.best_params_)
<categorify>
if not IS_KAGGLE:
train_tfrec_fns = [f'EdNet-user-history-{idx}.tfrecord' for idx in range(41)]
train_tfrec_fns = sorted(train_tfrec_fns, key=lambda x: int(x.replace('EdNet-user-history-', '' ).replace('.tfrecord', '')))
 if tpu is None or IS_KAGGLE:
train_tfrec_paths = [os.path.join(train_tfrec_dir, fn)for fn in train_tfrec_fns]
else:
train_tfrec_paths = [train_tfrec_dir + f'/{fn}' for fn in train_tfrec_fns]
train_tfrec_paths
numerical_col = ['Fare']
categorical_col = ['Pclass','Sex','Fam_type','Age Bin','Embarked']
num_trans = SimpleImputer(strategy = 'median')
 cat_trans = Pipeline(steps = [ 
('imputer',SimpleImputer(strategy = 'most_frequent')) , 
('onehot',OneHotEncoder())
 ])
 preprocessor = ColumnTransformer(
 transformers = [ 
('num',num_trans,numerical_col), 
('cat',cat_trans,categorical_col)
 ])
 titanic_pipeline = Pipeline(steps=[('preprocessor', preprocessor), 
('model', RandomForestClassifier(random_state=0,
max_depth = 5,
n_estimators = 500 
))
])
 titanic_pipeline.fit(X,y)
 print('Cross validation score: {:.3f}'.format(cross_val_score(titanic_pipeline, X, y, cv=10 ).mean()))<save_to_csv>
if not IS_KAGGLE:
valid_tfrec_fns = [
'EdNet-user-history-with-valid-block-pos.tfrecord',
]
if tpu is None:
valid_tfrec_paths = [os.path.join(valid_tfrec_dir, fn)for fn in valid_tfrec_fns]
else:
valid_tfrec_paths = [valid_tfrec_dir + f'/{fn}' for fn in valid_tfrec_fns]
valid_tfrec_paths
predictions = titanic_pipeline.predict(X_test)
 output = pd.DataFrame({'PassengerId': test_df.PassengerId, 'Survived': predictions})
 output.to_csv('my_submission2.csv', index=False)
 print("Your submission was successfully saved!" )<set_options>
train_raw_features = {
'user_id': tf.io.FixedLenFeature([], dtype=tf.int64),
'row_id': tf.io.RaggedFeature(value_key='row_id', dtype=tf.int64),
'timestamp': tf.io.RaggedFeature(value_key='timestamp', dtype=tf.int64),
'content_id': tf.io.RaggedFeature(value_key='content_id', dtype=tf.int64),
'content_type_id': tf.io.RaggedFeature(value_key='content_type_id', dtype=tf.int64),
'task_container_id': tf.io.RaggedFeature(value_key='task_container_id', dtype=tf.int64),
'user_answer': tf.io.RaggedFeature(value_key='user_answer', dtype=tf.int64),
'answered_correctly': tf.io.RaggedFeature(value_key='answered_correctly', dtype=tf.int64),
'prior_question_elapsed_time': tf.io.RaggedFeature(value_key='prior_question_elapsed_time', dtype=tf.float32),
'prior_question_had_explanation': tf.io.RaggedFeature(value_key='prior_question_had_explanation', dtype=tf.int64),
'n_prev_seen': tf.io.RaggedFeature(value_key='n_prev_seen', dtype=tf.int64),
'n_prev_correctness': tf.io.RaggedFeature(value_key='n_prev_correctness', dtype=tf.int64),
}
def parse_train_example(example):

 _parsed = tf.io.parse_single_example(example, train_raw_features)
 parsed = {}
parsed['user_id'] = _parsed['user_id']
parsed['seq_len'] = tf.reduce_sum(tf.ones_like(_parsed['row_id'], dtype=tf.int32))
parsed['prev_seq_len'] = parsed['seq_len']
parsed['start'] = tf.constant(0, dtype=tf.int32)
 parsed['end'] = parsed['seq_len'] - 1
for k in train_raw_features:
data = _parsed[k]
if k not in ['row_id', 'user_id', 'timestamp', 'prior_question_elapsed_time']:
data = tf.cast(data, dtype=tf.int32)
 elif k == 'prior_question_elapsed_time':
data = tf.cast(data, dtype=DTYPE)
 if k != 'user_id':
parsed[k] = data
parsed['shifted_answered_correctly'] = tf.concat([[START_TOKEN], parsed['answered_correctly']], axis=0)
 parsed['shifted_user_answer'] = tf.concat([[START_TOKEN], parsed['user_answer']], axis=0)
 pad_mask = tf.cast(parsed['row_id'] == PAD_TOKEN, dtype=tf.int32)
 pred_time_mask = tf.zeros_like(parsed['timestamp'], dtype=tf.int32)
 pred_time_mask = pred_time_mask *(1 - pad_mask)+(PAD_TOKEN)* pad_mask
parsed['pred_time_mask'] = pred_time_mask
parsed['abs_pos'] = tf.range(parsed['seq_len'], dtype=tf.int32)
 parsed['shifted_abs_pos'] = tf.concat([[START_TOKEN], parsed['abs_pos'][:-1]], axis=0)
 diff_t = parsed['timestamp'] - tf.concat([[1], parsed['timestamp'][:-1]], axis=0)
 bundle_start_mask = tf.cast(diff_t != 0, dtype=tf.int32)
 jump_t = tf.boolean_mask(diff_t, bundle_start_mask > 0)
 task_container_id_at_jumps = tf.boolean_mask(parsed['task_container_id'], bundle_start_mask > 0)
 task_containder_id_to_jump_t = tf.scatter_nd(
 indices=task_container_id_at_jumps[:, tf.newaxis], updates=jump_t, shape=tf.math.reduce_max(task_container_id_at_jumps + 1)[tf.newaxis], name=None 
)
 lag_time = tf.gather(params=task_containder_id_to_jump_t , indices=parsed['task_container_id'])
 lag_time = tf.math.maximum(tf.cast(0, dtype=tf.int64), lag_time)
 _mask = tf.cast(tf.math.logical_and(lag_time == 0, parsed['timestamp'] > 0), dtype=tf.int64)
 lag_time = lag_time *(1 - _mask)+ 1000 * _mask
parsed['lag_time'] = lag_time
return parsed
train_features_with_valid_info = {
'user_id': tf.io.FixedLenFeature([], dtype=tf.int64),
'row_id': tf.io.RaggedFeature(value_key='row_id', dtype=tf.int64),
'timestamp': tf.io.RaggedFeature(value_key='timestamp', dtype=tf.int64),
'content_id': tf.io.RaggedFeature(value_key='content_id', dtype=tf.int64),
'content_type_id': tf.io.RaggedFeature(value_key='content_type_id', dtype=tf.int64),
'task_container_id': tf.io.RaggedFeature(value_key='task_container_id', dtype=tf.int64),
'user_answer': tf.io.RaggedFeature(value_key='user_answer', dtype=tf.int64),
'answered_correctly': tf.io.RaggedFeature(value_key='answered_correctly', dtype=tf.int64),
'prior_question_elapsed_time': tf.io.RaggedFeature(value_key='prior_question_elapsed_time', dtype=tf.float32),
'prior_question_had_explanation': tf.io.RaggedFeature(value_key='prior_question_had_explanation', dtype=tf.int64),
'n_prev_seen': tf.io.RaggedFeature(value_key='n_prev_seen', dtype=tf.int64),
'n_prev_correctness': tf.io.RaggedFeature(value_key='n_prev_correctness', dtype=tf.int64),
'n_valid_blocks': tf.io.FixedLenFeature([], dtype=tf.int64),
'valid_blocks_start_pos': tf.io.RaggedFeature(value_key='valid_blocks_start_pos', dtype=tf.int64),
'valid_blocks_end_pos': tf.io.RaggedFeature(value_key='valid_blocks_end_pos', dtype=tf.int64),
}
def parse_train_example_with_valid_info(example):
_parsed = tf.io.parse_single_example(example, train_features_with_valid_info)
 parsed = {}
parsed['user_id'] = _parsed['user_id']
parsed['seq_len'] = tf.reduce_sum(tf.ones_like(_parsed['row_id'], dtype=tf.int32))
parsed['prev_seq_len'] = parsed['seq_len']
parsed['start'] = tf.constant(0, dtype=tf.int32)
 parsed['end'] = parsed['seq_len'] - 1
for k in train_features_with_valid_info:
data = _parsed[k]
if k not in ['row_id', 'user_id', 'timestamp', 'prior_question_elapsed_time']:
data = tf.cast(data, dtype=tf.int32)
 elif k == 'prior_question_elapsed_time':
data = tf.cast(data, dtype=DTYPE)
 if k != 'user_id':
parsed[k] = data
parsed['shifted_answered_correctly'] = tf.concat([[START_TOKEN], parsed['answered_correctly']], axis=0)
 parsed['shifted_user_answer'] = tf.concat([[START_TOKEN], parsed['user_answer']], axis=0)
 pad_mask = tf.cast(parsed['row_id'] == PAD_TOKEN, dtype=tf.int32)
 pred_time_mask = tf.zeros_like(parsed['timestamp'], dtype=tf.int32)
 pred_time_mask = pred_time_mask *(1 - pad_mask)+(PAD_TOKEN)* pad_mask
parsed['pred_time_mask'] = pred_time_mask
parsed['abs_pos'] = tf.range(parsed['seq_len'], dtype=tf.int32)
 parsed['shifted_abs_pos'] = tf.concat([[START_TOKEN], parsed['abs_pos'][:-1]], axis=0)
 diff_t = parsed['timestamp'] - tf.concat([[1], parsed['timestamp'][:-1]], axis=0)
 bundle_start_mask = tf.cast(diff_t != 0, dtype=tf.int32)
 jump_t = tf.boolean_mask(diff_t, bundle_start_mask > 0)
 task_container_id_at_jumps = tf.boolean_mask(parsed['task_container_id'], bundle_start_mask > 0)
 task_containder_id_to_jump_t = tf.scatter_nd(
 indices=task_container_id_at_jumps[:, tf.newaxis], updates=jump_t, shape=tf.math.reduce_max(task_container_id_at_jumps + 1)[tf.newaxis], name=None 
)
 lag_time = tf.gather(params=task_containder_id_to_jump_t , indices=parsed['task_container_id'])
 lag_time = tf.math.maximum(tf.cast(0, dtype=tf.int64), lag_time)
 _mask = tf.cast(tf.math.logical_and(lag_time == 0, parsed['timestamp'] > 0), dtype=tf.int64)
 lag_time = lag_time *(1 - _mask)+ 1000 * _mask
parsed['lag_time'] = lag_time
return parsed
warnings.filterwarnings('ignore' )<import_modules>
if not IS_KAGGLE:
train_raw_ds = tf.data.TFRecordDataset(train_tfrec_paths, num_parallel_reads=1)
 train_raw_ds = train_raw_ds.map(parse_train_example, num_parallel_calls=1, deterministic=True)
 for x in train_raw_ds.take(1):
print(x )
import os
import pandas as pd
import numpy as np
from datetime import datetime
import scipy
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import RobustScaler
from sklearn.pipeline import Pipeline
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.model_selection import GridSearchCV
from xgboost.sklearn import XGBRegressor
from sklearn.ensemble import GradientBoostingRegressor
from mlxtend.regressor import StackingCVRegressor
from sklearn import metrics
import statsmodels.api as sm<load_from_csv>
if not IS_KAGGLE:
valid_raw_ds = tf.data.TFRecordDataset(valid_tfrec_paths, num_parallel_reads=1)
 valid_raw_ds = valid_raw_ds.map(parse_train_example_with_valid_info, num_parallel_calls=1, deterministic=True)
 for x in valid_raw_ds.take(1):
print(x )
train_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 train_df.head()<load_from_csv>
def convert_split_index_dict(split_index_dict):
user_ids = []
split_indices = []
for k, v in split_index_dict.items() :
user_ids.append(k)
 split_indices.append(v)
 user_id_tensor = tf.constant(user_ids, dtype=tf.int64)
 split_index_tensor = tf.constant(split_indices, dtype=tf.int32)
 initializer = tf.lookup.KeyValueTensorInitializer(user_id_tensor, split_index_tensor)
 split_index_table = tf.lookup.StaticHashTable(
 initializer, default_value=tf.int32.limits[1], name=None 
)
 return split_index_table
def split_train_example(raw_example, split_index_table):

 user_id = raw_example['user_id']
seq_len = raw_example['seq_len']
split_index = split_index_table.lookup(user_id)
 tf.debugging.Assert(tf.reduce_all(split_index >= 0), [split_index])
 example = {}
if split_index == tf.int32.limits[1]:
split_index = seq_len
example['user_id'] = raw_example['user_id']
example['seq_len'] = split_index
example['prev_seq_len'] = raw_example['seq_len']
example['start'] = tf.constant(0, dtype=tf.int32)
 example['end'] = split_index - 1
for k in raw_example:
if k not in ['user_id', 'seq_len', 'prev_seq_len', 'start', 'end']:
example[k] = raw_example[k][0:split_index]
return example
def split_train_ds(train_raw_ds, split_indices, num_parallel_calls=None, deterministic=None):
reduced_raw_ds = train_raw_ds.map(lambda example: split_train_example(example, split_indices), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 return reduced_raw_ds
test_df = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv')
 test_df.head()<drop_column>
def extract_subseqs(seqs, ending_indices, window_size, seq_len):

 orig_window_size = window_size
window_size += 1
tf.debugging.Assert(tf.rank(seqs)>= 2, [tf.rank(seqs)])
 tf.debugging.Assert(tf.reduce_all(window_size > 0), [window_size])
 tf.debugging.Assert(tf.reduce_all(ending_indices >= -1), [ending_indices])
 batch_size = tf.reduce_sum(tf.ones_like(seq_len, dtype=tf.int32))
tf.debugging.Assert(tf.reduce_all(( ending_indices < seq_len)) , [ending_indices, seq_len])
 _ending_indices = ending_indices[:, tf.newaxis]
_ending_indices += 1
ranges = tf.range(window_size)[tf.newaxis, :]
indices = _ending_indices - ranges
indices = tf.reverse(indices, axis=[1])
 indices_to_seqs = tf.stack([tf.broadcast_to(tf.range(batch_size)[:, tf.newaxis], shape=[batch_size, window_size]), indices], axis=2)
 indices_to_ext_seqs = tf.math.maximum(indices_to_seqs, 0)
 invalid_values = tf.cast(tf.ones(shape=seqs.shape[2:])* PAD_TOKEN, seqs.dtype)
 invalid_values = tf.repeat([[invalid_values]], repeats=batch_size, axis=0)
 extended_seqs = tf.concat([invalid_values, seqs], axis=1)
 sampled_subseqs = tf.gather_nd(extended_seqs, indices=indices_to_ext_seqs)
 sampled_subseqs = sampled_subseqs[:, 1:]
prev_last_element = sampled_subseqs[:, 0]
return sampled_subseqs, prev_last_element
def random_ending_indices(seq_len, seed=None):
batch_size = tf.reduce_sum(tf.ones_like(seq_len))
max_seq_len = tf.cast(tf.math.reduce_max(seq_len), dtype=DTYPE)
 ending_indices = -1.0 + tf.random.uniform(minval=0.0, maxval=1.0, dtype=DTYPE, shape=[batch_size], seed=seed)* tf.cast(seq_len + 1, dtype=DTYPE)
 ending_indices = tf.cast(tf.math.floor(ending_indices), dtype=tf.int32)
 tf.debugging.Assert(tf.math.reduce_min(seq_len - ending_indices)>= 1, [seq_len, ending_indices])
 tf.debugging.Assert(tf.math.reduce_min(ending_indices)>= -1, [seq_len, ending_indices])
 return ending_indices
def random_subseqs(seqs, window_size, seq_len, seed=None):

 ending_indices = random_ending_indices(seq_len=seq_len, seed=seed)
 sampled_subseqs, _ = extract_subseqs(seqs, ending_indices, window_size, seq_len=seq_len)
 return sampled_subseqs, ending_indices
def extract_subseqs_from_raw_batch(raw_batch, ending_indices, window_size):

 tf.debugging.Assert(tf.reduce_all(raw_batch['start'] == tf.zeros_like(raw_batch['start'], dtype=tf.int32)) , [raw_batch['start']])
 batch = {}
batch['user_id'] = raw_batch['user_id']
start = tf.math.maximum(ending_indices - window_size + 1, 0)
 end = ending_indices
seq_len =(end - start)+ 1
batch['seq_len'] = seq_len
batch['prev_seq_len'] = raw_batch['seq_len']
batch['start'] = start
batch['end'] = end
for k in raw_batch:
if k not in ['user_id', 'seq_len', 'prev_seq_len', 'start', 'end'] + [
'n_valid_blocks', 'valid_blocks_start_pos', 'valid_blocks_end_pos', 'valid_block_pos', 'valid_block_idx', 'valid_start', 'valid_end'
]:
subseqs, prev_last_element = extract_subseqs(seqs=raw_batch[k], ending_indices=ending_indices, window_size=window_size, seq_len=raw_batch['prev_seq_len'])
 batch[k] = subseqs
elif k in ['n_valid_blocks', 'valid_block_idx', 'valid_start', 'valid_end']:
batch[k] = raw_batch[k]
return batch
def random_subseqs_from_raw_batch(raw_batch, window_size, only_last, seed=None):
if only_last:
ending_indices = raw_batch['seq_len'] - 1
else:
ending_indices = random_ending_indices(raw_batch['seq_len'], seed=seed)
 ending_indices = tf.math.maximum(ending_indices, window_size - 1)
 _mask = tf.cast(ending_indices >= raw_batch['seq_len'], tf.int32)
 ending_indices = ending_indices *(1 - _mask)+(raw_batch['seq_len'] - 1)* _mask
batch = extract_subseqs_from_raw_batch(raw_batch, ending_indices, window_size)
 return batch, ending_indices
def random_subseqs_from_batched_raw_ds(batched_raw_ds, window_size, only_last=False, seed=None, num_parallel_calls=None, deterministic=None):
batched_ds = batched_raw_ds.map(lambda raw_batch: random_subseqs_from_raw_batch(raw_batch, window_size, only_last, seed), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 return batched_ds
train_Ids = train_df.pop('Id')
 test_Ids = test_df.pop('Id')
 test_df['SalePrice'] = np.NaN<drop_column>
def get_causal_attention_mask(nd, ns, dtype, only_before):

 i = tf.range(nd)[:, tf.newaxis]
j = tf.range(ns)
 m = i >=(j - ns + nd)+ tf.cast(only_before, dtype=tf.int32)
 return tf.cast(m, dtype)
 def get_attention_mask_from_timestamp_batch(timestamp_tensors, dtype, only_before):

 t = timestamp_tensors
batch_size = tf.math.reduce_sum(tf.ones_like(t[:, :1], dtype=tf.int32))
seq_len = tf.math.reduce_sum(tf.ones_like(t[:1, :], dtype=tf.int32))
x = tf.broadcast_to(t[:, :, tf.newaxis], shape=[batch_size, seq_len, seq_len])
 y = tf.broadcast_to(t[:, tf.newaxis, :], shape=[batch_size, seq_len, seq_len])+ tf.cast(only_before, dtype=tf.int64)
 m = x >= y
return tf.cast(m, dtype )
housing_all_df.drop(columns=['Utilities'], inplace = True )<count_missing_values>
if not IS_KAGGLE:
print(get_causal_attention_mask(3, 3, tf.int32, only_before=tf.constant(False)))
 print(get_causal_attention_mask(3, 3, tf.int32, only_before=tf.constant(True)))
 timestamp_tensors = tf.constant([[0, 0, 1, 2, 2, 2, 3, 3], [0, 1, 1, 2, 2, 3, 3, 3]], dtype=tf.int64)
 print(get_attention_mask_from_timestamp_batch(timestamp_tensors, tf.int32, only_before=tf.constant(False)))
 print(get_attention_mask_from_timestamp_batch(timestamp_tensors, tf.int32, only_before=tf.constant(True)) )
null_counts = housing_all_df.isnull().sum()
null_counts[null_counts.values > 0].index<count_values>
@tf.function
def get_attention_masks(batch, training, encoder_decoder, generative, allow_bundle_atten):
pad_mask = tf.cast(( batch['content_type_id'] == PAD_TOKEN), dtype=tf.int32)
 question_mask = tf.cast(( batch['content_type_id'] == 0), dtype=tf.int32)
 pred_time_mask = batch['pred_time_mask'] * tf.cast(batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 content_ids = batch['content_id']
_pad_mask = tf.cast(( batch['timestamp'] == PAD_TOKEN), dtype=tf.int32)
 seq_len = tf.math.reduce_sum(tf.ones_like(content_ids[0, :], dtype=tf.int32))
non_pad_mask = 1 - pad_mask
causal_attention_mask = get_causal_attention_mask(nd=seq_len, ns=seq_len, dtype=tf.int32, only_before=tf.constant(False))
timestamp_attention_mask = get_attention_mask_from_timestamp_batch(batch['timestamp'], dtype=tf.int32, only_before=tf.constant(False))
timestamp_attention_mask_only_before = get_attention_mask_from_timestamp_batch(batch['timestamp'], dtype=tf.int32, only_before=tf.constant(True))
if tf.cast(allow_bundle_atten, dtype=tf.bool):
c_mask = non_pad_mask[:, tf.newaxis, :] * timestamp_attention_mask
else:
c_mask = non_pad_mask[:, tf.newaxis, :] * causal_attention_mask[tf.newaxis, :, :]
r_mask = c_mask
if tf.cast(encoder_decoder, dtype=tf.bool):
d_mask = non_pad_mask[:, tf.newaxis, :] * causal_attention_mask[tf.newaxis, :, :]
if(1 - training)*(1 - generative)== 1:
pred_time_question_mask = pred_time_mask * question_mask
d_mask = d_mask * tf.math.maximum(( 1 - pred_time_question_mask)[:, tf.newaxis, :], tf.eye(seq_len, dtype=tf.int32)[tf.newaxis, :, :])
 r_mask = d_mask
c_mask = c_mask * tf.math.maximum(( 1 - pred_time_question_mask)[:, tf.newaxis, :], tf.eye(seq_len, dtype=tf.int32)[tf.newaxis, :, :])
 else:
r_mask = d_mask
r_c_mask = c_mask
c_r_mask = non_pad_mask[:, tf.newaxis, :] * timestamp_attention_mask_only_before
return c_mask, r_mask, r_c_mask, c_r_mask
def add_input_ids_and_targets(batch, training, generative, use_abs_pos):

 content_ids = batch['content_id']
content_type_ids = batch['content_type_id']
answered_correctly = batch['answered_correctly']
user_answer = batch['user_answer']
question_mask = tf.cast(( content_type_ids == 0), dtype=tf.int32)
 lecture_mask = tf.cast(( content_type_ids == 1), dtype=tf.int32)
 pad_mask = tf.cast(( content_type_ids == PAD_TOKEN), dtype=tf.int32)
 _pad_mask = tf.cast(( batch['timestamp'] == PAD_TOKEN), dtype=tf.int32)
 pred_time_mask = batch['pred_time_mask'] * tf.cast(batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 pred_time_question_mask = pred_time_mask * question_mask
n_questions_in_pred_time = tf.math.reduce_sum(pred_time_question_mask, axis=1)
 response_false_mask = tf.cast(( answered_correctly == RESPONSE_FALSE_TOKEN), dtype=tf.int32)
 response_true_mask = tf.cast(( answered_correctly == RESPONSE_TRUE_TOKEN), dtype=tf.int32)
 _batch = {}
for k in batch:
_batch[k] = batch[k]
_batch['abs_pos_ids'] = tf.cast(tf.math.minimum(tf.math.maximum(0, _batch['abs_pos'] + 1), MAX_HISTORY_LEN), dtype=DTYPE)/ tf.cast(MAX_HISTORY_LEN, dtype=DTYPE)* WINDOW_SIZE
_batch['shifted_abs_pos_ids'] = tf.cast(tf.math.minimum(tf.math.maximum(0, _batch['shifted_abs_pos'] + 1), MAX_HISTORY_LEN - 1), dtype=DTYPE)/ tf.cast(MAX_HISTORY_LEN, dtype=DTYPE)* WINDOW_SIZE
_batch['pos_ids'] = tf.math.cumsum(1 - pad_mask, axis=1)
 _batch['shifted_pos_ids'] = tf.math.maximum(0, _batch['pos_ids'] - 1)
 content_input_ids = question_mask * question_id_to_input_id_table.lookup(content_ids)+ \
lecture_mask * lecture_id_to_input_id_table.lookup(content_ids)+ \
pad_mask * PAD_ID
_batch['c_input_ids'] = content_input_ids
response_input_ids_masked = pad_mask * PAD_ID + pred_time_question_mask * MASK_ID + lecture_mask * RESPONSE_LECTURE_ID +(1 - pred_time_question_mask)* response_false_mask * RESPONSE_FALSE_ID +(1 - pred_time_question_mask)* response_true_mask * RESPONSE_TRUE_ID
_batch['r_input_ids'] = response_input_ids_masked
shifted_answered_correctly = batch['shifted_answered_correctly']
shifted_pad_mask = tf.cast(( shifted_answered_correctly == PAD_TOKEN), dtype=tf.int32)
 shifted_start_mask = tf.cast(( shifted_answered_correctly == START_TOKEN), dtype=tf.int32)
 shifted_masking_mask = tf.cast(( shifted_answered_correctly == MASK_TOKEN), dtype=tf.int32)
 shifted_response_lecture_mask = tf.cast(( shifted_answered_correctly == RESPONSE_LECTURE_TOKEN), dtype=tf.int32)
 shifted_response_false_mask = tf.cast(( shifted_answered_correctly == RESPONSE_FALSE_TOKEN), dtype=tf.int32)
 shifted_response_true_mask = tf.cast(( shifted_answered_correctly == RESPONSE_TRUE_TOKEN), dtype=tf.int32)
 decoder_input_ids = shifted_pad_mask * PAD_ID + shifted_start_mask * START_ID + shifted_masking_mask * MASK_ID + shifted_response_lecture_mask * RESPONSE_LECTURE_ID + shifted_response_false_mask * RESPONSE_FALSE_ID + shifted_response_true_mask * RESPONSE_TRUE_ID
shifted_user_answer = batch['shifted_user_answer']
shifted_answer_0_mask = tf.cast(( shifted_user_answer == ANSWER_0_TOKEN), dtype=tf.int32)
 shifted_answer_1_mask = tf.cast(( shifted_user_answer == ANSWER_1_TOKEN), dtype=tf.int32)
 shifted_answer_2_mask = tf.cast(( shifted_user_answer == ANSWER_2_TOKEN), dtype=tf.int32)
 shifted_answer_3_mask = tf.cast(( shifted_user_answer == ANSWER_3_TOKEN), dtype=tf.int32)
 decoder_answer_input_ids = shifted_pad_mask * PAD_ID + shifted_start_mask * START_ID + shifted_masking_mask * MASK_ID + \
shifted_response_lecture_mask * ANSWER_LECTURE_ID + \
shifted_answer_0_mask * ANSWER_0_ID + shifted_answer_1_mask * ANSWER_1_ID + \
shifted_answer_2_mask * ANSWER_2_ID + shifted_answer_3_mask * ANSWER_3_ID
if not generative:
pred_time_question_start_mask = tf.cast(tf.math.cumsum(pred_time_question_mask, axis=1)== 1, dtype=tf.int32)
 pred_time_question_start_value_response = tf.math.reduce_sum(pred_time_question_start_mask * decoder_input_ids, axis=1)
 pred_time_question_start_value_answer = tf.math.reduce_sum(pred_time_question_start_mask * decoder_answer_input_ids, axis=1)
 prev_lecture_mask = tf.cast(pred_time_question_start_value_response == MASK_ID, tf.int32)
 pred_time_question_start_value_response = RESPONSE_LECTURE_ID * prev_lecture_mask + pred_time_question_start_value_response *(1 - prev_lecture_mask)
 pred_time_question_start_value_answer = ANSWER_LECTURE_ID * prev_lecture_mask + pred_time_question_start_value_answer *(1 - prev_lecture_mask)
 decoder_input_ids = decoder_input_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pred_time_question_start_value_response[:, tf.newaxis]
decoder_answer_input_ids = decoder_answer_input_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pred_time_question_start_value_answer[:, tf.newaxis]
decoder_input_ids = decoder_input_ids * tf.cast(decoder_input_ids != MASK_ID, dtype=tf.int32)+ RESPONSE_LECTURE_ID * tf.cast(decoder_input_ids == MASK_ID, dtype=tf.int32)
 decoder_answer_input_ids = decoder_answer_input_ids * tf.cast(decoder_answer_input_ids != MASK_ID, dtype=tf.int32)+ ANSWER_LECTURE_ID * tf.cast(decoder_answer_input_ids == MASK_ID, dtype=tf.int32)
 _batch['d_input_ids'] = decoder_input_ids
_batch['d_ans_input_ids'] = decoder_answer_input_ids
if not generative:
pos_ids = _batch['pos_ids']
shifted_pos_ids = _batch['shifted_pos_ids']
abs_pos_ids = _batch['abs_pos_ids']
shifted_abs_pos_ids = _batch['shifted_abs_pos_ids']
pos_ids_question_start_value = tf.math.reduce_sum(pred_time_question_start_mask * pos_ids, axis=1)
 shifted_pos_ids_question_start_value = tf.math.reduce_sum(pred_time_question_start_mask * shifted_pos_ids, axis=1)
 pos_ids = pos_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pos_ids_question_start_value[:, tf.newaxis]
shifted_pos_ids = shifted_pos_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * shifted_pos_ids_question_start_value[:, tf.newaxis]
abs_pos_ids_question_start_value = tf.math.reduce_sum(tf.cast(pred_time_question_start_mask, dtype=DTYPE)* abs_pos_ids, axis=1)
 shifted_abs_pos_ids_question_start_value = tf.math.reduce_sum(tf.cast(pred_time_question_start_mask, dtype=DTYPE)* shifted_abs_pos_ids, axis=1)
 abs_pos_ids = abs_pos_ids * tf.cast(1 - pred_time_question_mask, dtype=DTYPE)+ tf.cast(pred_time_question_mask, dtype=DTYPE)* abs_pos_ids_question_start_value[:, tf.newaxis]
shifted_abs_pos_ids = shifted_abs_pos_ids * tf.cast(1 - pred_time_question_mask, dtype=DTYPE)+ tf.cast(pred_time_question_mask, dtype=DTYPE)* shifted_abs_pos_ids_question_start_value[:, tf.newaxis]
_batch['pos_ids'] = pos_ids
_batch['shifted_pos_ids'] = shifted_pos_ids
_batch['abs_pos_ids'] = abs_pos_ids
_batch['shifted_abs_pos_ids'] = shifted_abs_pos_ids
_batch['tag_ids'] = tf.gather(params=c_inputs_ids_to_tags, indices=_batch['c_input_ids'])+ 1
_batch['part_ids'] = tf.gather(params=c_inputs_ids_to_part, indices=_batch['c_input_ids'])+ 1
_batch['prior_explanation_ids'] = tf.math.maximum(0, _batch['prior_question_had_explanation'] + 1)
 _batch['prior_question_elapsed_time_input'] = tf.cast(_batch['prior_question_elapsed_time'], dtype=DTYPE)/ 1000.0 / 300.0
lag_time = tf.cast(_batch['lag_time'], dtype=DTYPE)
 lag_time = lag_time / 1000.0 / 3600.0
lag_time = tf.math.minimum(lag_time, 72.0)
 _batch['lag_time'] = lag_time
task_container_pos_ids = tf.cast(_batch['task_container_id'], dtype=DTYPE)/ MAX_TASK_CONTAINER_ID * 10.0
_batch['task_container_pos_ids'] = task_container_pos_ids
answer_mask = tf.cast(batch['answered_correctly'] > -1, dtype=tf.int32)
 _batch['target'] = batch['answered_correctly'] * answer_mask +(NON_TARGET_ID)*(1 - answer_mask)
 _batch['answer_target'] = batch['user_answer'] * answer_mask +(NON_TARGET_ID)*(1 - answer_mask)
 _batch['correct_answer_id'] = tf.gather(params=c_inputs_ids_to_correct_answer_id, indices=_batch['c_input_ids'])
 targets = _batch['target']
if training == 0:
targets = targets * pred_time_mask + NON_TARGET_ID *(1 - pred_time_mask)
 pred_mask = targets != NON_TARGET_ID
nb_pred_places = tf.math.reduce_sum(tf.cast(pred_mask, dtype=tf.int32))
_batch['nb_pred_places'] = nb_pred_places * tf.ones_like(_batch['user_id'], dtype=tf.int32)
 scaling_factor = tf.constant(N_AGGREGATED_QUESTION_SCALING_FACTOR, dtype=tf.float32)
 _batch['n_questions_answered_scaled'] = tf.cast(_batch['n_questions_answered'], dtype=tf.float32)/ scaling_factor
_batch['n_lectures_watched_scaled'] = tf.cast(_batch['n_lectures_watched'], dtype=tf.float32)/ scaling_factor
n_questions_answered = tf.cast(tf.math.maximum(_batch['n_questions_answered'], 1), dtype=tf.float32)
 _batch['answered_correctly_ratio'] = tf.cast(_batch['n_questions_answered_correctly'], dtype=tf.float32)/ n_questions_answered
part_count_scaled = []
part_correctness_ratio = []
for part_idx in range(2, 9):
key = f'part_{part_idx}_count'
_part_count = tf.cast(_batch[key], dtype=tf.float32)
 _part_count_scaled = _part_count / scaling_factor
part_count_scaled.append(_part_count_scaled)
 key_2 = f'part_{part_idx}_correctness_count'
_part_correctness_count = tf.cast(_batch[key_2], dtype=tf.float32)
 _part_correctness_ratio = _part_correctness_count / tf.math.maximum(_part_count, 1.0)
 part_correctness_ratio.append(_part_correctness_ratio)
 del _batch[key]
del _batch[key_2]
part_count_scaled = tf.stack(part_count_scaled, axis=-1)
 part_correctness_ratio = tf.stack(part_correctness_ratio, axis=-1)
 _batch['part_count_scaled'] = part_count_scaled
_batch['part_correctness_ratio'] = part_correctness_ratio
_part_count_scaled = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
part_count_scaled
],
axis=-1 
)
 _part_correctness_ratio = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
part_correctness_ratio
],
axis=-1 
)
 current_part_count_scaled = tf.gather(params=_part_count_scaled, indices=_batch['part_ids'], batch_dims=2)
 current_part_correctness_ratio = tf.gather(params=_part_correctness_ratio, indices=_batch['part_ids'], batch_dims=2)
 _batch['current_part_count_scaled'] = current_part_count_scaled
_batch['current_part_correctness_ratio'] = current_part_correctness_ratio
correct_answer_count_scaled = []
correct_answer_correctness_ratio = []
for correct_answer_idx in range(ANSWER_0_ID, ANSWER_3_ID + 1):
key = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_count'
_correct_answer_count = tf.cast(_batch[key], dtype=tf.float32)
 _correct_answer_count_scaled = _correct_answer_count / scaling_factor
correct_answer_count_scaled.append(_correct_answer_count_scaled)
 key_2 = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_answered_correctly_count'
_correct_answer_answered_correctly_count = tf.cast(_batch[key_2], dtype=tf.float32)
 _correct_answer_answered_correctly_ratio = _correct_answer_answered_correctly_count / tf.math.maximum(_correct_answer_count, 1.0)
 correct_answer_correctness_ratio.append(_correct_answer_answered_correctly_ratio)
 del _batch[key]
del _batch[key_2]
correct_answer_count_scaled = tf.stack(correct_answer_count_scaled, axis=-1)
 correct_answer_correctness_ratio = tf.stack(correct_answer_correctness_ratio, axis=-1)
 _batch['correct_answer_count_scaled'] = correct_answer_count_scaled
_batch['correct_answer_correctness_ratio'] = correct_answer_correctness_ratio
_correct_answer_count_scaled = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
correct_answer_count_scaled
],
axis=-1 
)
 _correct_answer_correctness_ratio = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
correct_answer_correctness_ratio
],
axis=-1 
)
 _indices = tf.math.maximum(_batch['correct_answer_id'] - ANSWER_LECTURE_ID + 1, 0)
 current_correct_answer_count_scaled = tf.gather(params=_correct_answer_count_scaled, indices=_indices, batch_dims=2)
 current_correct_answer_correctness_ratio = tf.gather(params=_correct_answer_correctness_ratio, indices=_indices, batch_dims=2)
 _batch['current_correct_answer_count_scaled'] = current_correct_answer_count_scaled
_batch['current_correct_answer_correctness_ratio'] = current_correct_answer_correctness_ratio
_batch['current_question_count_scaled'] = tf.cast(_batch['n_prev_seen'], dtype=tf.float32)/ scaling_factor
current_question_count = tf.cast(tf.math.maximum(_batch['n_prev_seen'], 1), dtype=tf.float32)
 _batch['current_question_correctness_ratio'] = tf.cast(_batch['n_prev_correctness'], dtype=tf.float32)/ current_question_count
return _batch
def prepare_training_dataset(batched_ds, generative=False, use_abs_pos=False, num_parallel_calls=None, deterministic=None):
training = tf.constant(1, dtype=tf.int32)
 return batched_ds.map(lambda batch, _: add_input_ids_and_targets(batch, training, generative, use_abs_pos), num_parallel_calls=num_parallel_calls, deterministic=deterministic )
print("Null count: ", housing_all_df['MSZoning'].isnull().sum())
 print(housing_all_df['MSZoning'].value_counts())
 housing_all_df['MSZoning'].fillna(housing_all_df['MSZoning'].mode().values[0], inplace = True )<count_values>
def get_input_signatures_2() :
batch_size = None
seq_len = 128
input_signatures = {
'user_id': tf.TensorSpec(shape=[batch_size], dtype=tf.int64, name='user_id'),
'seq_len': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='seq_len'),
'prev_seq_len': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='prev_seq_len'),
'start': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='start'),
'end': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='end'),
'row_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='row_id'),
'timestamp': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='timestamp'),
'content_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='content_id'),
'content_type_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='content_type_id'),
'task_container_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='task_container_id'),
'user_answer': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='user_answer'),
'shifted_user_answer': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_user_answer'),
'answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='answered_correctly'),
'shifted_answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_answered_correctly'),
'prior_question_elapsed_time': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='prior_question_elapsed_time'),
'prior_question_had_explanation': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='prior_question_had_explanation'),
'pred_time_mask': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='pred_time_mask'),
'abs_pos': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='abs_pos'),
'shifted_abs_pos': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_abs_pos'),
'lag_time': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='lag_time'),
'n_questions_answered': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_questions_answered'),
'n_lectures_watched': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_lectures_watched'),
'n_questions_answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_questions_answered_correctly'),
'n_prev_seen': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_prev_seen'),
'n_prev_correctness': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_prev_correctness'),
'part_2_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_2_count'),
'part_3_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_3_count'),
'part_4_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_4_count'),
'part_5_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_5_count'),
'part_6_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_6_count'),
'part_7_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_7_count'),
'part_8_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_8_count'),
'part_2_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_2_correctness_count'),
'part_3_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_3_correctness_count'),
'part_4_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_4_correctness_count'),
'part_5_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_5_correctness_count'),
'part_6_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_6_correctness_count'),
'part_7_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_7_correctness_count'),
'part_8_correctness_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_8_correctness_count'),
'correct_answer_0_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_0_count'),
'correct_answer_1_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_1_count'),
'correct_answer_2_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_2_count'),
'correct_answer_3_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_3_count'),
'correct_answer_0_answered_correctly_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_0_answered_correctly_count'),
'correct_answer_1_answered_correctly_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_1_answered_correctly_count'),
'correct_answer_2_answered_correctly_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_2_answered_correctly_count'),
'correct_answer_3_answered_correctly_count': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_3_answered_correctly_count'),
}
return [input_signatures]
print("Null count: ", housing_all_df['PoolQC'].isnull().sum())
 housing_all_df['PoolQC'].value_counts()<filter>
@tf.function(
 input_signature=get_input_signatures_2() 
)
 def add_input_ids_and_targets_2(batch):

 training = tf.constant(0, dtype=tf.int32)
 generative = tf.constant(False, dtype=tf.bool)
 content_ids = batch['content_id']
content_type_ids = batch['content_type_id']
answered_correctly = batch['answered_correctly']
user_answer = batch['user_answer']
question_mask = tf.cast(( content_type_ids == 0), dtype=tf.int32)
 lecture_mask = tf.cast(( content_type_ids == 1), dtype=tf.int32)
 pad_mask = tf.cast(( content_type_ids == PAD_TOKEN), dtype=tf.int32)
 _pad_mask = tf.cast(( batch['timestamp'] == PAD_TOKEN), dtype=tf.int32)
 pred_time_mask = batch['pred_time_mask'] * tf.cast(batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 pred_time_question_mask = pred_time_mask * question_mask
n_questions_in_pred_time = tf.math.reduce_sum(pred_time_question_mask, axis=1)
 response_false_mask = tf.cast(( answered_correctly == RESPONSE_FALSE_TOKEN), dtype=tf.int32)
 response_true_mask = tf.cast(( answered_correctly == RESPONSE_TRUE_TOKEN), dtype=tf.int32)
 _batch = {}
for k in batch:
_batch[k] = batch[k]
_batch['abs_pos_ids'] = tf.cast(tf.math.minimum(tf.math.maximum(0, _batch['abs_pos'] + 1), MAX_HISTORY_LEN), dtype=DTYPE)/ tf.cast(MAX_HISTORY_LEN, dtype=DTYPE)* WINDOW_SIZE
_batch['shifted_abs_pos_ids'] = tf.cast(tf.math.minimum(tf.math.maximum(0, _batch['shifted_abs_pos'] + 1), MAX_HISTORY_LEN - 1), dtype=DTYPE)/ tf.cast(MAX_HISTORY_LEN, dtype=DTYPE)* WINDOW_SIZE
_batch['pos_ids'] = tf.math.cumsum(1 - pad_mask, axis=1)
 _batch['shifted_pos_ids'] = tf.math.maximum(0, _batch['pos_ids'] - 1)
 content_input_ids = question_mask * question_id_to_input_id_table.lookup(content_ids)+ \
lecture_mask * lecture_id_to_input_id_table.lookup(content_ids)+ \
pad_mask * PAD_ID
_batch['c_input_ids'] = content_input_ids
response_input_ids_masked = pad_mask * PAD_ID + pred_time_question_mask * MASK_ID + lecture_mask * RESPONSE_LECTURE_ID +(1 - pred_time_question_mask)* response_false_mask * RESPONSE_FALSE_ID +(1 - pred_time_question_mask)* response_true_mask * RESPONSE_TRUE_ID
_batch['r_input_ids'] = response_input_ids_masked
shifted_answered_correctly = batch['shifted_answered_correctly']
shifted_pad_mask = tf.cast(( shifted_answered_correctly == PAD_TOKEN), dtype=tf.int32)
 shifted_start_mask = tf.cast(( shifted_answered_correctly == START_TOKEN), dtype=tf.int32)
 shifted_masking_mask = tf.cast(( shifted_answered_correctly == MASK_TOKEN), dtype=tf.int32)
 shifted_response_lecture_mask = tf.cast(( shifted_answered_correctly == RESPONSE_LECTURE_TOKEN), dtype=tf.int32)
 shifted_response_false_mask = tf.cast(( shifted_answered_correctly == RESPONSE_FALSE_TOKEN), dtype=tf.int32)
 shifted_response_true_mask = tf.cast(( shifted_answered_correctly == RESPONSE_TRUE_TOKEN), dtype=tf.int32)
 decoder_input_ids = shifted_pad_mask * PAD_ID + shifted_start_mask * START_ID + shifted_masking_mask * MASK_ID + shifted_response_lecture_mask * RESPONSE_LECTURE_ID + shifted_response_false_mask * RESPONSE_FALSE_ID + shifted_response_true_mask * RESPONSE_TRUE_ID
shifted_user_answer = batch['shifted_user_answer']
shifted_answer_0_mask = tf.cast(( shifted_user_answer == ANSWER_0_TOKEN), dtype=tf.int32)
 shifted_answer_1_mask = tf.cast(( shifted_user_answer == ANSWER_1_TOKEN), dtype=tf.int32)
 shifted_answer_2_mask = tf.cast(( shifted_user_answer == ANSWER_2_TOKEN), dtype=tf.int32)
 shifted_answer_3_mask = tf.cast(( shifted_user_answer == ANSWER_3_TOKEN), dtype=tf.int32)
 decoder_answer_input_ids = shifted_pad_mask * PAD_ID + shifted_start_mask * START_ID + shifted_masking_mask * MASK_ID + \
shifted_response_lecture_mask * ANSWER_LECTURE_ID + \
shifted_answer_0_mask * ANSWER_0_ID + shifted_answer_1_mask * ANSWER_1_ID + \
shifted_answer_2_mask * ANSWER_2_ID + shifted_answer_3_mask * ANSWER_3_ID
pred_time_question_start_mask = tf.cast(tf.math.cumsum(pred_time_question_mask, axis=1)== 1, dtype=tf.int32)
 pred_time_question_start_value_response = tf.math.reduce_sum(pred_time_question_start_mask * decoder_input_ids, axis=1)
 pred_time_question_start_value_answer = tf.math.reduce_sum(pred_time_question_start_mask * decoder_answer_input_ids, axis=1)
 prev_lecture_mask = tf.cast(pred_time_question_start_value_response == MASK_ID, tf.int32)
 pred_time_question_start_value_response = RESPONSE_LECTURE_ID * prev_lecture_mask + pred_time_question_start_value_response *(1 - prev_lecture_mask)
 pred_time_question_start_value_answer = ANSWER_LECTURE_ID * prev_lecture_mask + pred_time_question_start_value_answer *(1 - prev_lecture_mask)
 decoder_input_ids = decoder_input_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pred_time_question_start_value_response[:, tf.newaxis]
decoder_answer_input_ids = decoder_answer_input_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pred_time_question_start_value_answer[:, tf.newaxis]
decoder_input_ids = decoder_input_ids * tf.cast(decoder_input_ids != MASK_ID, dtype=tf.int32)+ RESPONSE_LECTURE_ID * tf.cast(decoder_input_ids == MASK_ID, dtype=tf.int32)
 decoder_answer_input_ids = decoder_answer_input_ids * tf.cast(decoder_answer_input_ids != MASK_ID, dtype=tf.int32)+ ANSWER_LECTURE_ID * tf.cast(decoder_answer_input_ids == MASK_ID, dtype=tf.int32)
 _batch['d_input_ids'] = decoder_input_ids
_batch['d_ans_input_ids'] = decoder_answer_input_ids
pos_ids = _batch['pos_ids']
shifted_pos_ids = _batch['shifted_pos_ids']
abs_pos_ids = _batch['abs_pos_ids']
shifted_abs_pos_ids = _batch['shifted_abs_pos_ids']
pos_ids_question_start_value = tf.math.reduce_sum(pred_time_question_start_mask * pos_ids, axis=1)
 shifted_pos_ids_question_start_value = tf.math.reduce_sum(pred_time_question_start_mask * shifted_pos_ids, axis=1)
 pos_ids = pos_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * pos_ids_question_start_value[:, tf.newaxis]
shifted_pos_ids = shifted_pos_ids *(1 - pred_time_question_mask)+ pred_time_question_mask * shifted_pos_ids_question_start_value[:, tf.newaxis]
abs_pos_ids_question_start_value = tf.math.reduce_sum(tf.cast(pred_time_question_start_mask, dtype=DTYPE)* abs_pos_ids, axis=1)
 shifted_abs_pos_ids_question_start_value = tf.math.reduce_sum(tf.cast(pred_time_question_start_mask, dtype=DTYPE)* shifted_abs_pos_ids, axis=1)
 abs_pos_ids = abs_pos_ids * tf.cast(1 - pred_time_question_mask, dtype=DTYPE)+ tf.cast(pred_time_question_mask, dtype=DTYPE)* abs_pos_ids_question_start_value[:, tf.newaxis]
shifted_abs_pos_ids = shifted_abs_pos_ids * tf.cast(1 - pred_time_question_mask, dtype=DTYPE)+ tf.cast(pred_time_question_mask, dtype=DTYPE)* shifted_abs_pos_ids_question_start_value[:, tf.newaxis]
_batch['pos_ids'] = pos_ids
_batch['shifted_pos_ids'] = shifted_pos_ids
_batch['abs_pos_ids'] = abs_pos_ids
_batch['shifted_abs_pos_ids'] = shifted_abs_pos_ids
_batch['tag_ids'] = tf.gather(params=c_inputs_ids_to_tags, indices=_batch['c_input_ids'])+ 1
_batch['part_ids'] = tf.gather(params=c_inputs_ids_to_part, indices=_batch['c_input_ids'])+ 1
_batch['prior_explanation_ids'] = tf.math.maximum(0, _batch['prior_question_had_explanation'] + 1)
 _batch['prior_question_elapsed_time_input'] = tf.cast(_batch['prior_question_elapsed_time'], dtype=DTYPE)/ 1000.0 / 300.0
lag_time = tf.cast(_batch['lag_time'], dtype=DTYPE)
 lag_time = lag_time / 1000.0 / 3600.0
lag_time = tf.math.minimum(lag_time, 72.0)
 _batch['lag_time'] = lag_time
task_container_pos_ids = tf.cast(_batch['task_container_id'], dtype=DTYPE)/ MAX_TASK_CONTAINER_ID * 10.0
_batch['task_container_pos_ids'] = task_container_pos_ids
answer_mask = tf.cast(batch['answered_correctly'] > -1, dtype=tf.int32)
 _batch['target'] = batch['answered_correctly'] * answer_mask +(NON_TARGET_ID)*(1 - answer_mask)
 _batch['answer_target'] = batch['user_answer'] * answer_mask +(NON_TARGET_ID)*(1 - answer_mask)
 _batch['correct_answer_id'] = tf.gather(params=c_inputs_ids_to_correct_answer_id, indices=_batch['c_input_ids'])
 targets = _batch['target']
if training == 0:
targets = targets * pred_time_mask + NON_TARGET_ID *(1 - pred_time_mask)
 pred_mask = targets != NON_TARGET_ID
nb_pred_places = tf.math.reduce_sum(tf.cast(pred_mask, dtype=tf.int32))
_batch['nb_pred_places'] = nb_pred_places * tf.ones_like(_batch['user_id'], dtype=tf.int32)
 scaling_factor = tf.constant(N_AGGREGATED_QUESTION_SCALING_FACTOR, dtype=tf.float32)
 _batch['n_questions_answered_scaled'] = tf.cast(_batch['n_questions_answered'], dtype=tf.float32)/ scaling_factor
_batch['n_lectures_watched_scaled'] = tf.cast(_batch['n_lectures_watched'], dtype=tf.float32)/ scaling_factor
n_questions_answered = tf.cast(tf.math.maximum(_batch['n_questions_answered'], 1), dtype=tf.float32)
 _batch['answered_correctly_ratio'] = tf.cast(_batch['n_questions_answered_correctly'], dtype=tf.float32)/ n_questions_answered
part_count_scaled = []
part_correctness_ratio = []
for part_idx in range(2, 9):
key = f'part_{part_idx}_count'
_part_count = tf.cast(_batch[key], dtype=tf.float32)
 _part_count_scaled = _part_count / scaling_factor
part_count_scaled.append(_part_count_scaled)
 key_2 = f'part_{part_idx}_correctness_count'
_part_correctness_count = tf.cast(_batch[key_2], dtype=tf.float32)
 _part_correctness_ratio = _part_correctness_count / tf.math.maximum(_part_count, 1.0)
 part_correctness_ratio.append(_part_correctness_ratio)
 del _batch[key]
del _batch[key_2]
part_count_scaled = tf.stack(part_count_scaled, axis=-1)
 part_correctness_ratio = tf.stack(part_correctness_ratio, axis=-1)
 _batch['part_count_scaled'] = part_count_scaled
_batch['part_correctness_ratio'] = part_correctness_ratio
_part_count_scaled = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
part_count_scaled
],
axis=-1 
)
 _part_correctness_ratio = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
part_correctness_ratio
],
axis=-1 
)
 current_part_count_scaled = tf.gather(params=_part_count_scaled, indices=_batch['part_ids'], batch_dims=2)
 current_part_correctness_ratio = tf.gather(params=_part_correctness_ratio, indices=_batch['part_ids'], batch_dims=2)
 _batch['current_part_count_scaled'] = current_part_count_scaled
_batch['current_part_correctness_ratio'] = current_part_correctness_ratio
correct_answer_count_scaled = []
correct_answer_correctness_ratio = []
for correct_answer_idx in range(ANSWER_0_ID, ANSWER_3_ID + 1):
key = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_count'
_correct_answer_count = tf.cast(_batch[key], dtype=tf.float32)
 _correct_answer_count_scaled = _correct_answer_count / scaling_factor
correct_answer_count_scaled.append(_correct_answer_count_scaled)
 key_2 = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_answered_correctly_count'
_correct_answer_answered_correctly_count = tf.cast(_batch[key_2], dtype=tf.float32)
 _correct_answer_answered_correctly_ratio = _correct_answer_answered_correctly_count / tf.math.maximum(_correct_answer_count, 1.0)
 correct_answer_correctness_ratio.append(_correct_answer_answered_correctly_ratio)
 del _batch[key]
del _batch[key_2]
correct_answer_count_scaled = tf.stack(correct_answer_count_scaled, axis=-1)
 correct_answer_correctness_ratio = tf.stack(correct_answer_correctness_ratio, axis=-1)
 _batch['correct_answer_count_scaled'] = correct_answer_count_scaled
_batch['correct_answer_correctness_ratio'] = correct_answer_correctness_ratio
_correct_answer_count_scaled = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis] / scaling_factor,
correct_answer_count_scaled
],
axis=-1 
)
 _correct_answer_correctness_ratio = tf.concat(
 [
tf.constant(PAD_TOKEN, dtype=tf.float32)* tf.ones_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
tf.zeros_like(_batch['content_id'], dtype=tf.float32)[:, :, tf.newaxis],
correct_answer_correctness_ratio
],
axis=-1 
)
 _indices = tf.math.maximum(_batch['correct_answer_id'] - ANSWER_LECTURE_ID + 1, 0)
 current_correct_answer_count_scaled = tf.gather(params=_correct_answer_count_scaled, indices=_indices, batch_dims=2)
 current_correct_answer_correctness_ratio = tf.gather(params=_correct_answer_correctness_ratio, indices=_indices, batch_dims=2)
 _batch['current_correct_answer_count_scaled'] = current_correct_answer_count_scaled
_batch['current_correct_answer_correctness_ratio'] = current_correct_answer_correctness_ratio
_batch['current_question_count_scaled'] = tf.cast(_batch['n_prev_seen'], dtype=tf.float32)/ scaling_factor
current_question_count = tf.cast(tf.math.maximum(_batch['n_prev_seen'], 1), dtype=tf.float32)
 _batch['current_question_correctness_ratio'] = tf.cast(_batch['n_prev_correctness'], dtype=tf.float32)/ current_question_count
return _batch
housing_all_df[(housing_all_df['PoolQC'].isnull())&(housing_all_df['PoolArea'] > 0)][['PoolArea','PoolQC','OverallQual']]<categorify>
def add_aggregated_historical_information(parsed):

 question_mask = tf.cast(parsed['content_type_id'] == 0, dtype=tf.int32)
 lecture_mask = tf.cast(parsed['content_type_id'] == 1, dtype=tf.int32)
 correction_mask = tf.cast(parsed['answered_correctly'] == 1, dtype=tf.int32)
 content_ids = parsed['content_id']
non_pred_time_mask = 1 - parsed['pred_time_mask']
non_pred_time_question_mask = question_mask * non_pred_time_mask
non_pred_time_question_correction_mask = non_pred_time_question_mask * correction_mask
parsed['n_questions_answered'] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(non_pred_time_question_mask[:-1], axis=-1)
 ],
axis=-1 
)
 parsed['n_questions_answered_correctly'] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(non_pred_time_question_correction_mask[:-1], axis=-1)
 ],
axis=-1 
)
 parsed['n_lectures_watched'] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(lecture_mask[:-1], axis=-1)
 ],
axis=-1 
)
 content_input_ids = question_mask * question_id_to_input_id_table.lookup(content_ids)+ lecture_mask * lecture_id_to_input_id_table.lookup(content_ids)
 part_ids = tf.gather(params=c_inputs_ids_to_part, indices=content_input_ids)+ 1
for part_idx in range(2, 9):
key = f'part_{part_idx}_count'
part_mask = tf.cast(part_ids == part_idx, dtype=tf.int32)
 part_mask = part_mask * non_pred_time_question_mask
part_correct_mask = part_mask * non_pred_time_question_correction_mask
parsed[key] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(part_mask[:-1], axis=-1)
 ],
axis=-1 
)
 key = f'part_{part_idx}_correctness_count'
parsed[key] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(part_correct_mask[:-1], axis=-1)
 ],
axis=-1 
)
 correct_answer_id = tf.gather(params=c_inputs_ids_to_correct_answer_id, indices=content_input_ids)
 for correct_answer_idx in range(ANSWER_0_ID, ANSWER_3_ID + 1):
key = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_count'
correct_answer_mask = tf.cast(correct_answer_id == correct_answer_idx, dtype=tf.int32)
 correct_answer_mask = correct_answer_mask * non_pred_time_question_mask
correct_answer_answered_correctly_mask = correct_answer_mask * non_pred_time_question_correction_mask
parsed[key] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(correct_answer_mask[:-1], axis=-1)
 ],
axis=-1 
)
 key = f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_answered_correctly_count'
parsed[key] = tf.concat(
 [
tf.constant([0], dtype=tf.int32),
tf.math.cumsum(correct_answer_answered_correctly_mask[:-1], axis=-1)
 ],
axis=-1 
)
 return parsed
housing_all_df['PoolQC'].fillna(value='None', inplace=True)
 housing_all_df['PoolQC'] = housing_all_df['PoolQC'].map({'None':0,'Po':1,'Fa':2,'TA':3,'Gd':4,'Ex':5})
 housing_all_df[housing_all_df.index == 2420]['PoolQC'] = 4
housing_all_df[housing_all_df.index == 2503]['PoolQC'] = 6
housing_all_df[housing_all_df.index == 2599]['PoolQC'] = 3<drop_column>
def add_valid_block_info(valid_raw_example, num_parallel_calls=None, deterministic=None):

 example = {}
for k in valid_raw_example:
example[k] = valid_raw_example[k]
valid_block_pos = tf.stack([valid_raw_example['valid_blocks_start_pos'], valid_raw_example['valid_blocks_end_pos']], axis=1)
 example['valid_block_pos'] = valid_block_pos
n_valid_blocks = example['n_valid_blocks']
ds_1 = tf.data.Dataset.from_tensors(example ).repeat(tf.cast(n_valid_blocks, dtype=tf.int64))
ds_2 = tf.data.Dataset.range(tf.cast(n_valid_blocks, dtype=tf.int64), output_type=tf.int32)
 ds = tf.data.Dataset.zip(( ds_1, ds_2))
ds = ds.map(lambda ex, valid_block_idx: add_valid_block_idx(ex, valid_block_idx), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 return ds
def add_valid_block_idx(valid_example, valid_block_idx):

 valid_example['valid_block_idx'] = valid_block_idx
valid_example['valid_start'] = valid_example['valid_block_pos'][valid_block_idx][0]
valid_example['valid_end'] = valid_example['valid_block_pos'][valid_block_idx][1]
return valid_example
def remove_future_valid_blocks(valid_example):

 example = {}
valid_start = valid_example['valid_start']
valid_end = valid_example['valid_end']
example['user_id'] = valid_example['user_id']
example['seq_len'] = valid_end + 1
example['prev_seq_len'] = valid_example['seq_len']
example['start'] = tf.constant(0, dtype=tf.int32)
 example['end'] = valid_end
for k in valid_example:
if k not in ['user_id', 'seq_len', 'prev_seq_len', 'start', 'end']:
if k in ['n_valid_blocks', 'valid_blocks_start_pos', 'valid_blocks_end_pos', 'valid_block_pos', 'valid_block_idx', 'valid_start', 'valid_end']:
example[k] = valid_example[k]
else:
example[k] = valid_example[k][0:valid_end + 1]
if k == 'pred_time_mask':
n_valid_interactions = valid_end - valid_example['valid_start'] + 1
example[k] = tf.concat([valid_example[k][:valid_start], tf.ones(shape=[n_valid_interactions], dtype=tf.int32)], axis=0)
 return example
def extract_ending_subseqs(raw_batch, window_size):
ending_indices = raw_batch['seq_len'] - 1
return extract_subseqs_from_raw_batch(raw_batch, ending_indices, window_size)
 def prepare_validation_dataset(valid_raw_ds, batch_size=3, window_size=5, generative=False, use_abs_pos=False, seed=None, num_parallel_calls=None, deterministic=None):
valid_ds = valid_raw_ds.flat_map(lambda valid_raw_example: add_valid_block_info(valid_raw_example, num_parallel_calls=num_parallel_calls, deterministic=deterministic))
valid_ds = valid_ds.map(lambda example: remove_future_valid_blocks(example), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 valid_ds = valid_ds.map(lambda example: add_aggregated_historical_information(example), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 batched_valid_ds = valid_ds.apply(tf.data.experimental.dense_to_ragged_batch(batch_size=batch_size, drop_remainder=(IS_KAGGLE and tpu is not None)))
 batched_valid_ds = batched_valid_ds.map(lambda raw_batch: extract_ending_subseqs(raw_batch, window_size), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 training = tf.constant(0, dtype=tf.int32)
 valid_ds = batched_valid_ds.map(lambda batch: add_input_ids_and_targets(batch, training, generative, use_abs_pos), num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 return valid_ds
housing_all_df.drop(columns=['MiscFeature','MiscVal'], inplace=True )<count_values>
if not IS_KAGGLE:
train_raw_ds = tf.data.TFRecordDataset(train_tfrec_paths, num_parallel_reads=1)
 train_raw_ds = train_raw_ds.map(parse_train_example, num_parallel_calls=1, deterministic=True)
 train_raw_ds = train_raw_ds.map(lambda example: add_aggregated_historical_information(example))
train_valid_split_indices = load_data(train_valid_split_indices_paths[0])
 train_valid_split_table = convert_split_index_dict(train_valid_split_indices)
 reduced_raw_ds = split_train_ds(train_raw_ds, train_valid_split_table, num_parallel_calls=1, deterministic=True)
 batched_raw_ds = reduced_raw_ds.apply(tf.data.experimental.dense_to_ragged_batch(batch_size=5))
batched_ds = random_subseqs_from_batched_raw_ds(batched_raw_ds, window_size=10, seed=1, num_parallel_calls=1, deterministic=True)
 train_ds = prepare_training_dataset(batched_ds, use_abs_pos=False, num_parallel_calls=1, deterministic=True )
print("Null count: ", housing_all_df['Alley'].isnull().sum())
 print(housing_all_df['Alley'].value_counts())
 housing_all_df['Alley'].fillna(value='None', inplace = True )<count_values>
if not IS_KAGGLE:
valid_raw_ds = tf.data.TFRecordDataset([valid_tfrec_paths], num_parallel_reads=1)
 valid_raw_ds = valid_raw_ds.map(parse_train_example_with_valid_info, num_parallel_calls=1, deterministic=True)
 for x in valid_raw_ds.take(1):
for k in x:
print(f'{k} : shape = {x[k].shape}')
 print('--------')
 print('============================================')
 valid_ds = prepare_validation_dataset(valid_raw_ds, batch_size=5, window_size=10, use_abs_pos=False)
 for x in valid_ds.take(1):
for k in x:
print(f'{k} : shape = {x[k].shape}')
 print('--------' )
print("Null count: ", housing_all_df['Fence'].isnull().sum())
 print(housing_all_df['Fence'].value_counts())
 housing_all_df['Fence'].fillna(value='None', inplace=True )<data_type_conversions>
def get_initializer(seed):
return tf.keras.initializers.GlorotUniform(seed=seed)
 def gelu(x):

 x = tf.convert_to_tensor(x)
 cdf = 0.5 *(1.0 + tf.math.erf(x / tf.cast(tf.math.sqrt(2.0), dtype=x.dtype)))
 return x * cdf
def gelu_new(x):

 x = tf.convert_to_tensor(x)
 pi = tf.cast(math.pi, x.dtype)
 coeff = tf.cast(0.044715, x.dtype)
 cdf = 0.5 *(1.0 + tf.tanh(tf.sqrt(2.0 / pi)*(x + coeff * tf.pow(x, 3))))
return x * cdf
def gelu_fast(x):
x = tf.convert_to_tensor(x)
 coeff1 = tf.cast(7978845608, x.dtype)
 coeff2 = tf.cast(0.044715, x.dtype)
 return 0.5 * x *(1.0 + tf.tanh(x * coeff2 *(1.0 + coeff1 * x * x)))
 ACT2FN = {
"gelu": tf.keras.layers.Activation(gelu),
"relu": tf.keras.activations.relu,
"swish": tf.keras.activations.swish,
"silu": tf.keras.activations.swish,
"gelu_new": tf.keras.layers.Activation(gelu_new),
"tanh": tf.keras.activations.tanh,
"gelu_fast": tf.keras.layers.Activation(gelu_fast),
}
def get_tf_activation(activation_string):
if activation_string in ACT2FN:
return ACT2FN[activation_string]
else:
raise KeyError("function {} not found in ACT2FN mapping {}".format(activation_string, list(ACT2FN.keys())) )
housing_all_df['FireplaceQu'].fillna(value='None', inplace=True )<categorify>
class EdFormerConfig(PretrainedConfig):
def __init__(
 self,
model_type,
model_desc,
model_size='none',
content_vocab_size=CONTENT_VOCAB_SIZE,
response_vocab_size=RESPONSE_VOCAB_SIZE,
tag_vocab_size=TAG_VOCAB_SIZE,
part_vocab_size=PART_VOCAB_SIZE,
prior_explanation_vocab_size=PRIOR_EXPLANATION_VOCAB_SIZE,
max_position_embeddings=WINDOW_SIZE + 1,
sinusoidal_pos_embds=False,
n_layers=4,
n_heads=8,
dim=512,
hidden_dim=4 * 512,
activation=ACTIVATION,
dropout=0.1,
attention_dropout=0.1,
seq2seq_dropout=0.1,
initializer_range=0.02,
seed=SEED,
pad_token_id=PAD_ID,
use_user_answer=USE_USER_ANSWER,
use_user_answer_loss=USE_USER_ANSWER_LOSS,
use_correct_answer_for_encoder=USE_CORRECT_ANSWER_FOR_ENCODER,
use_correct_answer_for_decoder=USE_CORRECT_ANSWER_FOR_DECODER,
use_abs_pos=USE_ABS_POS,
use_task_container_pos=USE_TASK_CONTAINER_POS,
share_position_embeddings=SHARE_POS_EMBEDDING,
use_tags=USE_TAGS,
use_part=USE_PART,
use_prior_explanation=USE_PRIOR_EXPLANATION,
use_prior_question_elapsed_time_input=USE_PRIOR_QUESTION_ELAPSED_TIME_INPUT,
use_lag_time=USE_LAG_TIME,
use_lag_time_for_encoder=USE_LAG_TIME_FOR_ENCODER,
use_user_level_aggregated_historical_info=USE_USER_LEVEL_AGGREGATED_HISTORICAL_INFO,
use_part_aggregated_historical_info=USE_PART_AGGREGATED_HISTORICAL_INFO,
use_correct_answer_aggregated_historical_info=USE_CORRECT_ANSWER_AGGREGATED_HISTORICAL_INFO,
use_question_level_aggregated_historical_info=USE_QUESTION_LEVEL_AGGREGATED_HISTORICAL_INFO,
allow_bundle_atten=ALLOW_BUNDLE_ATTEN,
generative=GENERATIVE,
use_pre_classifier=USE_PRE_CLASSIFIER,
use_softmax=USE_SOFTMAX,
**kwargs 
):
super().__init__(**kwargs, pad_token_id=pad_token_id)
 self.model_type = model_type
self.model_size = model_size
self.model_desc = model_desc
self.content_vocab_size = content_vocab_size
self.response_vocab_size = response_vocab_size
self.tag_vocab_size = tag_vocab_size
self.part_vocab_size = part_vocab_size
self.prior_explanation_vocab_size = prior_explanation_vocab_size
self.use_user_answer = use_user_answer
self.use_correct_answer_for_encoder = use_correct_answer_for_encoder
self.use_correct_answer_for_decoder = use_correct_answer_for_decoder
self.use_user_answer_loss = use_user_answer_loss
self.use_abs_pos = use_abs_pos
self.use_task_container_pos = use_task_container_pos
self.share_position_embeddings = share_position_embeddings
self.use_tags = use_tags
self.use_part = use_part
self.use_prior_explanation = use_prior_explanation
self.allow_bundle_atten = allow_bundle_atten
self.generative = generative
self.use_prior_question_elapsed_time_input = use_prior_question_elapsed_time_input
self.use_part_aggregated_historical_info = use_part_aggregated_historical_info
self.use_correct_answer_aggregated_historical_info=use_correct_answer_aggregated_historical_info
self.use_question_level_aggregated_historical_info=use_question_level_aggregated_historical_info
self.use_lag_time = use_lag_time
self.use_lag_time_for_encoder = use_lag_time_for_encoder
self.use_user_level_aggregated_historical_info = use_user_level_aggregated_historical_info
self.use_pre_classifier = use_pre_classifier
self.use_softmax = use_softmax
self.max_position_embeddings = max_position_embeddings
self.sinusoidal_pos_embds = sinusoidal_pos_embds
self.n_layers = n_layers
self.n_heads = n_heads
self.dim = dim
self.hidden_dim = hidden_dim
self.activation = activation
self.dropout = dropout
self.attention_dropout = attention_dropout
self.seq2seq_dropout = seq2seq_dropout
self.initializer_range = initializer_range
self.seed = seed
def toJSON(self):
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=False, indent=4)
 def vocab_size(self, input_name):
if input_name == 'content':
return self.content_vocab_size
elif input_name == 'response':
return self.response_vocab_size
elif input_name == 'tag':
return self.tag_vocab_size
elif input_name == 'part':
return self.part_vocab_size
elif input_name == 'prior_explanation':
return self.prior_explanation_vocab_size
else:
raise ValueError('input name not used for model')
 @property
def hidden_size(self):
return self.dim
@property
def num_attention_heads(self):
return self.n_heads
@property
def num_hidden_layers(self):
return self.n_layers
class TFSharedEmbeddings(tf.keras.layers.Layer):
def __init__(self, vocab_size, hidden_size, seed=None, **kwargs):
super().__init__(**kwargs)
 self.vocab_size = vocab_size
self.hidden_size = hidden_size
self.seed = seed
def build(self, input_shape):
self.weight = self.add_weight(
 "weight", shape=[self.vocab_size, self.hidden_size], initializer=get_initializer(seed=self.seed)
)
 super().build(input_shape)
 def call(self, inputs: tf.Tensor, mode: str = "embedding")-> tf.Tensor:
if mode == "embedding":
return self._embedding(inputs)
 elif mode == "linear":
return self._linear(inputs)
 else:
raise ValueError("mode {} is not valid.".format(mode))
def _embedding(self, input_ids):

 return tf.gather(self.weight, input_ids)
 def _linear(self, inputs):
first_dims = shape_list(inputs)[:-1]
x = tf.reshape(inputs, [-1, self.hidden_size])
 logits = tf.matmul(x, self.weight, transpose_b=True)
 return tf.reshape(logits, first_dims + [self.vocab_size])
 class TFEmbeddings(tf.keras.layers.Layer):
def __init__(
 self, config, input_name,
position_embeddings_layer=None,
**kwargs 
):
super().__init__(**kwargs)
 self.pad_id = config.pad_token_id
self.input_name = input_name
self.vocab_size = config.vocab_size(input_name)
 self.use_user_answer = config.use_user_answer
self.use_correct_answer_for_encoder = config.use_correct_answer_for_encoder
self.use_correct_answer_for_decoder = config.use_correct_answer_for_decoder
self.use_abs_pos = config.use_abs_pos
self.use_task_container_pos = config.use_task_container_pos
self.use_tags = config.use_tags
self.use_part = config.use_part
self.use_prior_explanation = config.use_prior_explanation
self.use_prior_question_elapsed_time_input = config.use_prior_question_elapsed_time_input
self.use_lag_time = config.use_lag_time
self.use_lag_time_for_encoder = config.use_lag_time_for_encoder
self.use_user_level_aggregated_historical_info = config.use_user_level_aggregated_historical_info
self.use_part_aggregated_historical_info = config.use_part_aggregated_historical_info
self.use_correct_answer_aggregated_historical_info = config.use_correct_answer_aggregated_historical_info
self.use_question_level_aggregated_historical_info = config.use_question_level_aggregated_historical_info
self.dim = config.dim
self.seed = config.seed
assert config.activation in ["relu", "gelu"], "activation({})must be in ['relu', 'gelu']".format(
 config.activation 
)
 self.activation = get_tf_activation(config.activation)
 self.word_embeddings = TFSharedEmbeddings(
 self.vocab_size, config.dim, seed=config.seed, name="word_embeddings" 
)
 self.word_embeddings_2 = TFSharedEmbeddings(
 ANSWER_VOCAB_SIZE, config.dim, seed=config.seed, name="word_embeddings_2" 
)
 if position_embeddings_layer is None:
self.position_embeddings = tf.keras.layers.Embedding(
 config.max_position_embeddings,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="position_embeddings", 
)
 else:
self.position_embeddings = position_embeddings_layer
self.correct_answer_embeddings = tf.keras.layers.Embedding(
 ANSWER_VOCAB_SIZE,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="correct_answer_embeddings", 
)
 self.tag_embeddings = tf.keras.layers.Embedding(
 config.tag_vocab_size,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="tag_embeddings", 
)
 self.part_embeddings = tf.keras.layers.Embedding(
 config.part_vocab_size,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="part_embeddings", 
)
 self.prior_explanation_embeddings = tf.keras.layers.Embedding(
 config.prior_explanation_vocab_size,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="prior_explanation_embeddings", 
)
 self.LayerNorm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="LayerNorm")
 self.dropout = tf.keras.layers.Dropout(0.05)
 def build(self, input_shape):

 with tf.name_scope("word_embeddings"):
self.word_embeddings = self.add_weight(
 "weight", shape=[self.vocab_size, self.dim], initializer=get_initializer(self.seed)
)
 if self.input_name == 'response':
with tf.name_scope("word_embeddings_2"):
self.word_embeddings_2 = self.add_weight(
 "weight", shape=[ANSWER_VOCAB_SIZE, self.dim], initializer=get_initializer(self.seed)
)
 super().build(input_shape)
 def call(
 self,
input_ids=None, input_ids_2=None, position_ids=None,
tag_ids=None, part_ids=None,
prior_explanation_ids=None,
correct_answer_ids=None,
dense_embeddings_encoder=None,
dense_embeddings_decoder=None,
mode="embedding", training=False 
):
if mode == "embedding":
return self._embedding(
 input_ids, input_ids_2, position_ids,
tag_ids, part_ids,
prior_explanation_ids,
correct_answer_ids,
dense_embeddings_encoder,
dense_embeddings_decoder,
training=training 
)
 elif mode == "linear":
return self._linear(input_ids)
 else:
raise ValueError("mode {} is not valid.".format(mode))
def _embedding(
 self,
input_ids, input_ids_2, position_ids,
tag_ids, part_ids,
prior_explanation_ids,
correct_answer_ids,
dense_embeddings_encoder,
dense_embeddings_decoder,
training=False 
):
seq_length = shape_list(input_ids)[1]
inputs_embeds = tf.gather(self.word_embeddings, input_ids)
 position_embeddings = tf.cast(
 self.position_embeddings(position_ids), inputs_embeds.dtype 
)
 if self.input_name == 'content':
tag_embeddings = tf.cast(
 self.tag_embeddings(tag_ids), inputs_embeds.dtype 
)
 tag_mask = tf.cast(tag_ids != self.pad_id, dtype=tf.int32)
 tag_embeddings = tag_embeddings * tf.cast(tag_mask, dtype=inputs_embeds.dtype)[:, :, :, tf.newaxis]
nb_tags = tf.math.reduce_sum(tag_mask, axis=2)
 nb_tags = tf.cast(nb_tags, dtype=inputs_embeds.dtype)
 nb_tags = tf.math.maximum(nb_tags, tf.cast(1.0, dtype=inputs_embeds.dtype))
tag_embeddings = tf.math.reduce_sum(tag_embeddings, axis=2)/ nb_tags[:, :, tf.newaxis]
part_embeddings = tf.cast(
 self.part_embeddings(part_ids), inputs_embeds.dtype 
)
 correct_answer_embeddings = tf.cast(self.correct_answer_embeddings(correct_answer_ids), inputs_embeds.dtype)
 concated_embeddings = tf.concat(
 [
inputs_embeds[tf.newaxis, :, :, :],
position_embeddings[tf.newaxis, :, :, :],
tag_embeddings[tf.newaxis, :, :, :],
part_embeddings[tf.newaxis, :, :, :],
correct_answer_embeddings[tf.newaxis, :, :, :],
dense_embeddings_encoder
],
axis=0 
)
 if self.input_name == 'response':
inputs_embeds_2 = tf.gather(self.word_embeddings_2, input_ids_2)
 prior_explanation_embeddings = tf.cast(
 self.prior_explanation_embeddings(prior_explanation_ids), inputs_embeds.dtype 
)
 concated_embeddings = tf.concat(
 [
inputs_embeds[tf.newaxis, :, :, :],
inputs_embeds_2[tf.newaxis, :, :, :],
position_embeddings[tf.newaxis, :, :, :],
prior_explanation_embeddings[tf.newaxis, :, :, :],
dense_embeddings_decoder,
],
axis=0 
)
 embeddings = tf.math.reduce_sum(concated_embeddings, axis=0)
 embeddings = self.LayerNorm(embeddings)
 embeddings = self.dropout(embeddings, training=training)
 return embeddings
class TFMultiHeadSelfAttention(HFTFMultiHeadSelfAttention):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.q_lin = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="q_lin" 
)
 self.k_lin = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="k_lin" 
)
 self.v_lin = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="v_lin" 
)
 self.out_lin = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="out_lin" 
)
 def call(self, query, key, value, mask, head_mask, output_attentions, training=False):

 bs, q_length, dim = shape_list(query)
 k_length = shape_list(key)[1]
dim_per_head = tf.math.divide(self.dim, self.n_heads)
 dim_per_head = tf.cast(dim_per_head, dtype=tf.int32)
 def shape(x):

 return tf.transpose(tf.reshape(x,(bs, -1, self.n_heads, dim_per_head)) , perm=(0, 2, 1, 3))
def unshape(x):

 return tf.reshape(tf.transpose(x, perm=(0, 2, 1, 3)) ,(bs, -1, self.n_heads * dim_per_head))
q = shape(self.q_lin(query))
k = shape(self.k_lin(key))
v = shape(self.v_lin(value))
q = tf.cast(q, dtype=DTYPE)
 q = tf.multiply(q, tf.math.rsqrt(tf.cast(dim_per_head, dtype=DTYPE)))
 k = tf.cast(k, dtype=q.dtype)
 scores = tf.matmul(q, k, transpose_b=True)
 mask = mask[:, tf.newaxis, :, :]
mask = tf.cast(mask, dtype=scores.dtype)
 scores = scores - 1e30 *(1.0 - mask)
 weights = tf.nn.softmax(scores, axis=-1)
 weights = self.dropout(weights, training=training)
 weights = weights * mask
if head_mask is not None:
weights = weights * head_mask
context = tf.matmul(weights, v)
 context = unshape(context)
 context = self.out_lin(context)
 if output_attentions:
return(context, weights)
 else:
return(context,)
 class TFFFN(HFTFFFN):
def __init__(self, config, **kwargs):
super(TFFFN, self ).__init__(**kwargs)
 self.lin1 = tf.keras.layers.Dense(
 config.hidden_dim,
kernel_initializer=get_initializer(config.seed),
name="lin1" 
)
 self.lin2 = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="lin2" 
)
 self.activation = get_tf_activation(config.activation)
 class TFTransformerBlock(HFTFTransformerBlock):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.attention = TFMultiHeadSelfAttention(config, name="attention")
 class TFContentBlock(TFTransformerBlock):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.attention = TFMultiHeadSelfAttention(config, name="c_attention")
 class TFResponseBlock(TFTransformerBlock):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.attention = TFMultiHeadSelfAttention(config, name="r_attention")
 self.r_c_attentioin = TFMultiHeadSelfAttention(config, name="r_c_attentioin")
 self.r_c_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-12, name="r_c_attn_layer_norm")
 def call(self, r, c_hidden, r_mask, r_c_mask, head_mask, output_attentions, training=False):
r_output = self.attention(r, r, r, r_mask, head_mask, output_attentions, training=training)
 if output_attentions:
r_output, r_weights = r_output
else:
r_output = r_output[0]
r_output = self.sa_layer_norm(r_output + r)
 r_c_output = self.r_c_attentioin(
 query=r_output,
key=c_hidden,
value=c_hidden,
mask=r_c_mask,
head_mask=head_mask,
output_attentions=output_attentions,
training=training 
)
 if output_attentions:
r_c_output, r_c_weights = r_c_output
else:
r_c_output = r_c_output[0]
r_c_output = self.r_c_attn_layer_norm(r_c_output + r_output)
 ffn_output = self.ffn(r_c_output, training=training)
 ffn_output = self.output_layer_norm(ffn_output + r_c_output)
 output =(ffn_output,)
 if output_attentions:
output =(r_weights, r_c_weights)+ output
return output
class TFCRBlock(TFTransformerBlock):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.attention = TFMultiHeadSelfAttention(config, name="cr_attention")
 class TFContentCoder(TFTransformer):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.layer = [TFContentBlock(config, name="layer_._{}".format(i)) for i in range(config.n_layers)]
class TFResponseCoder(TFTransformer):
def __init__(self, config, **kwargs):
super().__init__(config, **kwargs)
 self.layer = [TFResponseBlock(config, name="layer_._{}".format(i)) for i in range(config.n_layers)]
def call(self, r_embeds, c_hidden, r_mask, r_c_mask, head_mask, output_attentions, output_hidden_states, return_dict, training=False):

 all_hidden_states =() if output_hidden_states else None
all_attentions =() if output_attentions else None
hidden_state = r_embeds
for i, layer_module in enumerate(self.layer):
if output_hidden_states:
all_hidden_states = all_hidden_states +(hidden_state,)
 layer_outputs = layer_module(hidden_state, c_hidden, r_mask, r_c_mask, head_mask[i], output_attentions, training=training)
 hidden_state = layer_outputs[-1]
if output_attentions:
assert len(layer_outputs)== 3
r_attn = layer_outputs[0]
r_c_attn = layer_outputs[1]
all_attentions = all_attentions +(( r_attn, r_c_attn),)
 else:
assert len(layer_outputs)== 1, f"Incorrect number of outputs {len(layer_outputs)} instead of 1"
if output_hidden_states:
all_hidden_states = all_hidden_states +(hidden_state,)
 if not return_dict:
return tuple(v for v in [hidden_state, all_hidden_states, all_attentions] if v is not None)
 return TFBaseModelOutput(
 last_hidden_state=hidden_state, hidden_states=all_hidden_states, attentions=all_attentions 
)
 def process_mask(mask, input_shpae):
if mask is None:
mask = tf.ones(input_shpae)
 mask = tf.cast(mask, dtype=tf.float32)
 return mask
class TFEdFormerEncoderDecoder(tf.keras.layers.Layer):
def __init__(self, config, **kwargs):
super().__init__(**kwargs)
 self.encoder = TFContentCoder(config, name="encoder")
 self.decoder = TFResponseCoder(config, name="decoder")
 def call(
 self,
c_embeds,
r_embeds,
d_embeds,
c_mask,
r_mask,
r_c_mask,
c_r_mask,
head_mask,
output_attentions,
output_hidden_states,
return_dict,
training=False, 
):
c_outputs = self.encoder(
 c_embeds, c_mask, head_mask,
output_attentions, output_hidden_states, return_dict, training 
)
 if not return_dict:
c_hidden = c_outputs[0]
else:
c_hidden = c_outputs.last_hidden_state
r_outputs = self.decoder(
 d_embeds, c_hidden, r_mask, r_c_mask, head_mask,
output_attentions, output_hidden_states, return_dict, training 
)
 if not return_dict:
r_hidden = r_outputs[0]
else:
r_hidden = r_outputs.last_hidden_state
hidden_states = r_hidden
return(hidden_states, c_outputs, r_outputs)
 class TFEdFormerMainLayer(tf.keras.layers.Layer):
config_class = EdFormerConfig
def __init__(self, config, **kwargs):
super().__init__(**kwargs)
 self.num_hidden_layers = config.num_hidden_layers
self.output_attentions = config.output_attentions
self.output_hidden_states = config.output_hidden_states
self.return_dict = config.use_return_dict
self.use_user_answer = config.use_user_answer
self.use_correct_answer_for_encoder = config.use_correct_answer_for_encoder
self.use_correct_answer_for_decoder = config.use_correct_answer_for_decoder
self.use_abs_pos = config.use_abs_pos
self.use_task_container_pos = config.use_task_container_pos
self.use_tags = config.use_tags
self.use_part = config.use_part
self.use_prior_explanation = config.use_prior_explanation
self.use_prior_question_elapsed_time_input = config.use_prior_question_elapsed_time_input
self.use_lag_time = config.use_lag_time
self.use_lag_time_for_encoder = config.use_lag_time_for_encoder
self.use_user_level_aggregated_historical_info = config.use_user_level_aggregated_historical_info
self.use_part_aggregated_historical_info = config.use_part_aggregated_historical_info
self.use_correct_answer_aggregated_historical_info = config.use_correct_answer_aggregated_historical_info
self.use_question_level_aggregated_historical_info = config.use_question_level_aggregated_historical_info
if config.share_position_embeddings:
position_embeddings = tf.keras.layers.Embedding(
 config.max_position_embeddings,
config.dim,
embeddings_initializer=get_initializer(config.seed),
name="position_embeddings", 
)
 else:
position_embeddings = None
self.prior_question_elapsed_time_embeddings = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="prior_question_elapsed_time_embeddings", 
)
 self.lag_time_embeddings = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="lag_time_embeddings", 
)
 self.key_names_for_encoder = {
'n_questions_answered_scaled',
'n_lectures_watched_scaled',
'part_count_scaled',
'current_part_count_scaled',
'correct_answer_count_scaled',
'current_correct_answer_count_scaled',
'current_question_count_scaled',
}
self.key_name_need_newaxis = {
'n_questions_answered_scaled',
'n_lectures_watched_scaled',
'current_part_count_scaled',
'current_correct_answer_count_scaled',
'current_question_count_scaled',
'prior_question_elapsed_time_input',
'lag_time',
'answered_correctly_ratio',
'current_part_correctness_ratio',
'current_correct_answer_correctness_ratio',
'current_question_correctness_ratio',
}
self.dense_layer_name_mapping = {
'n_questions_answered_scaled': ['n_questions_answered_embeddings', 'n_questions_answered_layer_norm'],
'n_lectures_watched_scaled': ['n_lectures_watched_embeddings', 'n_lectures_watched_layer_norm'],
'part_count_scaled': ['part_count_embeddings', 'part_count_layer_norm'],
'current_part_count_scaled': ['current_part_count_embeddings', 'current_part_count_layer_norm'],
'correct_answer_count_scaled': ['correct_answer_count_embeddings', 'correct_answer_count_layer_norm'],
'current_correct_answer_count_scaled': ['current_correct_answer_count_embeddings', 'current_correct_answer_count_layer_norm'],
'current_question_count_scaled': ['current_question_count_embeddings', 'current_question_count_layer_norm'],
'prior_question_elapsed_time_input': ['prior_question_elapsed_time_input_embeddings', 'prior_question_elapsed_time_input_layer_norm'],
'lag_time': ['lag_time_embeddings', 'lag_time_layer_norm'],
'answered_correctly_ratio': ['answered_correctly_ratio_embeddings', 'answered_correctly_ratio_layer_norm'],
'part_correctness_ratio': ['part_correctness_ratio_embeddings', 'part_correctness_ratio_layer_norm'],
'current_part_correctness_ratio': ['current_part_correctness_ratio_embeddings', 'current_part_correctness_ratio_layer_norm'],
'correct_answer_correctness_ratio': ['correct_answer_correctness_ratio_embeddings', 'correct_answer_correctness_ratio_layer_norm'],
'current_correct_answer_correctness_ratio': ['current_correct_answer_correctness_ratio_embeddings', 'current_correct_answer_correctness_ratio_layer_norm'],
'current_question_correctness_ratio': ['current_question_correctness_ratio_embeddings', 'current_question_correctness_ratio_layer_norm'],
}
self.dense_layer_keys = [
'n_questions_answered_scaled',
'n_lectures_watched_scaled',
'part_count_scaled',
'current_part_count_scaled',
'correct_answer_count_scaled',
'current_correct_answer_count_scaled',
'current_question_count_scaled',
'prior_question_elapsed_time_input',
'lag_time',
'answered_correctly_ratio',
'part_correctness_ratio',
'current_part_correctness_ratio',
'correct_answer_correctness_ratio',
'current_correct_answer_correctness_ratio',
'current_question_correctness_ratio'
]
self.dense_layer_mapping = dict()
for key_name, layer_names in self.dense_layer_name_mapping.items() :
setattr(
 self,
layer_names[0],
tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name=layer_names[0], 
)
)
 setattr(
 self,
layer_names[1],
tf.keras.layers.LayerNormalization(epsilon=1e-12, name=layer_names[1])
)
 self.dense_layer_mapping[key_name] = [getattr(self, layer_names[0]), getattr(self, layer_names[1])]
assert config.activation in ["relu", "gelu"], "activation({})must be in ['relu', 'gelu']".format(
 config.activation 
)
 self.activation = get_tf_activation(config.activation)
 self.dropout = tf.keras.layers.Dropout(0.05)
 self.content_embeddings = TFEmbeddings(
 config,
input_name = 'content',
position_embeddings_layer=position_embeddings,
name="content_embeddings" 
)
 self.response_embeddings = TFEmbeddings(
 config,
input_name = 'response',
position_embeddings_layer=position_embeddings,
name="response_embeddings" 
)
 self.coder = TFEdFormerEncoderDecoder(config, name='coder')
 def call(
 self,
inputs,
c_mask=None,
r_mask=None,
r_c_mask=None,
c_r_mask=None,
head_mask=None,
output_attentions=None,
output_hidden_states=None,
return_dict=None,
training=False, 
):
c_input_ids = inputs.get('c_input_ids')
 r_input_ids = inputs.get('r_input_ids')
 d_input_ids = inputs.get('d_input_ids')
 d_ans_input_ids = inputs.get('d_ans_input_ids')
 tag_ids = inputs.get('tag_ids')
 part_ids = inputs.get('part_ids')
 batch_size = tf.math.reduce_sum(tf.ones_like(c_input_ids[:, 0], dtype=tf.int32))
seq_len = tf.math.reduce_sum(tf.ones_like(c_input_ids[0, :], dtype=tf.int32))
pos_ids = inputs['pos_ids']
shifted_pos_ids = inputs['shifted_pos_ids']
prior_explanation_ids = inputs.get('prior_explanation_ids')
 correct_answer_ids = inputs.get('correct_answer_id')
 c_mask = inputs.get('c_mask', c_mask)
 r_mask = inputs.get('r_mask', r_mask)
 r_c_mask = inputs.get('r_c_mask', r_c_mask)
 c_r_mask = inputs.get('c_r_mask', c_r_mask)
 output_attentions = inputs.get('output_attentions', output_attentions)
 output_hidden_states = inputs.get('output_hidden_states', output_hidden_states)
 return_dict = inputs.get('return_dict', return_dict)
 output_attentions = output_attentions if output_attentions is not None else self.output_attentions
output_hidden_states = output_hidden_states if output_hidden_states is not None else self.output_hidden_states
return_dict = return_dict if return_dict is not None else self.return_dict
c_mask = process_mask(c_mask, [batch_size, seq_len, seq_len])
 r_mask = process_mask(r_mask, [batch_size, seq_len, seq_len])
 r_c_mask = process_mask(r_c_mask, [batch_size, seq_len, seq_len])
 c_r_mask = process_mask(c_r_mask, [batch_size, seq_len, seq_len])
 head_mask = [None] * self.num_hidden_layers
dense_embeddings_encoder = []
dense_embeddings_decoder = []
for key_name, layers in self.dense_layer_mapping.items() :
dense_layer = layers[0]
layer_norm = layers[1]
layer_input = inputs[key_name]
if key_name in self.key_name_need_newaxis:
layer_input = layer_input[:, :, tf.newaxis]
layer_output = dense_layer(layer_input)
 layer_output = layer_norm(
 self.dropout(
 self.activation(
 layer_output 
),
training=training 
)+ layer_output 
)
 if key_name in self.key_names_for_encoder:
dense_embeddings_encoder.append(layer_output)
 layer_output = dense_layer(layer_input)
 layer_output = layer_norm(
 self.dropout(
 self.activation(
 layer_output 
),
training=training 
)+ layer_output 
)
 dense_embeddings_decoder.append(layer_output)
 else:
dense_embeddings_decoder.append(layer_output)
 dense_embeddings_encoder = tf.stack(dense_embeddings_encoder, axis=0)
 dense_embeddings_decoder= tf.stack(dense_embeddings_decoder, axis=0)
 c_embedding_output = self.content_embeddings(
 input_ids=c_input_ids, input_ids_2=d_ans_input_ids, position_ids=pos_ids,
tag_ids=tag_ids, part_ids=part_ids,
prior_explanation_ids=prior_explanation_ids,
correct_answer_ids=correct_answer_ids,
dense_embeddings_encoder=dense_embeddings_encoder,
dense_embeddings_decoder=dense_embeddings_decoder,
training=training 
)
 d_embedding_output = self.response_embeddings(
 input_ids=d_input_ids, input_ids_2=d_ans_input_ids, position_ids=shifted_pos_ids,
tag_ids=tag_ids, part_ids=part_ids,
prior_explanation_ids=prior_explanation_ids,
correct_answer_ids=correct_answer_ids,
dense_embeddings_encoder=dense_embeddings_encoder,
dense_embeddings_decoder=dense_embeddings_decoder,
training=training 
)
 r_embedding_output = d_embedding_output
outputs = self.coder(
 c_embedding_output,
r_embedding_output,
d_embedding_output,
c_mask,
r_mask,
r_c_mask,
c_r_mask,
head_mask,
output_attentions,
output_hidden_states,
return_dict,
training=training 
)
 return outputs
class TFEdFormerPreTrainedModel(TFPreTrainedModel):

 config_class = EdFormerConfig
base_model_prefix = "edformer"
class TFEdFormerModel(TFEdFormerPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.edformer = TFEdFormerMainLayer(config, name="edformer")
 def call(self, inputs, **kwargs):
outputs = self.edformer(
 inputs,
**kwargs 
)
 return outputs
class TFEdFormerAnswerPredictionModel(TFPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
 self.use_pre_classifier = config.use_pre_classifier
self.edformer = TFEdFormerMainLayer(config, name="edformer")
 if self.use_pre_classifier:
self.pre_classifier = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="pre_classifier", 
)
 self.pre_classifier_2 = tf.keras.layers.Dense(
 config.dim,
kernel_initializer=get_initializer(config.seed),
name="pre_classifier_2", 
)
 assert config.activation in ["relu", "gelu"], "activation({})must be in ['relu', 'gelu']".format(
 config.activation 
)
 self.activation = get_tf_activation(config.activation)
 self.dropout = tf.keras.layers.Dropout(config.seq2seq_dropout)
 n_targets = 1
if config.use_softmax:
n_targets = 2
self.classifier = tf.keras.layers.Dense(
 n_targets, kernel_initializer=get_initializer(config.seed), name="classifier" 
)
 self.classifier_2 = tf.keras.layers.Dense(
 4, kernel_initializer=get_initializer(config.seed), name="classifier_2" 
)
 def call(
 self,
inputs=None,
c_mask=None,
r_mask=None,
r_c_mask=None,
c_r_mask=None,
head_mask=None,
output_attentions=None,
output_hidden_states=None,
return_dict=None,
training=False, 
):
return_dict = return_dict if return_dict is not None else self.edformer.return_dict
edformer_output = self.edformer(
 inputs,
c_mask=c_mask,
r_mask=r_mask,
r_c_mask=r_c_mask,
c_r_mask=c_r_mask,
head_mask=head_mask,
output_attentions=output_attentions,
output_hidden_states=output_hidden_states,
return_dict=return_dict,
training=training, 
)
 hidden_states, c_outputs, r_outputs = edformer_output
o = hidden_states
o_2 = o
if self.use_pre_classifier:
o = self.pre_classifier(o)
 o = self.activation(o)
 o = self.dropout(o, training=training)
 o_2 = self.pre_classifier_2(o_2)
 o_2 = self.activation(o_2)
 o_2 = self.dropout(o_2, training=training)
 logits = self.classifier(o)
 answer_logits = self.classifier_2(o_2)
 return(logits, answer_logits, c_outputs, r_outputs )
grouped_by_neighborhood = housing_all_df.groupby('Neighborhood')
 lot_frontage_median = grouped_by_neighborhood['LotFrontage'].median()
lot_frontage_substitute = housing_all_df['Neighborhood'].map(lot_frontage_median.to_dict())
 housing_all_df['LotFrontage'] = lot_frontage_substitute.where(pd.isnull(housing_all_df['LotFrontage']), housing_all_df['LotFrontage'] )<feature_engineering>
if not IS_KAGGLE or True:
content_input_ids = tf.constant(1, shape=[3, 5])
 response_input_ids = tf.constant(1, shape=[3, 5])
 pos_ids = tf.constant(1, shape=[3, 5])
 shifted_pos_ids = tf.constant(1, shape=[3, 5])
 d_input_ids = tf.constant(1, shape=[3, 5])
 d_ans_input_ids = tf.constant(1, shape=[3, 5])
 tag_ids = tf.constant(0, shape=[3, 5, N_TAGS_PER_CONTENT])
 part_ids = tf.constant(1, shape=[3, 5])
 prior_explanation_ids = tf.constant(1, shape=[3, 5])
 prior_question_elapsed_time_input = tf.constant(1.0, shape=[3, 5])
 lag_time = tf.constant(1.0, shape=[3, 5])
 abs_pos_ids = tf.constant(1.0, shape=[3, 5])
 shifted_abs_pos_ids = tf.constant(1.0, shape=[3, 5])
 task_container_pos_ids = tf.constant(1.0, shape=[3, 5])
 correct_answer_id = tf.constant(1, shape=[3, 5])
 n_questions_answered_scaled = tf.constant(1.0, shape=[3, 5])
 n_lectures_watched_scaled = tf.constant(1.0, shape=[3, 5])
 answered_correctly_ratio = tf.constant(1.0, shape=[3, 5])
 part_correctness_ratio = tf.constant(1.0, shape=[3, 5, PART_VOCAB_SIZE - 2])
 part_count_scaled = tf.constant(1.0, shape=[3, 5, PART_VOCAB_SIZE - 2])
 correct_answer_count_scaled = tf.constant(1.0, shape=[3, 5, ANSWER_3_ID - ANSWER_0_ID + 1])
 correct_answer_correctness_ratio = tf.constant(1.0, shape=[3, 5, ANSWER_3_ID - ANSWER_0_ID + 1])
 current_part_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_part_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 current_correct_answer_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_correct_answer_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 current_question_count_scaled = tf.constant(1.0, shape=[3, 5])
 current_question_correctness_ratio = tf.constant(1.0, shape=[3, 5])
 inputs = {
'c_input_ids': content_input_ids,
'r_input_ids': response_input_ids,
'd_input_ids': d_input_ids,
'd_ans_input_ids': d_ans_input_ids,
'pos_ids': pos_ids,
'shifted_pos_ids': shifted_pos_ids,
'tag_ids': tag_ids,
'part_ids': part_ids,
'prior_explanation_ids': prior_explanation_ids,
'prior_question_elapsed_time_input': prior_question_elapsed_time_input,
'lag_time': lag_time,
'abs_pos_ids': abs_pos_ids,
'shifted_abs_pos_ids': shifted_abs_pos_ids,
'task_container_pos_ids': task_container_pos_ids,
'correct_answer_id': correct_answer_id,
'n_questions_answered_scaled': n_questions_answered_scaled,
'n_lectures_watched_scaled': n_lectures_watched_scaled,
'answered_correctly_ratio': answered_correctly_ratio,
'part_correctness_ratio': part_correctness_ratio,
'part_count_scaled': part_count_scaled,
'correct_answer_count_scaled': correct_answer_count_scaled,
'correct_answer_correctness_ratio': correct_answer_correctness_ratio,
'current_part_count_scaled': current_part_count_scaled,
'current_part_correctness_ratio': current_part_correctness_ratio,
'current_correct_answer_count_scaled': current_correct_answer_count_scaled,
'current_correct_answer_correctness_ratio': current_correct_answer_correctness_ratio,
'current_question_count_scaled': current_question_count_scaled,
'current_question_correctness_ratio': current_question_correctness_ratio,
}
model_type = 'ed'
config = EdFormerConfig(
 model_type=model_type, model_desc='dummy',
share_position_embeddings=True, use_tags=True, user_part=True,
use_prior_explanation=True, use_prior_question_elapsed_time_input=True,
use_lag_time=True,
use_lag_time_for_encoder=True,
use_user_answer=True 
)
 predictor = TFEdFormerAnswerPredictionModel(config)
 @tf.function
def foo(inputs):
logits, logits_2, c_outputs, r_outputs = predictor(inputs=inputs, output_attentions=True, output_hidden_states=True, return_dict=False)
 return logits
logits = foo(inputs)
 print(logits )
housing_all_df['GarageYrBlt'] = housing_all_df['YearBuilt'].where(pd.isnull(housing_all_df['GarageYrBlt']), housing_all_df['GarageYrBlt'])
 housing_all_df.loc[housing_all_df['GarageType'].isnull() &(housing_all_df['GarageArea'] == 0), 'GarageType'] = 'No Garage'
housing_all_df.loc[housing_all_df['GarageFinish'].isnull() &(housing_all_df['GarageArea'] == 0), 'GarageFinish'] = 'No Garage'
housing_all_df.loc[housing_all_df['GarageQual'].isnull() &(housing_all_df['GarageArea'] == 0), 'GarageQual'] = 'None'
housing_all_df.loc[housing_all_df['GarageCond'].isnull() &(housing_all_df['GarageArea'] == 0), 'GarageCond'] = 'None'<filter>
class LinearLR(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, total_steps, lr=1e-4, end_lr=1e-6, warmup_steps=WARMUP_STEPS):
self.total_steps = tf.cast(total_steps, dtype=DTYPE)
 self.lr = lr
self.end_lr = end_lr
self.warmup_steps = tf.cast(warmup_steps, dtype=DTYPE)
 def __call__(self, step):
is_warmup = tf.cast(step < self.warmup_steps, dtype=DTYPE)
 warmup_lr = is_warmup * self.lr *(step + 1)/ self.warmup_steps
_lr = tf.math.minimum(self.lr, 1 * self.end_lr)
 decay_lr =(1 - is_warmup)*(_lr -(_lr - self.end_lr)/ tf.math.maximum(self.total_steps - self.warmup_steps, 1)*(step - self.warmup_steps + 1))
decay_lr =(1 - is_warmup)* tf.math.maximum(self.end_lr, decay_lr)
 lr = warmup_lr + decay_lr
return lr
class NoamLR(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, hidden_size, lr, end_lr, warmup_steps):
self.lr = tf.cast(lr, DTYPE)
 self.end_lr = tf.cast(end_lr, DTYPE)
 self.warmup_steps = tf.cast(warmup_steps, DTYPE)
 self.hidden_size = tf.cast(hidden_size, DTYPE)
 def __call__(self, step):
scaling = self.lr /(self.hidden_size**-0.5 * self.warmup_steps**-0.5)
 lr = scaling * self.hidden_size**-0.5 * tf.math.minimum(( step + 1)* self.warmup_steps**-1.5,(step + 1)**-0.5)
 is_warmup = tf.cast(step < self.warmup_steps, dtype=DTYPE)
 lr = lr * is_warmup +(1.0 - is_warmup)* tf.math.maximum(self.end_lr, lr)
 return lr
housing_all_df[(housing_all_df['GarageQual'].isnull())|(housing_all_df['GarageCond'].isnull())][['GarageCars','GarageArea',
'GarageType','GarageFinish','GarageQual','GarageCond']]<feature_engineering>
def get_input_signatures(batch_size, seq_len, valid=False):
input_signatures = {
'user_id': tf.TensorSpec(shape=[batch_size], dtype=tf.int64, name='user_id'),
'seq_len': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='seq_len'),
'prev_seq_len': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='prev_seq_len'),
'start': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='start'),
'end': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='end'),
'row_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='row_id'),
'timestamp': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int64, name='timestamp'),
'content_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='content_id'),
'content_type_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='content_type_id'),
'task_container_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='task_container_id'),
'user_answer': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='user_answer'),
'shifted_user_answer': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_user_answer'),
'answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='answered_correctly'),
'shifted_answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_answered_correctly'),
'prior_question_elapsed_time': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='prior_question_elapsed_time'),
'prior_question_elapsed_time_input': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='prior_question_elapsed_time_input'),
'prior_question_had_explanation': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='prior_question_had_explanation'),
'pred_time_mask': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='pred_time_mask'),
'abs_pos': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='abs_pos'),
'shifted_abs_pos': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_abs_pos'),
'pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='pos_ids'),
'shifted_pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='shifted_pos_ids'),
'abs_pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='abs_pos_ids'),
'shifted_abs_pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='shifted_abs_pos_ids'),
'task_container_pos_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='task_container_pos_ids'),
'c_input_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='c_input_ids'),
'r_input_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='r_input_ids'),
'd_input_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='d_input_ids'),
'd_ans_input_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='d_ans_input_ids'),
'tag_ids': tf.TensorSpec(shape=[batch_size, seq_len, N_TAGS_PER_CONTENT], dtype=tf.int32, name='tag_ids'),
'part_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='part_ids'),
'prior_explanation_ids': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='prior_explanation_ids'),
'lag_time': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='lag_time'),
'n_questions_answered': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_questions_answered'),
'n_lectures_watched': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_lectures_watched'),
'n_questions_answered_correctly': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_questions_answered_correctly'),
'n_questions_answered_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='n_questions_answered_scaled'),
'n_lectures_watched_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='n_lectures_watched_scaled'),
'answered_correctly_ratio': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='answered_correctly_ratio'),
'part_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len, PART_VOCAB_SIZE - 2], dtype=DTYPE, name='part_count_scaled'),
'part_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len, PART_VOCAB_SIZE - 2], dtype=DTYPE, name='part_correctness_ratio'),
'current_part_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_correccurrent_part_count_scaledt_answer_count_scaled'),
'current_part_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_part_correctness_ratio'),
'correct_answer_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len, ANSWER_3_ID - ANSWER_0_ID + 1], dtype=DTYPE, name='correct_answer_count_scaled'),
'correct_answer_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len, ANSWER_3_ID - ANSWER_0_ID + 1], dtype=DTYPE, name='correct_answer_correctness_ratio'),
'current_correct_answer_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_correct_answer_count_scaled'),
'current_correct_answer_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_correct_answer_correctness_ratio'),
'n_prev_seen': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_prev_seen'),
'n_prev_correctness': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='n_prev_correctness'),
'current_question_count_scaled': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_question_count_scaled'),
'current_question_correctness_ratio': tf.TensorSpec(shape=[batch_size, seq_len], dtype=DTYPE, name='current_question_correctness_ratio'),
'correct_answer_id': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='correct_answer_id'),
'target': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='target'),
'answer_target': tf.TensorSpec(shape=[batch_size, seq_len], dtype=tf.int32, name='answer_target'),
'nb_pred_places': tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='nb_pred_places'),
}
if valid:
input_signatures['n_valid_blocks'] = tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='n_valid_blocks')
 input_signatures['valid_block_idx'] = tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='valid_block_idx')
 input_signatures['valid_start'] = tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='valid_start')
 input_signatures['valid_end'] = tf.TensorSpec(shape=[batch_size], dtype=tf.int32, name='valid_end')
 return [input_signatures]
housing_all_df.loc[housing_all_df.index==2576, 'GarageCars']= 0
housing_all_df.loc[housing_all_df.index==2576, 'GarageArea']= 0
housing_all_df.loc[housing_all_df.index==2576, 'GarageType']= 'No Garage'
housing_all_df.loc[housing_all_df.index==2576, 'GarageFinish']= 'No Garage'
housing_all_df.loc[housing_all_df.index==2576, 'GarageQual']= 'None'
housing_all_df.loc[housing_all_df.index==2576, 'GarageCond']= 'None'<feature_engineering>
class TrainConfig:
def __init__(
 self,
ckpt_path=CKPT_TRAIN_PATH,
window_size=WINDOW_SIZE,
loss_weight_window_size=LOSS_WEIGHT_WINDOW_SIZE,
n_epochs=N_EPOCHS,
batch_size=BATCH_SIZE,
pred_batch_size=PRED_BATCH_SIZE,
shuffle_buf_size=SHUFFLE_BUFFER_SIZE,
seed=SEED,
deterministic=DETERMINISTIC,
num_parallel_reads=N_PARALLEL_READS,
num_parallel_calls=N_PARALLEL_CALLS,
steps_per_call=STEPS_PER_CALL,
max_n_contents_per_user_for_sampling_prob=MAX_N_CONTENTS_PER_USER_FOR_SAMPLING_PROB,
valid_fold=VALID_FOLD 
):
self.ckpt_path = ckpt_path
self.window_size = window_size
self.loss_weight_window_size = loss_weight_window_size
self.n_epochs = n_epochs
self.batch_size = batch_size
self.pred_batch_size = pred_batch_size
self.shuffle_buf_size = shuffle_buf_size
self.seed=seed
self.deterministic = deterministic
self.num_parallel_reads = num_parallel_reads
self.num_parallel_calls = num_parallel_calls
self.steps_per_call = steps_per_call
self.max_n_contents_per_user_for_sampling_prob = max_n_contents_per_user_for_sampling_prob
self.valid_fold = valid_fold
n_contents_dict = load_data(n_contents_dict_path)
 n_training_examples = 0
for k, v in n_contents_dict.items() :
n_training_examples += math.ceil(v * 1.0 / self.window_size)
 del n_contents_dict
if self.valid_fold not in [0, 1, 2, 3]:
valid_info = {}
n_valid_examples = 0
else:
valid_info = load_data(valid_info_paths[self.valid_fold])
 n_valid_examples = 0
for k, v in valid_info.items() :
n_valid_examples += v['bundle_info']['n_blocks']
del valid_info
self.n_training_examples = n_training_examples
n_training_steps_per_epoch = 1 * self.n_training_examples // self.batch_size
self.n_training_calls_per_epoch = n_training_steps_per_epoch // self.steps_per_call
self.n_training_steps_per_epoch = self.n_training_calls_per_epoch * self.steps_per_call
self.n_training_steps = self.n_epochs * self.n_training_steps_per_epoch
self.n_valid_examples = n_valid_examples
if IS_KAGGLE and tpu is not None:
self.n_valid_examples -= self.n_valid_examples % self.pred_batch_size
self.n_valid_steps = self.n_valid_examples // self.pred_batch_size + int(self.n_valid_examples % self.pred_batch_size > 0)
 self.n_steps_in_last_valid_call = self.n_valid_steps % self.steps_per_call
self.n_valid_calls = self.n_valid_steps // self.steps_per_call + int(self.n_steps_in_last_valid_call > 0)
 self.lr = None
self.end_lr = None
self.warmup_steps = None
def toJSON(self):
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=False, indent=4)
 class Train_Manager:
def __init__(self, config, train_config):
self.config = config
self.train_config = train_config
n_contents_dict = load_data(n_contents_dict_path)
 user_id_list = []
n_contents_list = []
if self.train_config.valid_fold in [0, 1, 2, 3]:
train_valid_split_indices = load_data(train_valid_split_indices_paths[self.train_config.valid_fold])
 self.valid_user_ids = list(train_valid_split_indices.keys())
 else:
train_valid_split_indices = {}
self.valid_user_ids = []
for user_id, n_contents_orig in n_contents_dict.items() :
if self.train_config.valid_fold in [0, 1, 2, 3] and user_id in train_valid_split_indices:
split_index = train_valid_split_indices[user_id]
else:
split_index = n_contents_orig
n_contents_in_train = split_index
user_id_list.append(user_id)
 if n_contents_in_train <= 32:
n_contents_in_train = 32
n_contents_list.append(n_contents_in_train)
 user_id_tensor = tf.constant(user_id_list, dtype=tf.int64)
 n_contents_tensor = tf.constant(n_contents_list, dtype=tf.int32)
 n_contents_tensor = tf.math.minimum(n_contents_tensor, self.train_config.max_n_contents_per_user_for_sampling_prob)
 sample_prob_tensor = tf.cast(n_contents_tensor, dtype=tf.float32)/ tf.cast(tf.math.reduce_sum(n_contents_tensor), dtype=tf.float32)
 sample_prob_tensor = 0.9 * sample_prob_tensor / tf.math.reduce_max(sample_prob_tensor)
 del user_id_list
del n_contents_list
del n_contents_dict
initializer = tf.lookup.KeyValueTensorInitializer(user_id_tensor, sample_prob_tensor)
 training_sample_prob_table = tf.lookup.StaticHashTable(initializer, default_value=0.0)
 self.training_sample_prob_table = training_sample_prob_table
self.train_ds = strategy.experimental_distribute_dataset(self.get_train_ds(from_valid=False))
self.train_ds_from_valid = strategy.experimental_distribute_dataset(self.get_train_ds(from_valid=True))
self.train_ds_from_valid_only_last = strategy.experimental_distribute_dataset(self.get_train_ds(from_valid=True, only_last=True))
self.valid_ds = strategy.experimental_distribute_dataset(self.get_valid_ds(debug=False))
self.valid_ds_debug = self.get_valid_ds(debug=True)
 def toJSON(self):
config = json.loads(self.config.toJSON())
 train_config = json.loads(self.train_config.toJSON())
 for k, v in train_config.items() :
config[k] = v
return json.dumps(config, sort_keys=False, ensure_ascii=False, indent=4)
 def get_train_ds(self, from_valid=False, only_last=False):
only_last = False
train_raw_ds = tf.data.TFRecordDataset(train_tfrec_paths, num_parallel_reads=self.train_config.num_parallel_reads)
 train_raw_ds = train_raw_ds.map(parse_train_example, num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 train_raw_ds = train_raw_ds.map(lambda example: add_aggregated_historical_information(example), num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 if self.train_config.valid_fold in [0, 1, 2, 3]:
train_valid_split_indices = load_data(train_valid_split_indices_paths[self.train_config.valid_fold])
 train_valid_split_table = convert_split_index_dict(train_valid_split_indices)
 reduced_raw_ds = split_train_ds(train_raw_ds, train_valid_split_table, num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 else:
reduced_raw_ds = train_raw_ds
reduced_raw_ds_augmented = reduced_raw_ds.flat_map(
 lambda x: tf.data.Dataset.from_tensors(x ).repeat(
 tf.cast(
 tf.math.ceil(tf.cast(x['seq_len'], tf.float32)/ self.train_config.window_size),
dtype=tf.int64 
)
)
)
 batched_raw_ds = reduced_raw_ds_augmented.repeat().shuffle(self.train_config.shuffle_buf_size, seed=self.train_config.seed ).apply(tf.data.experimental.dense_to_ragged_batch(self.train_config.batch_size))
batched_ds = random_subseqs_from_batched_raw_ds(batched_raw_ds, window_size=self.train_config.window_size, only_last=only_last, seed=self.train_config.seed, num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 train_ds = prepare_training_dataset(batched_ds, generative=self.config.generative, use_abs_pos=self.config.use_abs_pos, num_parallel_calls=self.train_config.num_parallel_calls, deterministic=self.train_config.deterministic)
 train_ds = train_ds.prefetch(8)
 return train_ds
def get_valid_ds(self, debug=False):
p = valid_tfrec_paths
num_parallel_reads = self.train_config.num_parallel_reads
num_parallel_calls = self.train_config.num_parallel_calls
deterministic = self.train_config.deterministic
valid_user_id_tensor = tf.constant(self.valid_user_ids, dtype=tf.int64)
 repeat_tensor = tf.ones_like(valid_user_id_tensor, dtype=tf.int32)
 initializer = tf.lookup.KeyValueTensorInitializer(valid_user_id_tensor, repeat_tensor)
 valid_users_table = tf.lookup.StaticHashTable(
 initializer, default_value=tf.constant(0, dtype=tf.int32), name=None 
)
 valid_raw_ds = tf.data.TFRecordDataset(p, num_parallel_reads=num_parallel_reads)
 valid_raw_ds = valid_raw_ds.map(parse_train_example_with_valid_info, num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 valid_raw_ds.filter(lambda x: tf.math.equal(valid_users_table.lookup(x['user_id']), 1))
valid_ds = prepare_validation_dataset(valid_raw_ds, batch_size=self.train_config.pred_batch_size, window_size=self.train_config.window_size, generative=self.config.generative, use_abs_pos=self.config.use_abs_pos, num_parallel_calls=num_parallel_calls, deterministic=deterministic)
 valid_ds = valid_ds.prefetch(8)
 def get_train_objs(self, lr=LR, end_lr=END_LR, warmup_steps=WARMUP_STEPS):
self.train_config.lr = lr
self.train_config.end_lr = end_lr,
self.train_config.warmup_steps = warmup_steps
with strategy.scope() :
predictor = TFEdFormerAnswerPredictionModel(self.config)
 _lr = LinearLR(total_steps=self.train_config.n_training_steps, lr=lr, end_lr=end_lr, warmup_steps=warmup_steps)
 optimizer = tf.keras.optimizers.Adam(
 _lr,
beta_1=tf.Variable(0.9),
beta_2=tf.Variable(0.999),
epsilon=tf.Variable(1e-8)
)
 optimizer.iterations
optimizer.decay = tf.Variable(0.0)
 if self.config.use_softmax:
loss_obj = tf.keras.losses.SparseCategoricalCrossentropy(
 from_logits=True, reduction=tf.keras.losses.Reduction.NONE,
name='sparse_categorical_crossentropy_for_correctness' 
)
 else:
loss_obj = tf.keras.losses.BinaryCrossentropy(
 from_logits=True, label_smoothing=0, reduction=tf.keras.losses.Reduction.NONE,
name='binary_crossentropy_for_correctness' 
)
 loss_obj_answer = tf.keras.losses.SparseCategoricalCrossentropy(
 from_logits=True, reduction=tf.keras.losses.Reduction.NONE,
name='sparse_categorical_crossentropy_for_answer' 
)
 loss_metric = tf.keras.metrics.Sum()
acc_metric = tf.keras.metrics.BinaryAccuracy()
auc_metric = tf.keras.metrics.AUC(num_thresholds=2000)
 loss_metric_answer = tf.keras.metrics.Sum()
acc_metric_answer = tf.keras.metrics.SparseCategoricalAccuracy()
metrics =(loss_metric, acc_metric, auc_metric, loss_metric_answer, acc_metric_answer)
 return(predictor, optimizer, loss_obj, loss_obj_answer, metrics)
 def train_valid(
 self, predictor, optimizer, ckpt_manager,
loss_obj, loss_obj_answer, metrics,
last_epoch=0,
from_valid=False, valid_epochs=None, only_valid=False 
):
use_user_answer_loss = tf.cast(tf.constant(self.config.use_user_answer_loss), dtype=DTYPE)
 batch_size = self.train_config.batch_size
seq_len = self.train_config.window_size
encoder_decoder = tf.constant(self.config.model_type=='ed', dtype=tf.int32)
 generative = tf.constant(self.config.generative, dtype=tf.int32)
 allow_bundle_atten = tf.constant(self.config.allow_bundle_atten, dtype=tf.int32)
 loss_metric, acc_metric, auc_metric, loss_metric_answer, acc_metric_answer = metrics
@tf.function(
 input_signature=get_input_signatures(int(batch_size / strategy.num_replicas_in_sync), seq_len, valid=False)
)
 def train_step(train_batch):
training = tf.constant(1, dtype=tf.int32)
 c_mask, r_mask, r_c_mask, c_r_mask = get_attention_masks(train_batch, training, encoder_decoder, generative, allow_bundle_atten)
 targets = train_batch['target']
answer_targets = train_batch['answer_target']
pred_mask = targets != NON_TARGET_ID
pred_indices = tf.where(pred_mask)
 selected_targets = tf.gather_nd(targets, pred_indices)
 selected_answer_targets = tf.gather_nd(answer_targets, pred_indices)
 with tf.GradientTape() as tape: 
(logits, answer_logits, c_outputs, r_outputs)= predictor(
 train_batch,
c_mask,
r_mask,
r_c_mask,
c_r_mask,
output_attentions=False, output_hidden_states=False, training=True 
)
 selected_logits = tf.gather_nd(logits, pred_indices)
 selected_answer_logits = tf.gather_nd(answer_logits, pred_indices)
 losses = loss_obj(selected_targets[:, tf.newaxis], selected_logits)
 answer_losses = loss_obj_answer(selected_answer_targets, selected_answer_logits)
 total_loss = tf.math.reduce_sum(losses)
 total_answer_loss = tf.math.reduce_sum(answer_losses)
 loss =(total_loss + total_answer_loss * use_user_answer_loss)/ tf.cast(train_batch['nb_pred_places'][0], dtype=DTYPE)
 grads = tape.gradient(loss, predictor.trainable_variables)
 optimizer.apply_gradients(zip(grads, predictor.trainable_variables))
if self.config.use_softmax:
preds = tf.math.softmax(logits)
 selected_preds = tf.math.softmax(selected_logits)
 else:
preds = tf.math.sigmoid(logits)
 selected_preds = tf.math.sigmoid(selected_logits)
 answer_preds = tf.math.softmax(answer_logits, axis=-1)
 selected_answer_preds = tf.math.softmax(selected_answer_logits, axis=-1)
 loss_metric.update_state(total_loss)
 acc_metric.update_state(selected_targets[:, tf.newaxis], selected_preds[:, -1:])
 auc_metric.update_state(selected_targets, selected_preds[:, -1])
 loss_metric_answer.update_state(total_answer_loss)
 acc_metric_answer.update_state(selected_answer_targets, selected_answer_preds)
 batch_size = None
@tf.function(
 input_signature=get_input_signatures(batch_size, seq_len, valid=True)
)
 def valid_step(valid_batch):
training = tf.constant(0, dtype=tf.int32)
 c_mask, r_mask, r_c_mask, c_r_mask = get_attention_masks(valid_batch, training, encoder_decoder, generative, allow_bundle_atten)
 logits, answer_logits, _, _ = predictor(
 valid_batch,
c_mask,
r_mask,
r_c_mask,
c_r_mask,
output_attentions=False, output_hidden_states=False, training=False 
)
 pred_time_mask = valid_batch['pred_time_mask'] * tf.cast(valid_batch['pred_time_mask'] != PAD_TOKEN, dtype=tf.int32)
 targets = valid_batch['target']
answer_targets = valid_batch['answer_target']
pred_time_mask = pred_time_mask[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:]
targets = targets[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:]
answer_targets = answer_targets[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:]
logits = logits[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:, :]
answer_logits = answer_logits[:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:, :]
row_ids = valid_batch['row_id'][:, -MAX_PRED_TIME_QUESTION_BUNDLE_LEN:]
user_ids = valid_batch['user_id'][:, tf.newaxis] * tf.ones_like(row_ids, dtype=tf.int64)
 targets = targets * pred_time_mask + NON_TARGET_ID *(1 - pred_time_mask)
 answer_targets = answer_targets * pred_time_mask + NON_TARGET_ID *(1 - pred_time_mask)
 pred_mask = targets != NON_TARGET_ID
pred_indices = tf.where(pred_mask)
 selected_targets = tf.gather_nd(targets, pred_indices)
 selected_logits = tf.gather_nd(logits, pred_indices)
 if self.config.use_softmax:
preds = tf.math.softmax(logits)
 selected_preds = tf.math.softmax(selected_logits)
 else:
preds = tf.math.sigmoid(logits)
 selected_preds = tf.math.sigmoid(selected_logits)
 answer_preds = tf.math.softmax(answer_logits, axis=-1)
 selected_answer_targets = tf.gather_nd(answer_targets, pred_indices)
 selected_answer_logits = tf.gather_nd(answer_logits, pred_indices)
 selected_answer_preds = tf.math.softmax(selected_answer_logits, axis=-1)
 losses = loss_obj(selected_targets[:, tf.newaxis], selected_logits)
 answer_losses = loss_obj_answer(selected_answer_targets, selected_answer_logits)
 total_loss = tf.math.reduce_sum(losses)
 total_answer_loss = tf.math.reduce_sum(answer_losses)
 loss =(total_loss + total_answer_loss * use_user_answer_loss)/ tf.cast(tf.math.reduce_mean(valid_batch['nb_pred_places']), dtype=DTYPE)
 loss_metric.update_state(total_loss)
 acc_metric.update_state(selected_targets[:, tf.newaxis], selected_preds[:, -1:])
 auc_metric.update_state(selected_targets, selected_preds[:, -1])
 loss_metric_answer.update_state(total_answer_loss)
 acc_metric_answer.update_state(selected_answer_targets, selected_answer_preds)
 return targets, preds, row_ids, user_ids, pred_mask, answer_targets, answer_preds
@tf.function
def dist_train_step(dist_train_batch):
strategy.run(train_step, args=(dist_train_batch,))
@tf.function
def dist_valid_step(dist_valid_batch):
targets, preds, row_ids, user_ids, pred_mask, answer_targets, answer_preds = strategy.run(valid_step, args=(dist_valid_batch,))
return targets, preds, row_ids, user_ids, pred_mask, answer_targets, answer_preds
@tf.function
def dist_train_multi_steps(dist_train_iter):
total_nb_pred_places = tf.constant(0.0, dtype=DTYPE)
 for _ in tf.range(self.train_config.steps_per_call):
dist_train_batch = next(dist_train_iter)
 dist_train_step(dist_train_batch)
 if type(dist_train_batch['nb_pred_places'])== tf.Tensor:
nb_pred_places = tf.math.reduce_mean(dist_train_batch['nb_pred_places'])
 else:
nb_pred_places = tf.math.reduce_mean(tf.concat(dist_train_batch['nb_pred_places'].values, axis=0))
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 return total_nb_pred_places
@tf.function
def dist_valid_multi_steps(dist_valid_iter):
total_nb_pred_places = tf.constant(0.0, dtype=DTYPE)
 t_arr_1 = tf.TensorArray(tf.int32, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_2 = tf.TensorArray(DTYPE, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_3 = tf.TensorArray(tf.bool, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_4 = tf.TensorArray(tf.int64, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_5 = tf.TensorArray(tf.int64, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_6 = tf.TensorArray(tf.int32, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_7 = tf.TensorArray(DTYPE, size=0, dynamic_size=True, clear_after_read=False)
 for step_idx in tf.range(self.train_config.steps_per_call):
dist_valid_batch = next(dist_valid_iter)
 _targets, _preds, _row_ids, _user_ids, _pred_mask, _answer_targets, _answer_preds = dist_valid_step(dist_valid_batch)
 if type(dist_valid_batch['nb_pred_places'])== tf.Tensor:
nb_pred_places = tf.math.reduce_mean(dist_valid_batch['nb_pred_places'])
 else:
_targets = tf.concat(_targets.values, axis=0)
 _preds = tf.concat(_preds.values, axis=0)
 _pred_mask = tf.concat(_pred_mask.values, axis=0)
 _row_ids = tf.concat(_row_ids.values, axis=0)
 _user_ids = tf.concat(_user_ids.values, axis=0)
 _answer_targets = tf.concat(_answer_targets.values, axis=0)
 _answer_preds = tf.concat(_answer_preds.values, axis=0)
 nb_pred_places = tf.math.reduce_mean(tf.concat(dist_valid_batch['nb_pred_places'].values, axis=0))
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 t_arr_1 = t_arr_1.write(step_idx, _targets)
 t_arr_2 = t_arr_2.write(step_idx, _preds)
 t_arr_3 = t_arr_3.write(step_idx, _pred_mask)
 t_arr_4 = t_arr_4.write(step_idx, _row_ids)
 t_arr_5 = t_arr_5.write(step_idx, _user_ids)
 t_arr_6 = t_arr_6.write(step_idx, _answer_targets)
 t_arr_7 = t_arr_7.write(step_idx, _answer_preds)
 targets = t_arr_1.concat()
preds = t_arr_2.concat()
pred_mask = t_arr_3.concat()
row_ids = t_arr_4.concat()
user_ids = t_arr_5.concat()
answer_targets = t_arr_6.concat()
answer_preds = t_arr_7.concat()
return targets, preds, row_ids, user_ids, pred_mask, total_nb_pred_places, answer_targets, answer_preds
@tf.function
def dist_valid_multi_steps_last_call(dist_valid_iter):
total_nb_pred_places = tf.constant(0.0, dtype=DTYPE)
 t_arr_1 = tf.TensorArray(tf.int32, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_2 = tf.TensorArray(DTYPE, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_3 = tf.TensorArray(tf.bool, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_4 = tf.TensorArray(tf.int64, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_5 = tf.TensorArray(tf.int64, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_6 = tf.TensorArray(tf.int32, size=0, dynamic_size=True, clear_after_read=False)
 t_arr_7 = tf.TensorArray(DTYPE, size=0, dynamic_size=True, clear_after_read=False)
 for step_idx in tf.range(self.train_config.n_steps_in_last_valid_call):
dist_valid_batch = next(dist_valid_iter)
 _targets, _preds, _row_ids, _user_ids, _pred_mask, _answer_targets, _answer_preds = dist_valid_step(dist_valid_batch)
 if type(dist_valid_batch['nb_pred_places'])== tf.Tensor:
nb_pred_places = tf.math.reduce_mean(dist_valid_batch['nb_pred_places'])
 else:
_targets = tf.concat(_targets.values, axis=0)
 _preds = tf.concat(_preds.values, axis=0)
 _pred_mask = tf.concat(_pred_mask.values, axis=0)
 _row_ids = tf.concat(_row_ids.values, axis=0)
 _user_ids = tf.concat(_user_ids.values, axis=0)
 _answer_targets = tf.concat(_answer_targets.values, axis=0)
 _answer_preds = tf.concat(_answer_preds.values, axis=0)
 nb_pred_places = tf.math.reduce_mean(tf.concat(dist_valid_batch['nb_pred_places'].values, axis=0))
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 t_arr_1 = t_arr_1.write(step_idx, _targets)
 t_arr_2 = t_arr_2.write(step_idx, _preds)
 t_arr_3 = t_arr_3.write(step_idx, _pred_mask)
 t_arr_4 = t_arr_4.write(step_idx, _row_ids)
 t_arr_5 = t_arr_5.write(step_idx, _user_ids)
 t_arr_6 = t_arr_6.write(step_idx, _answer_targets)
 t_arr_7 = t_arr_7.write(step_idx, _answer_preds)
 targets = t_arr_1.concat()
preds = t_arr_2.concat()
pred_mask = t_arr_3.concat()
row_ids = t_arr_4.concat()
user_ids = t_arr_5.concat()
answer_targets = t_arr_6.concat()
answer_preds = t_arr_7.concat()
return targets, preds, row_ids, user_ids, pred_mask, total_nb_pred_places, answer_targets, answer_preds
n_training_steps_per_epoch = self.train_config.n_training_steps_per_epoch
n_training_steps = self.train_config.n_epochs * n_training_steps_per_epoch
if MAX_TRAIN_ITER_STEPS is not None:
n_training_steps_per_epoch = MAX_TRAIN_ITER_STEPS
n_training_steps = self.train_config.n_epochs * n_training_steps_per_epoch
self.train_config.n_training_steps_per_epoch = n_training_steps_per_epoch
self.train_config.n_training_steps = n_training_steps
with open('train_config.json', 'w', encoding='UTF-8')as fp:
json.dump(json.loads(self.toJSON()), fp, ensure_ascii=False, indent=4)
 if not IS_KAGGLE:
!gsutil cp -r './train_config.json' '{self.train_config.ckpt_path}'
n_epochs = self.train_config.n_epochs
n_training_examples = self.train_config.n_training_examples
if from_valid:
n_epochs = 3
n_training_examples = int(2380588 / self.train_config.window_size)
 n_training_steps_per_epoch = int(n_training_examples / self.train_config.batch_size)
 n_training_steps = n_epochs * n_training_steps_per_epoch
print(f'n_epochs: {n_epochs}')
 print(f'n_training_examples: {n_training_examples}')
 print(f'n_training_steps_per_epoch: {n_training_steps_per_epoch}')
 print(f'n_training_steps: {n_training_steps}')
 print(f'n_valid_examples: {self.train_config.n_valid_examples}')
 print(f'n_valid_steps: {self.train_config.n_valid_steps}')
 training_history = dict()
def train(last_epoch=0, from_valid=False, valid_epochs=None):
start_epoch = last_epoch + 1
end_epoch = self.train_config.n_epochs + 1
train_ds = self.train_ds
if from_valid:
end_epoch = start_epoch + 3
train_ds = self.train_ds_from_valid
dist_train_iter = iter(train_ds)
 for epoch in range(start_epoch, end_epoch):
training_history[epoch] = {
'loss': [],
'acc': [],
'auc': [],
'answer_loss': [],
'answer_acc': [],
'timing': []
}
n_steps = 0
n_preds = 0
n_last_preds = 0
total_nb_pred_places = 0.0
start_epoch_t = datetime.datetime.now()
for call_idx in range(self.train_config.n_training_calls_per_epoch):
start_t = datetime.datetime.now()
nb_pred_places = dist_train_multi_steps(dist_train_iter)
 n_steps += self.train_config.steps_per_call
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 avg_loss =(loss_metric.result() / total_nb_pred_places ).numpy()
acc = acc_metric.result().numpy()
auc = auc_metric.result().numpy()
avg_answer_loss =(loss_metric_answer.result() / total_nb_pred_places ).numpy()
answer_acc = acc_metric_answer.result().numpy()
if(call_idx + 1)% max(1, self.train_config.n_training_calls_per_epoch // 100)== 0:
training_history[epoch]['loss'].append(float(avg_loss))
training_history[epoch]['acc'].append(float(acc))
training_history[epoch]['auc'].append(float(auc))
training_history[epoch]['answer_loss'].append(float(avg_answer_loss))
training_history[epoch]['answer_acc'].append(float(answer_acc))
elapsed =(datetime.datetime.now() - start_t ).total_seconds()
if(call_idx + 1)% max(1, self.train_config.n_training_calls_per_epoch // 10)== 0:
print(f'epoch: {epoch} - step: {n_steps}')
 print(f'timing per step: {elapsed / self.train_config.steps_per_call}')
 print(f'loss: {avg_loss}')
 print(f'acc: {acc}')
 print(f'auc: {auc}')
 print(f'answer_loss: {avg_answer_loss}')
 print(f'answer_acc: {answer_acc}')
 print(f'lr: {optimizer._decayed_lr(DTYPE)}')
 print('-' * 80)
 end_epoch_t = datetime.datetime.now()
elapsed_epoch_t =(end_epoch_t - start_epoch_t ).total_seconds()
training_history[epoch]['timing'] = elapsed_epoch_t
training_history[epoch]['train_loss'] = float(avg_loss)
 training_history[epoch]['train_acc'] = float(acc)
 training_history[epoch]['train_auc'] = float(auc)
 training_history[epoch]['train_answer_loss'] = float(avg_answer_loss)
 training_history[epoch]['train_answer_acc'] = float(answer_acc)
 if IS_KAGGLE and tpu:
ckpt_manager.checkpoint.save(
 file_prefix=ckpt_manager.directory + 'ckpt', options=tf.train.CheckpointOptions(experimental_io_device="/job:localhost")
)
 else:
ckpt_manager.save()
print(f'epoch: {epoch} - train')
 print(f'train loss: {avg_loss}')
 print(f'train acc: {acc}')
 print(f'train auc: {auc}')
 print(f'train answer loss: {avg_answer_loss}')
 print(f'train answer acc: {answer_acc}')
 print('-' * 80)
 loss_metric.reset_states()
acc_metric.reset_states()
auc_metric.reset_states()
loss_metric_answer.reset_states()
acc_metric_answer.reset_states()
fn = f'training_history-{epoch}.json'
with open(fn, 'w', encoding='UTF-8')as fp:
json.dump(training_history, fp, ensure_ascii=False, indent=4)
 if not IS_KAGGLE:
!gsutil cp -r './{fn}' '{self.train_config.ckpt_path}'
!rm -rf './{fn}'
if valid_epochs is None:
valid_epochs = [self.train_config.n_epochs]
if epoch in valid_epochs:
valid(epoch)
 def valid(epoch):
if epoch not in training_history:
training_history[epoch] = {
'loss': [],
'acc': [],
'auc': [],
'answer_loss': [],
'answer_acc': [],
'timing': []
}
valid_user_ids = []
valid_row_ids = []
valid_targets = []
valid_preds = []
valid_answer_targets = []
valid_answer_preds = []
n_steps = 0
total_nb_pred_places = 0.0
dist_valid_iter = iter(self.valid_ds)
 start_valid = datetime.datetime.now()
n_valid_calls = self.train_config.n_valid_calls
if self.train_config.n_steps_in_last_valid_call > 0:
n_valid_calls -= 1
for call_idx in range(n_valid_calls):
start_t = datetime.datetime.now()
targets, preds, row_ids, user_ids, pred_mask, nb_pred_places, answer_targets, answer_preds = dist_valid_multi_steps(dist_valid_iter)
 n_steps += self.train_config.steps_per_call
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 pred_indices = tf.where(pred_mask)
 selected_targets = tf.gather_nd(targets, pred_indices)
 selected_preds = tf.gather_nd(preds, pred_indices)
 selected_answer_targets = tf.gather_nd(answer_targets, pred_indices)
 selected_answer_preds = tf.gather_nd(answer_preds, pred_indices)
 selected_row_ids = tf.gather_nd(row_ids, pred_indices)
 selected_user_ids = tf.gather_nd(user_ids, pred_indices)
 avg_loss =(loss_metric.result() / total_nb_pred_places ).numpy()
acc = acc_metric.result().numpy()
auc = auc_metric.result().numpy()
avg_answer_loss =(loss_metric_answer.result() / total_nb_pred_places ).numpy()
answer_acc = acc_metric_answer.result().numpy()
elapsed =(datetime.datetime.now() - start_t ).total_seconds()
if call_idx % 1 == 0:
print(f'epoch: {epoch} - valid step: {n_steps}')
 print(f'valid timing per steps: {elapsed / self.train_config.steps_per_call}')
 print(f'valid loss: {avg_loss}')
 print(f'valid acc: {acc}')
 print(f'valid auc: {auc}')
 print(f'valid answer loss: {avg_answer_loss}')
 print(f'valid answer acc: {answer_acc}')
 print('-' * 80)
 valid_row_ids.extend(selected_row_ids.numpy().tolist())
 valid_user_ids.extend(selected_user_ids.numpy().tolist())
 valid_targets.extend(selected_targets.numpy().tolist())
 valid_preds.extend(selected_preds[:, -1].numpy().tolist())
 valid_answer_targets.extend(selected_answer_targets.numpy().tolist())
 valid_answer_preds.extend(selected_answer_preds.numpy().tolist())
 if self.train_config.n_steps_in_last_valid_call > 0:
start_t = datetime.datetime.now()
targets, preds, row_ids, user_ids, pred_mask, nb_pred_places, answer_targets, answer_preds = dist_valid_multi_steps_last_call(dist_valid_iter)
 n_steps += self.train_config.n_steps_in_last_valid_call
total_nb_pred_places += tf.cast(nb_pred_places, dtype=DTYPE)
 pred_indices = tf.where(pred_mask)
 selected_targets = tf.gather_nd(targets, pred_indices)
 selected_preds = tf.gather_nd(preds, pred_indices)
 selected_answer_targets = tf.gather_nd(answer_targets, pred_indices)
 selected_answer_preds = tf.gather_nd(answer_preds, pred_indices)
 selected_row_ids = tf.gather_nd(row_ids, pred_indices)
 selected_user_ids = tf.gather_nd(user_ids, pred_indices)
 avg_loss =(loss_metric.result() / total_nb_pred_places ).numpy()
acc = acc_metric.result().numpy()
auc = auc_metric.result().numpy()
avg_answer_loss =(loss_metric_answer.result() / total_nb_pred_places ).numpy()
answer_acc = acc_metric_answer.result().numpy()
elapsed =(datetime.datetime.now() - start_t ).total_seconds()
valid_row_ids.extend(selected_row_ids.numpy())
 valid_user_ids.extend(selected_user_ids.numpy())
 valid_preds.extend(selected_preds[:, -1].numpy())
 valid_answer_targets.extend(selected_answer_targets.numpy())
 valid_answer_preds.extend(selected_answer_preds.numpy())
 end_valid = datetime.datetime.now()
elapsed_valid =(end_valid - start_valid ).total_seconds()
training_history[epoch]['valid_timing'] = elapsed_valid
training_history[epoch]['valid_loss'] = float(avg_loss)
 training_history[epoch]['valid_acc'] = float(acc)
 training_history[epoch]['valid_auc'] = float(auc)
 training_history[epoch]['valid_answer_loss'] = float(avg_answer_loss)
 training_history[epoch]['valid_answer_acc'] = float(answer_acc)
 print(f'epoch: {epoch} - valid')
 print(f'valid loss: {avg_loss}')
 print(f'valid acc: {acc}')
 print(f'valid auc: {auc}')
 print(f'valid answer loss: {avg_answer_loss}')
 print(f'valid answer acc: {answer_acc}')
 print('=' * 80)
 loss_metric.reset_states()
acc_metric.reset_states()
auc_metric.reset_states()
loss_metric_answer.reset_states()
acc_metric_answer.reset_states()
valid_submission_dict = {
'row_ids': valid_row_ids,
'user_ids': valid_user_ids,
'targets': valid_targets,
'preds': valid_preds
}
if self.config.use_user_answer_loss:
valid_answer_preds = np.array(valid_answer_preds)
 valid_submission_dict['answer_targets'] = valid_answer_targets
valid_submission_dict['answer_preds_0'] = valid_answer_preds[:, 0]
valid_submission_dict['answer_preds_1'] = valid_answer_preds[:, 1]
valid_submission_dict['answer_preds_2'] = valid_answer_preds[:, 2]
valid_submission_dict['answer_preds_3'] = valid_answer_preds[:, 3]
fn = f'training_history-{epoch}.json'
if only_valid:
fn = f'training_history-{epoch}-only-valid.json'
with open(fn, 'w', encoding='UTF-8')as fp:
json.dump(training_history, fp, ensure_ascii=False, indent=4)
 if not IS_KAGGLE:
!gsutil cp -r './{fn}' '{self.train_config.ckpt_path}'
!rm -rf './{fn}'
if only_valid:
valid(last_epoch)
 else:
train(last_epoch, from_valid, valid_epochs )
housing_all_df.loc[housing_all_df['GarageFinish'].isnull() ,'GarageFinish'] = housing_all_df['GarageFinish'].mode().values[0]
housing_all_df.loc[housing_all_df['GarageCond'].isnull() ,'GarageCond'] = housing_all_df['GarageCond'].mode().values[0]
housing_all_df.loc[housing_all_df['GarageQual'].isnull() ,'GarageQual'] = housing_all_df['GarageQual'].mode().values[0]<count_values>
if not IS_KAGGLE:
config = EdFormerConfig(model_type='ed', model_desc='dummy')
 train_config = TrainConfig()
train_manager = Train_Manager(config, train_config )
print("Null count: ", housing_all_df['Exterior1st'].isnull().sum())
 print(housing_all_df['Exterior1st'].value_counts())
 housing_all_df['Exterior1st'].fillna(value=housing_all_df['Exterior1st'].mode().values[0], inplace=True )<count_values>
if not IS_KAGGLE:
train_manager.train_ds
print("Null count: ",housing_all_df['Exterior2nd'].isnull().sum())
 print(housing_all_df['Exterior2nd'].value_counts())
 housing_all_df['Exterior2nd'].fillna(housing_all_df['Exterior2nd'].mode().values[0], inplace = True )<count_missing_values>
if not IS_KAGGLE:
train_manager.valid_ds
print("Null count: ", housing_all_df['MasVnrType'].isnull().sum())
 print(housing_all_df['MasVnrType'].value_counts())
 housing_all_df.loc[housing_all_df['MasVnrType'].isnull() & housing_all_df['MasVnrArea'].isnull() , 'MasVnrType'] = 'None'
housing_all_df.loc[housing_all_df['MasVnrType'].isnull() & housing_all_df['MasVnrArea'], 'MasVnrType'] = housing_all_df['MasVnrType'].mode().values[0]
housing_all_df['MasVnrArea'].fillna(value=0, inplace=True )<feature_engineering>
if not IS_KAGGLE:
train_manager.valid_ds_debug
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtFinSF1'] = 0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtFinSF2'] =0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtUnfSF'] =0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtFullBath'] =0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'BsmtHalfBath'] =0
housing_all_df.loc[housing_all_df['TotalBsmtSF'].isnull() ,'TotalBsmtSF'] =0<feature_engineering>
def convert_dt(_dt, pred=False, unique_question_id_train=None, unique_lecture_id_train=None):

 _dt[dt.f.prior_question_elapsed_time] = dt.float32
_dt[dt.f.prior_question_elapsed_time == None, 'prior_question_elapsed_time'] = -1.0
_dt[dt.f.prior_question_had_explanation] = dt.int8
_dt[dt.f.prior_question_had_explanation == None, 'prior_question_had_explanation'] = -1
_dt[dt.f.content_type_id] = dt.int8
if pred:
_dt['answered_correctly'] = MASK_TOKEN
_dt['user_answer'] = MASK_TOKEN
try:
del _dt['prior_group_answers_correct']
del _dt['prior_group_responses']
except:
pass
if DEBUG:
if unique_question_id_train is not None:
assert set(_dt[dt.f.content_type_id == 0, 'content_id'].to_list() [0] ).issubset(unique_question_id_train)
 if unique_lecture_id_train is not None:
assert set(_dt[dt.f.content_type_id == 1, 'content_id'].to_list() [0] ).issubset(unique_lecture_id_train)
 def convert_df_to_dt(df, test=False, unique_question_id_train=None, unique_lecture_id_train=None):

 if DEBUG:
_question_ids = set(df[df['content_type_id'] == 0]['content_id'].values.tolist())
 _lecture_ids = set(df[df['content_type_id'] == 1]['content_id'].values.tolist())
 if unique_question_id_train is not None:
assert _question_ids.issubset(unique_question_id_train)
 if unique_lecture_id_train is not None:
assert _lecture_ids.issubset(unique_lecture_id_train)
 _dt = dt.Frame(df.astype({"prior_question_had_explanation": float}))
if test:
prior_group_answers_correct = eval(_dt[0, 'prior_group_answers_correct'])
 prior_group_responses = eval(_dt[0, 'prior_group_responses'])
 if prior_group_answers_correct is None:
prior_group_answers_correct = []
if prior_group_responses is None:
prior_group_responses = []
if DEBUG:
assert type(prior_group_answers_correct)== list
assert type(prior_group_responses)== list
convert_dt(_dt, pred=test, unique_question_id_train=unique_question_id_train, unique_lecture_id_train=unique_lecture_id_train)
 if test:
return _dt, prior_group_answers_correct, prior_group_responses
else:
return _dt
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtQual'].isnull()), 'BsmtQual'] = 'None'
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtCond'].isnull()), 'BsmtCond'] = 'None'
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtExposure'].isnull()), 'BsmtExposure'] = 'None'
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtFinType1'].isnull()), 'BsmtFinType1'] = 'None'
housing_all_df.loc[(housing_all_df['TotalBsmtSF'] == 0)&(housing_all_df['BsmtFinType2'].isnull()), 'BsmtFinType2'] = 'None'<feature_engineering>
def sample_sequence_lengths(upper_bound, sample_size, expected_mean, seed):
seq_lengths = tf.range(1, upper_bound+1)
 seq_len_mean = tf.math.reduce_mean(tf.cast(seq_lengths, dtype=DTYPE))
N = sample_size
mu = expected_mean
sigma = mu / 2.5
normal_dist = tfp.distributions.Normal(
 loc=mu, scale=sigma 
)
 seq_len_sample = normal_dist.sample(sample_shape=[N], seed=seed)
 avg_selected_seq_len = tf.math.reduce_mean(seq_len_sample)
 mask = tf.math.logical_and(2 * mu >= seq_len_sample, seq_len_sample >= 1)
 seq_len_sample = tf.gather_nd(seq_len_sample, indices=tf.where(mask))
seq_len_sample = tf.cast(tf.math.round(seq_len_sample), dtype=tf.int32)
 uniform_dist = tfp.distributions.Uniform(
 low=0, high=len(seq_lengths)
)
 selected_indices = tf.cast(uniform_dist.sample(sample_shape=[N - seq_len_sample.shape[0]], seed=seed), dtype=tf.int32)
 seq_len_sample_2 = tf.gather(seq_lengths, indices=selected_indices)
 seq_len_sample_2 = seq_len_sample_2
seq_len_sample = tf.concat([seq_len_sample, seq_len_sample_2], axis=0)
 return seq_len_sample
housing_all_df.loc[housing_all_df['BsmtQual'].isnull() , 'BsmtQual'] = housing_all_df['BsmtQual'].mode().values[0]
housing_all_df.loc[housing_all_df['BsmtCond'].isnull() , 'BsmtCond'] = housing_all_df['BsmtCond'].mode().values[0]
housing_all_df.loc[housing_all_df['BsmtExposure'].isnull() , 'BsmtExposure'] = housing_all_df['BsmtExposure'].mode().values[0]
housing_all_df.loc[housing_all_df['BsmtFinType2'].isnull() , 'BsmtFinType2'] = housing_all_df['BsmtFinType2'].mode().values[0]
housing_all_df.loc[housing_all_df['BsmtFullBath'].isnull() , 'BsmtFullBath'] = 0
housing_all_df.loc[housing_all_df['BsmtHalfBath'].isnull() , 'BsmtHalfBath'] = 0<count_missing_values>
class User_Interaction_Manager(collections.abc.Iterator):
def __init__(self, user_id, user_valid_info, train_dt):
self.user_id = user_id
self.user_valid_info = user_valid_info
self._n_remaining_blocks = self.n_blocks
self._current_block_starting_index = 0
self._train_dt = train_dt
@property
def n_blocks(self):
return self.user_valid_info['bundle_info']['n_blocks']
@property
def n_remaining_blocks(self):
return self._n_remaining_blocks
def _get_next_block_starting_index(self):
if self._current_block_starting_index not in self.user_valid_info['bundle_info']['block_starting_index_dict']:
next_index = self._current_block_starting_index + 1
else:
next_index = self.user_valid_info['bundle_info']['block_starting_index_dict'][self._current_block_starting_index]
if DEBUG:
assert 0 <= next_index <= self.user_valid_info['seq_len']
return next_index
def _get_user_dt(self, next_block_starting_index):
row_start = self.user_valid_info['row_start']
block_row_start = row_start + self._current_block_starting_index
block_row_end = row_start + next_block_starting_index - 1
if DEBUG:
assert block_row_end <= self.user_valid_info['row_end']
user_dt = self._get_user_dt_from_row_indices(block_row_start, block_row_end)
 return user_dt
def _get_user_dt_from_row_indices(self, row_start, row_end):
user_dt = self._train_dt[row_start:(row_end + 1), :]
return user_dt
def has_next(self):
return not(self._n_remaining_blocks <= 0 or self._current_block_starting_index >= self.user_valid_info['seq_len'])
 def __next__(self):
if self._n_remaining_blocks <= 0:
raise StopIteration
if self._current_block_starting_index >= self.user_valid_info['seq_len']:
raise StopIteration
next_block_starting_index = self._get_next_block_starting_index()
user_dt = self._get_user_dt(next_block_starting_index)
 self._n_remaining_blocks -= 1
self._current_block_starting_index = next_block_starting_index
if DEBUG:
if self._n_remaining_blocks <= 0:
assert self._current_block_starting_index == self.user_valid_info['seq_len']
if self._current_block_starting_index >= self.user_valid_info['seq_len']:
assert self._n_remaining_blocks == 0
assert user_dt.nrows > 0
return user_dt
class Valid_Iter_Manager(Pred_Iter_Manager):
def __init__(self, valid_info, train_dt, unique_question_id_train=None, unique_lecture_id_train=None):
super(Valid_Iter_Manager, self ).__init__(unique_question_id_train, unique_lecture_id_train)
 self._valid_info = valid_info
self._n_users = len(set(self._valid_info.keys()))
_n_blocks = 0
for user_id, user_valid_info in self._valid_info.items() :
_n_blocks += user_valid_info['bundle_info']['n_blocks']
self._n_blocks = _n_blocks
self._remaining_users = set(self._valid_info.keys())
 self._n_remaining_blocks = self._n_blocks
self._current_answered_correctly = None
self._current_user_answer = None
self._user_manager_dict = dict()
for user_id, user_valid_info in self._valid_info.items() :
self._user_manager_dict[user_id] = User_Interaction_Manager(user_id, user_valid_info, train_dt)
 self.train_dt = train_dt
self._split_indices = {user_id: valid_info_for_user['split_index'] for user_id, valid_info_for_user in self._valid_info.items() }
@classmethod
def create(cls, valid_info_path, train_dt_path_or_obj, unique_question_id_train_path=None, unique_lecture_id_train_path=None):
args = []
if valid_info_path is None:
valid_info = {}
else:
valid_info = load_data(valid_info_path)
 args.append(valid_info)
 for p in [unique_question_id_train_path, unique_lecture_id_train_path]:
if p is not None:
args.append(load_data(p))
if type(train_dt_path_or_obj)== str:
train_dt_path_or_obj = load_data(train_dt_path_or_obj)
 args = args[:1] + [train_dt_path_or_obj] + args[1:]
return cls(*args)
 def get_split_index(self, user_id):
if user_id not in self._split_indices:
return -1
if DEBUG:
assert 0 <= self._split_indices[user_id] < self._valid_info[user_id]['orig_seq_len']
return self._split_indices[user_id]
def iter_dataset(self, max_steps=None):
print('entering iter_dataset() ')
 expected_mean_blocks_per_batch = 30.0
expected_n_batches = int(self._n_blocks / expected_mean_blocks_per_batch)+ 1
sampled_block_size = [30] * 100000
sample_mean_blocks_per_batch = 30.0
if DEBUG:
assert len(sampled_block_size)>= expected_n_batches
print(f'expected_mean_blocks_per_batch_in_pred: {expected_mean_blocks_per_batch}')
 print(f'sample_mean_blocks_per_batch_in_pred: {sample_mean_blocks_per_batch}')
 print(f'expected_n_batches_in_pred: {expected_n_batches}')
 total_blocks_yield = 0
print('start looping...')
 while True:
if max_steps is not None and self.current_batch_no >= max_steps:
break
try:
n_blocks_to_select = 30.0
if self.current_batch_no < len(sampled_block_size):
n_blocks_to_select = sampled_block_size[self.current_batch_no]
valid_df, valid_dt, prior_group_answers_correct, prior_group_responses, n_blocks_in_batch = self._step(n_blocks_to_select)
 if DEBUG:
assert n_blocks_in_batch > 0
total_blocks_yield += n_blocks_in_batch
if total_blocks_yield > self._n_blocks:
raise ValueError('total_blocks_yield > self._n_blocks! The code has some bugs to fix!')
 if DEBUG:
assert total_blocks_yield == self._n_blocks - self._n_remaining_blocks
self.current_batch_no += 1
if self.current_batch_no % PRINTING_STEPS == 0:
print(f'current_batch_no: {self.current_batch_no}')
 print(f'n_blocks processed in current batch: {n_blocks_in_batch}')
 print(f'n_blocks processed: {total_blocks_yield}')
 print(f'averaged n_blocks processed per batch: {total_blocks_yield / self.current_batch_no}')
 print(f'n_users_completed: {self._n_users - len(self._remaining_users)}')
 yield valid_df, valid_dt, prior_group_answers_correct, prior_group_responses
except StopIteration:
break
if DEBUG:
assert len(self._user_manager_dict)== 0
assert len(self._remaining_users)== 0
assert self._n_remaining_blocks == 0
print('finished looping.')
 def _step(self, n_users):
return self._blocks_from_random_users(n_users)
 def _blocks_from_random_users(self, n_users):
if DEBUG:
assert len(self._remaining_users)>= 0
if len(self._remaining_users)== 0:
raise StopIteration
n_users_to_select = min(n_users, len(self._remaining_users))
selected_users = random.sample(self._remaining_users, k=n_users_to_select)
 if DEBUG:
assert len(selected_users)== n_users_to_select
user_dts = []
for user_id in selected_users:
if self._user_manager_dict[user_id].has_next() :
user_dts.append(next(self._user_manager_dict[user_id]))
if not self._user_manager_dict[user_id].has_next() :
del self._user_manager_dict[user_id]
self._remaining_users.remove(user_id)
 valid_dt = dt.rbind(user_dts)
 n_blocks_selected = n_users_to_select
self._n_remaining_blocks -= n_blocks_selected
prior_group_answers_correct = self._current_answered_correctly
prior_group_responses = self._current_user_answer
if prior_group_answers_correct is None:
prior_group_answers_correct = []
if prior_group_responses is None:
prior_group_responses = []
if DEBUG:
assert type(prior_group_answers_correct)== list
assert type(prior_group_responses)== list
self._current_answered_correctly = valid_dt[:, 'answered_correctly'].to_list() [0]
self._current_user_answer = valid_dt[:, 'user_answer'].to_list() [0]
convert_dt(valid_dt, pred=True, unique_question_id_train=self._unique_question_id_train, unique_lecture_id_train=self._unique_lecture_id_train)
 valid_df = valid_dt.to_pandas()
return valid_df, valid_dt, prior_group_answers_correct, prior_group_responses, n_blocks_selected
print("Null counts: " , housing_all_df['Electrical'].isnull().sum())
 print(housing_all_df['Electrical'].value_counts())
 housing_all_df['Electrical'].fillna(housing_all_df['Electrical'].mode().values[0], inplace=True )<count_missing_values>
class Test_Iter_Manager(Pred_Iter_Manager):
def __init__(self, unique_question_id_train=None, unique_lecture_id_train=None):
super(Test_Iter_Manager, self ).__init__(unique_question_id_train, unique_lecture_id_train)
 self.env = riiideducation.make_env()
self._iter_test = iter(self.env.iter_test())
 @classmethod
def create(cls, unique_question_id_train_path=None, unique_lecture_id_train_path=None):
args = []
for p in [unique_question_id_train_path, unique_lecture_id_train_path]:
if p is not None:
args.append(load_data(p))
return cls(*args)
 def _has_next(self):
return self._has_next
def iter_dataset(self, max_steps=None):
print('entering iter_dataset() ')
 total_blocks_yield = 0
print('start looping...')
 while True:
try:
test_df, test_dt, prior_group_answers_correct, prior_group_responses, n_blocks_in_batch = self._step(n_users=None)
 assert n_blocks_in_batch > 0
total_blocks_yield += n_blocks_in_batch
self.current_batch_no += 1
if self.current_batch_no % PRINTING_STEPS == 0:
print(f'current_batch_no: {self.current_batch_no}')
 print(f'n_blocks processed in current batch: {n_blocks_in_batch}')
 print(f'n_blocks processed: {total_blocks_yield}')
 print(f'averaged n_blocks processed per batch: {total_blocks_yield / self.current_batch_no}')
 yield test_df, test_dt, prior_group_answers_correct, prior_group_responses
except StopIteration:
break
print('finished looping.')
 def _step(self, n_users=None):
test_df, _ = next(self._iter_test)
 test_dt, prior_group_answers_correct, prior_group_responses = convert_df_to_dt(
 test_df, test=True,
unique_question_id_train=self._unique_question_id_train,
unique_lecture_id_train=self._unique_lecture_id_train 
)
 n_blocks = len(set(test_dt[:, 'user_id'].to_list() [0]))
return test_df, test_dt, prior_group_answers_correct, prior_group_responses, n_blocks
print("Null count: " , housing_all_df['KitchenQual'].isnull().sum())
 print(housing_all_df['KitchenQual'].value_counts())
 housing_all_df['KitchenQual'].fillna(housing_all_df['KitchenQual'].mode().values[0], inplace=True )<count_missing_values>
def get_user_dt(user_id, _dt, user_id_to_row_id, split_index=-1):

(row_id_start, row_id_end)= user_id_to_row_id[user_id]
if split_index > -1:
row_id_end = row_id_start + split_index - 1
user_dt = _dt[row_id_start:row_id_end + 1, :]
return user_dt
attrs = [
'user_id',
'row_id',
'timestamp',
'content_id',
'content_type_id',
'task_container_id',
'user_answer',
'answered_correctly',
'prior_question_elapsed_time',
'prior_question_had_explanation'
]
extended_attrs = ['user_id'] + ['seq_len', 'prev_seq_len', 'start', 'end'] + \
[k for k in attrs if k not in ['user_id']] + \
['pred_time_mask', 'abs_pos', 'shifted_abs_pos', 'shifted_answered_correctly', 'shifted_user_answer'] + \
['lag_time']
aggregated_attrs_less = [
'n_questions_answered',
'n_questions_answered_correctly',
'n_lectures_watched'
]
for part_idx in range(2, 9):
aggregated_attrs_less.append(f'part_{part_idx}_count')
 aggregated_attrs_less.append(f'part_{part_idx}_correctness_count')
 for correct_answer_idx in range(ANSWER_0_ID, ANSWER_3_ID + 1):
	aggregated_attrs_less.append(f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_count')
 	aggregated_attrs_less.append(f'correct_answer_{correct_answer_idx - ANSWER_0_ID}_answered_correctly_count')
 aggregated_attrs = aggregated_attrs_less + ['n_prev_seen', 'n_prev_correctness']
attr_dtypes = {}
for k in extended_attrs:
if k in ['row_id', 'user_id', 'timestamp', 'lag_time']:
attr_dtypes[k] = tf.int64
elif k in ['prior_question_elapsed_time']:
attr_dtypes[k] = DTYPE
else:
attr_dtypes[k] = tf.int32
for k in aggregated_attrs:
attr_dtypes[k] = tf.int32
class User_Record:

 def __init__(self, user_dt=None, user_id=None):

 user_ids = user_dt[:, 'user_id'].to_list() [0] if user_dt is not None else []
self.user_id = user_ids[0] if user_dt is not None else user_id
self.row_id = user_dt[:, 'row_id'].to_list() [0] if user_dt is not None else []
self.timestamp = user_dt[:, 'timestamp'].to_list() [0] if user_dt is not None else []
self.content_id = user_dt[:, 'content_id'].to_list() [0] if user_dt is not None else []
self.content_type_id = user_dt[:, 'content_type_id'].to_list() [0] if user_dt is not None else []
self.task_container_id = user_dt[:, 'task_container_id'].to_list() [0] if user_dt is not None else []
self.user_answer = user_dt[:, 'user_answer'].to_list() [0] if user_dt is not None else []
self.answered_correctly = user_dt[:, 'answered_correctly'].to_list() [0] if user_dt is not None else []
self.prior_question_elapsed_time = user_dt[:, 'prior_question_elapsed_time'].to_list() [0] if user_dt is not None else []
self.prior_question_had_explanation = user_dt[:, 'prior_question_had_explanation'].to_list() [0] if user_dt is not None else []
for aggregated_feature_idx, aggregated_feature_name in enumerate(aggregated_attrs):
setattr(self, aggregated_feature_name, [])
 def extend(self, other):

 for k in attrs + aggregated_attrs:
if k != 'user_id':
getattr(self, k ).extend(getattr(other, k))
def update_answer_results(self, prior_correctnesses, prior_answers, prior_user_info_for_post_update, question_history_at_training_end):


n_pred_time_steps = prior_user_info_for_post_update['n_pred_time_steps']
aggregated_info_to_update = prior_user_info_for_post_update['aggregated_info_to_update']
self.answered_correctly = self.answered_correctly[:-len(prior_correctnesses)] + prior_correctnesses
self.user_answer = self.user_answer[:-len(prior_answers)] + prior_answers
for idx in range(n_pred_time_steps):
index_in_pred_record = -(n_pred_time_steps - idx)
 if prior_correctnesses[idx] == 1:
current_question_id_str = str(self.content_id[index_in_pred_record])
 question_history_at_training_end[str(self.user_id)][current_question_id_str][1] += 1
if idx > 0:
for k in aggregated_attrs_less:
aggregated_info_to_update[k][idx] = aggregated_info_to_update[k][idx-1]
prev_correctness = prior_correctnesses[idx - 1]
if prev_correctness == 1:
aggregated_info_to_update['n_questions_answered_correctly'][idx] = aggregated_info_to_update['n_questions_answered_correctly'][idx-1] + 1
prev_part_key = aggregated_info_to_update['prev_part_key'][idx]
prev_correct_answer_key = aggregated_info_to_update['prev_correct_answer_key'][idx]
aggregated_info_to_update[prev_part_key][idx] = aggregated_info_to_update[prev_part_key][idx-1] + 1
aggregated_info_to_update[prev_correct_answer_key][idx] = aggregated_info_to_update[prev_correct_answer_key][idx-1] + 1
for k in aggregated_attrs_less:
if k == 'n_questions_answered_correctly' or 'correctness_count' in k or 'answered_correctly_count' in k:
setattr(self, k, getattr(self, k)[:-n_pred_time_steps] + aggregated_info_to_update[k])
 def toJSON(self):
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=False, indent=4)
 def __str__(self):
return self.toJSON()
def __len__(self):
return len(self.row_id)
 class Record_Buffer:

 def __init__(self, record_dict=None):

 if record_dict is None:
self.buffer = {}
else:
self.buffer = record_dict
def __contains__(self, x):
return x in self.buffer
def __getitem__(self, x):
if x not in self.buffer:
raise KeyError(str(x))
return self.buffer[x]
def __len__(self):
return len(self.buffer)
 def __del__(self):
del self.buffer
def update(self, record):

 if record.user_id not in self.buffer:
self.buffer[record.user_id] = User_Record(user_id=record.user_id)
 self.buffer[record.user_id].extend(record)
 def update_answer_results(self, user_id, prior_correctnesses, prior_answers, prior_user_info_for_post_update, question_history_at_training_end):

 record = self.buffer[user_id]
record.update_answer_results(prior_correctnesses, prior_answers, prior_user_info_for_post_update, question_history_at_training_end)
 def toJSON(self):
return json.dumps(self, default=lambda o: o.__dict__, sort_keys=False, indent=4)
 def __str__(self):
return self.toJSON()
def combine_user_record(record_1, record_2, check=False):

 record = User_Record(user_id=record_1.user_id)
 record.extend(record_1)
 record.extend(record_2)
 return record
class Pred_Manager:

 def __init__(
 self, config, train_config, pred_iter_manager, train_dt, user_id_to_row_id_train,
question_history_at_training_end, single_question_history_at_training_end_optimized,
max_train_buffer_size=30000, probe=False 
):
self.config = config
self.train_config = train_config
self.pred_iter_manager = pred_iter_manager
self.train_dt = train_dt
self.user_id_to_row_id_train = user_id_to_row_id_train
self.question_history_at_training_end = question_history_at_training_end
self.single_question_history_at_training_end_optimized = single_question_history_at_training_end_optimized
self.train_record_buffer = Record_Buffer()
self.pred_record_buffer = Record_Buffer()
self.current_batch_row_ids = None
self.current_batch_users = None
self.max_train_buffer_size = max_train_buffer_size
self.users_in_pred = set()
self.common_users_in_train_and_pred = set()
self.new_users_in_pred = set()
self.n_blocks_in_pred = 0
self.n_blocks_from_common_users_in_train_and_pred = 0
self.n_blocks_from_new_users_in_pred = 0
self.avg_n_blocks_in_pred = 0
self.avg_n_blocks_from_common_users_in_train_and_pred = 0
self.avg_n_blocks_from_new_users_in_pred = 0
self.users_in_train = set(user_id_to_row_id_train.keys())
 self.users_trained = set()
for user_id in self.users_in_train:
split_index = self.pred_iter_manager.get_split_index(user_id)
 if split_index == -1 or split_index > 0:
self.users_trained.add(user_id)
 self.probe = probe
self.user_performance_hdf5_fp = h5py.File(user_performance_hdf5_path, "r")
 self.user_performance_hdf5 = self.user_performance_hdf5_fp['user_performance']
@classmethod
def create(cls, config, train_config, pred_iter_manager_class,
train_dt_path_or_obj,
user_id_to_row_id_train_path,
unique_question_id_train_path,
unique_lecture_id_train_path,
question_history_at_training_end_path,
single_question_history_at_training_end_optimized_path,
valid_info_path=None,
max_train_buffer_size=30000,
probe=False,
debug=False 
):
if pred_iter_manager_class == Valid_Iter_Manager:
data_paths = [valid_info_path, train_dt_path_or_obj]
elif pred_iter_manager_class == Test_Iter_Manager:
data_paths = []
assert type(unique_question_id_train_path)== str and type(unique_lecture_id_train_path)== str
data_paths.extend([unique_question_id_train_path, unique_lecture_id_train_path])
 pred_iter_manager = pred_iter_manager_class.create(*data_paths)
 user_id_to_row_id_train = load_data(user_id_to_row_id_train_path)
 start_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
print(f'start_memeory before loading question_history_at_training_end.json: {start_memeory}')
 question_history_at_training_end = load_data(question_history_at_training_end_path)
 end_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
print(f'end_memeory after loading question_history_at_training_end.json: {end_memeory}')
 used_memory = start_memeory - end_memeory
print(f'used_memory for loading question_history_at_training_end.json: {used_memory}')
 print('----------------------------------------')
 start_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
print(f'start_memeory before loading single_question_history_at_training_end_optimized.json: {start_memeory}')
 single_question_history_at_training_end_optimized = load_data(single_question_history_at_training_end_optimized_path)
 end_memeory = psutil.virtual_memory().available / 1024.0 / 1024.0
print(f'end_memeory after loading single_question_history_at_training_end_optimized.json: {end_memeory}')
 used_memory = start_memeory - end_memeory
print(f'used_memory for loading single_question_history_at_training_end_optimized.json: {used_memory}')
 print('----------------------------------------')
 if pred_iter_manager_class == Valid_Iter_Manager:
train_dt = pred_iter_manager.train_dt
elif type(train_dt_path_or_obj)!= str:
train_dt = train_dt_path_or_obj
else:
train_dt = load_data(train_dt_path_or_obj)
 args = [
config, train_config, pred_iter_manager, train_dt, user_id_to_row_id_train,
question_history_at_training_end, single_question_history_at_training_end_optimized,
max_train_buffer_size, probe
]
return cls(*args)
 def reset_train_record_buffer(self):

 user_ids = list(self.train_record_buffer.buffer.keys())
 for user_id in user_ids:
del self.train_record_buffer.buffer[user_id]
del self.train_record_buffer
self.train_record_buffer = Record_Buffer()
def update_batch_users(self, user_ids):

 self.current_batch_users = user_ids
def update_answer_results(self, prev_batch_users, prior_group_answers_correct, prior_group_responses, prior_info_for_post_update):

 d1 = defaultdict(list)
 d2 = defaultdict(list)
 for user_id, prior_ans_correct, prior_ans in zip(prev_batch_users, prior_group_answers_correct, prior_group_responses):
d1[user_id].append(prior_ans_correct)
 d2[user_id].append(prior_ans)
 for user_id in d1:
self.pred_record_buffer.update_answer_results(
 user_id,
d1[user_id], d2[user_id],
prior_info_for_post_update[user_id],
self.question_history_at_training_end 
)
 def update_aggregated_info_buffer(self, user_id, pred_record, record_to_watch, index_to_watch, n_pred_time_steps, pred_time_step_index, buffer):
user_id_str = str(user_id)
 current_index_in_pred_record = -(n_pred_time_steps - pred_time_step_index)
 n_prev_seen, n_prev_correctness = None, None
if pred_record.content_type_id[current_index_in_pred_record] == 0:
current_question_id = pred_record.content_id[current_index_in_pred_record]
current_question_id_str = str(current_question_id)
 if user_id_str not in self.question_history_at_training_end:
self.question_history_at_training_end[user_id_str] = dict()
if current_question_id_str not in self.question_history_at_training_end[user_id_str]:
self.question_history_at_training_end[user_id_str][current_question_id_str] = [None, None]
else:
n_prev_seen, n_prev_correctness = self.question_history_at_training_end[user_id_str][current_question_id_str]
if n_prev_seen is None:
if user_id_str in self.single_question_history_at_training_end_optimized:
if type(self.single_question_history_at_training_end_optimized[user_id_str]['correct'])== list:
self.single_question_history_at_training_end_optimized[user_id_str]['correct'] = set(self.single_question_history_at_training_end_optimized[user_id_str]['correct'])
 if type(self.single_question_history_at_training_end_optimized[user_id_str]['incorrect'])== list:
self.single_question_history_at_training_end_optimized[user_id_str]['incorrect'] = set(self.single_question_history_at_training_end_optimized[user_id_str]['incorrect'])
 if current_question_id_str in self.single_question_history_at_training_end_optimized[user_id_str]['correct']:
n_prev_seen, n_prev_correctness = [1, 1]
self.single_question_history_at_training_end_optimized[user_id_str]['correct'].remove(current_question_id_str)
 elif current_question_id_str in self.single_question_history_at_training_end_optimized[user_id_str]['incorrect']:
n_prev_seen, n_prev_correctness = [1, 0]
self.single_question_history_at_training_end_optimized[user_id_str]['incorrect'].remove(current_question_id_str)
 if n_prev_seen is None:
n_prev_seen, n_prev_correctness = [0, 0]
self.question_history_at_training_end[user_id_str][current_question_id_str][0] = n_prev_seen + 1
self.question_history_at_training_end[user_id_str][current_question_id_str][1] = n_prev_correctness
if n_prev_seen is None:
n_prev_seen, n_prev_correctness = [0, 0]
buffer['n_prev_seen'].append(n_prev_seen)
 buffer['n_prev_correctness'].append(n_prev_correctness)
 if record_to_watch is None:
for k in aggregated_attrs_less:
buffer[k].append(0)
 for k in ['prev_part_key', 'prev_correct_answer_key']:
buffer[k].append(None)
 return
if pred_time_step_index == 0:
for k in aggregated_attrs_less:
buffer[k].append(getattr(record_to_watch, k)[-1])
 else:
for k in aggregated_attrs_less:
buffer[k].append(buffer[k][-1])
 for k in ['prev_part_key', 'prev_correct_answer_key']:
buffer[k].append(None)
 content_id = record_to_watch.content_id[index_to_watch]
content_type_id = record_to_watch.content_type_id[index_to_watch]
if content_type_id == 0:
content_input_id = question_id_to_input_id_dict[content_id]
part_id = c_inputs_ids_to_part_dict[content_input_id] + 1
correct_answer = c_inputs_ids_to_correct_answer_id_dict[content_input_id] - ANSWER_0_ID
question_id = str(content_id)
 answered_correctly =(record_to_watch.answered_correctly[index_to_watch] == 1)
 buffer['prev_part_key'][-1] = f'part_{part_id}_correctness_count'
buffer['prev_correct_answer_key'][-1] = f'correct_answer_{correct_answer}_answered_correctly_count'
is_lecture = int(content_type_id == 1)
 buffer['n_lectures_watched'][-1] += is_lecture
if is_lecture == 0:
buffer['n_questions_answered'][-1] += 1
k = f'part_{part_id}_count'
buffer[k][-1] += 1
k = f'correct_answer_{correct_answer}_count'
buffer[k][-1] += 1
if answered_correctly is True:
for k in [
f'n_questions_answered_correctly',
f'part_{part_id}_correctness_count',
f'correct_answer_{correct_answer}_answered_correctly_count'
]:
buffer[k][-1] += 1
def get_aggregated_info_to_update(self, user_id, train_record, pred_record, n_pred_time_steps):
if len(pred_record)> n_pred_time_steps:
record_to_watch = pred_record
index_to_watch = -(n_pred_time_steps + 1)
 elif len(train_record)> 0:
record_to_watch = train_record
index_to_watch = -1
else:
record_to_watch = None
index_to_watch = None
aggregated_info_buffer = {
k: [] for k in aggregated_attrs + ['prev_part_key', 'prev_correct_answer_key']
}
for pred_time_step_index in range(n_pred_time_steps):
if pred_time_step_index >= 1:
record_to_watch = pred_record
index_to_watch = -(n_pred_time_steps - pred_time_step_index)- 1
self.update_aggregated_info_buffer(user_id, pred_record, record_to_watch, index_to_watch, n_pred_time_steps, pred_time_step_index, aggregated_info_buffer)
 return aggregated_info_buffer
def update(self, pred_dt, prior_group_answers_correct, prior_group_responses, prior_info_for_post_update):

 user_ids = pred_dt['user_id'].to_list() [0]
prev_batch_users = self.current_batch_users
if prev_batch_users is not None:
self.update_answer_results(prev_batch_users, prior_group_answers_correct, prior_group_responses, prior_info_for_post_update)
 self.update_batch_users(user_ids)
 if DEBUG:
row_ids = pred_dt['row_id'].to_list() [0]
prev_batch_row_ids = self.current_batch_row_ids
if isinstance(self.pred_iter_manager, Test_Iter_Manager):
debug_pred_batch_row_ids(row_ids, prev_batch_row_ids)
 self.current_batch_row_ids = row_ids
record_batch = [User_Record(user_dt=pred_dt[idx, :])for idx in range(pred_dt.nrows)]
if DEBUG:
debug_record_batch(record_batch)
 for record in record_batch:
self.pred_record_buffer.update(record)
 self.users_in_pred.update(set(user_ids))
n_new_users_in_batch = 0
for x in set(user_ids):
if x in self.users_trained:
self.common_users_in_train_and_pred.add(x)
 else:
self.new_users_in_pred.add(x)
 n_new_users_in_batch += 1
nb_common_users = len(self.common_users_in_train_and_pred)
 nb_new_users = len(self.new_users_in_pred)
 common_users_in_batch = set(user_ids ).intersection(self.users_trained)
 new_users_in_batch = set(user_ids ).difference(self.users_trained)
 self.n_blocks_in_pred += len(set(user_ids))
self.n_blocks_from_common_users_in_train_and_pred += len(common_users_in_batch)
 self.n_blocks_from_new_users_in_pred += len(new_users_in_batch)
 self.avg_n_blocks_in_pred = self.n_blocks_in_pred / len(self.users_in_pred)
 if nb_common_users > 0:
self.avg_n_blocks_from_common_users_in_train_and_pred = self.n_blocks_from_common_users_in_train_and_pred / nb_common_users
if nb_new_users > 0:
self.avg_n_blocks_from_new_users_in_pred = self.n_blocks_from_new_users_in_pred / nb_new_users
_pred_record_batch = []
n_pred_time_steps_dict = dict()
for x in user_ids:
if x not in n_pred_time_steps_dict:
_pred_record_batch.append(self.pred_record_buffer[x])
 n_pred_time_steps_dict[x] = 1
else:
n_pred_time_steps_dict[x] += 1
continue
user_ids_without_duplication = [x.user_id for x in _pred_record_batch]
n_pred_time_steps_batch = [n_pred_time_steps_dict[x.user_id] for x in _pred_record_batch]
train_record_batch = [self.get_training_record(x)for x in user_ids_without_duplication]
check = isinstance(self.pred_iter_manager, Valid_Iter_Manager)
 info_for_post_update = {}
pred_record_batch = []
for user_id, train_record, _pred_record, n_pred_time_steps in zip(
 user_ids_without_duplication,
train_record_batch,
_pred_record_batch,
n_pred_time_steps_batch 
):
aggregated_info_to_update = self.get_aggregated_info_to_update(user_id, train_record, _pred_record, n_pred_time_steps)
 pred_record = combine_user_record(train_record, _pred_record, check=check)
 pred_record_batch.append(pred_record)
 info_for_post_update[user_id] = {
'user_id': user_id,
'pred_record': _pred_record,
'n_pred_time_steps': n_pred_time_steps,
'aggregated_info_to_update': aggregated_info_to_update
}
for k in aggregated_attrs:
if k in ['n_lectures_watched', 'n_prev_seen', 'n_prev_correctness']:
getattr(pred_record, k ).extend(aggregated_info_to_update[k])
 else:
getattr(pred_record, k ).extend([aggregated_info_to_update[k][0]] * n_pred_time_steps)
 getattr(_pred_record, k ).extend(aggregated_info_to_update[k])
 if len(self.train_record_buffer)>= 3000:
self.reset_train_record_buffer()
gc.collect()
return pred_record_batch, n_pred_time_steps_batch, info_for_post_update
def get_training_record(self, user_id):

 if user_id in self.train_record_buffer:
return self.train_record_buffer[user_id]
if user_id not in self.user_id_to_row_id_train:
assert isinstance(self.pred_iter_manager, Test_Iter_Manager)
 else:
split_index = self.pred_iter_manager.get_split_index(user_id)
 user_dt = get_user_dt(user_id, self.train_dt, self.user_id_to_row_id_train, split_index)
 if isinstance(self.pred_iter_manager, Test_Iter_Manager):
assert user_dt.nrows > 0
if user_dt.nrows > 0:
convert_dt(user_dt, pred=False)
 user_record = User_Record(user_dt=user_dt)
 self.train_record_buffer.update(user_record)
 row_ids = self.train_record_buffer[user_id].row_id
row_start, row_end = row_ids[0], row_ids[-1]
seq_len = row_end - row_start + 1
user_performance = self.user_performance_hdf5[row_start:row_end + 1, :]
for feature_idx, k in enumerate(aggregated_attrs):
feature_performance = user_performance[:, feature_idx].tolist()
getattr(self.train_record_buffer[user_id], k ).extend(feature_performance)
 return self.train_record_buffer[user_id]
user_record = User_Record(user_id=user_id)
 self.train_record_buffer.update(user_record)
 return self.train_record_buffer[user_id]
def convert_to_pred_batch(self, pred_record_batch, n_pred_time_steps_batch):
batch_record = self.convert_record_batch(pred_record_batch, n_pred_time_steps_batch)
 batch = Pred_Manager.batch_record_to_tensor(batch_record)
 training = tf.constant(0, dtype=tf.int32)
 return batch
def record_to_dict(self, record, n_pred_time_steps):
record_as_dict = {}
orig_seq_len = len(record.row_id)
 used_seq_len = orig_seq_len
pad_seq = []
if orig_seq_len < self.train_config.window_size:
pad_seq = [PAD_TOKEN] *(self.train_config.window_size - orig_seq_len)
 if orig_seq_len > self.train_config.window_size:
used_seq_len = self.train_config.window_size
record_as_dict['user_id'] = record.user_id
record_as_dict['seq_len'] = used_seq_len
record_as_dict['prev_seq_len'] = orig_seq_len
record_as_dict['start'] = max(orig_seq_len - self.train_config.window_size, 0)
 record_as_dict['end'] = orig_seq_len - 1
answered_correctly = getattr(record, 'answered_correctly')
 user_answer = getattr(record, 'user_answer')
 for k in attrs + aggregated_attrs:
if k not in ['user_id']:
record_as_dict[k] =(pad_seq + getattr(record, k)) [- self.train_config.window_size:]
shifted_answered_correctly = [START_TOKEN] + answered_correctly[:-1]
record_as_dict['shifted_answered_correctly'] =(pad_seq + shifted_answered_correctly)[- self.train_config.window_size:]
shifted_user_answer = [START_TOKEN] + user_answer[:-1]
record_as_dict['shifted_user_answer'] =(pad_seq + shifted_user_answer)[- self.train_config.window_size:]
pred_time_mask = [0] * orig_seq_len
pred_time_mask =(pad_seq + pred_time_mask)[- self.train_config.window_size:]
pred_time_mask = pred_time_mask[:self.train_config.window_size - n_pred_time_steps] + [1] * n_pred_time_steps
record_as_dict['pred_time_mask'] = pred_time_mask
abs_pos = list(range(orig_seq_len))
shifted_abs_pos = [START_TOKEN] + abs_pos[:-1]
record_as_dict['abs_pos'] =(pad_seq + abs_pos)[- self.train_config.window_size:]
record_as_dict['shifted_abs_pos'] =(pad_seq + shifted_abs_pos)[- self.train_config.window_size:]
timestamps = getattr(record, 'timestamp')
 task_container_ids = getattr(record, 'task_container_id')
 nb_actual_elts = min(self.train_config.window_size, len(timestamps))
_start_index = max(0, len(timestamps)-(self.train_config.window_size + MAX_PRED_TIME_QUESTION_BUNDLE_LEN))
timestamps = timestamps[_start_index:]
task_container_ids = task_container_ids[_start_index:]
nb_elt_to_add =(self.train_config.window_size + MAX_PRED_TIME_QUESTION_BUNDLE_LEN)- len(timestamps)
 if nb_elt_to_add > 0:
timestamps = [0] * nb_elt_to_add + timestamps
task_container_ids = [PAD_TOKEN] * nb_elt_to_add + task_container_ids
lag_time = []
current_lag_time = 0
for _t, _prev_t, _bundle, _prev_bundle in zip(timestamps[1:], timestamps[:-1], task_container_ids[1:], task_container_ids[:-1]):
_diff = _t - _prev_t
if _diff == 0 and _bundle == _prev_bundle:
_diff = current_lag_time
elif _diff == 0 and _bundle != _prev_bundle:
pass
else:
current_lag_time = _diff
lag_time.append(_diff)
 lag_time = lag_time[-nb_actual_elts:]
timestamps = timestamps[1:][-nb_actual_elts:]
_mask = tf.cast(tf.math.logical_and(tf.constant(lag_time, dtype=tf.int64)== 0, tf.constant(timestamps, dtype=tf.int64)> 0), dtype=tf.int64)
 lag_time = tf.constant(lag_time, dtype=tf.int64)*(1 - _mask)+ tf.constant(1000, dtype=tf.int64)* _mask
lag_time = lag_time.numpy().tolist()
lag_time =(pad_seq + lag_time)[- self.train_config.window_size:]
record_as_dict['lag_time'] = lag_time
return record_as_dict
def convert_record_batch(self, record_batch, n_pred_time_steps_batch):

 records_as_dicts = [Pred_Manager.record_to_dict(self, record, n_pred_time_steps)for record, n_pred_time_steps in zip(record_batch, n_pred_time_steps_batch)]
batch_record = {k: [x[k] for x in records_as_dicts] for k in extended_attrs + aggregated_attrs}
return batch_record
@staticmethod
def batch_record_to_tensor(batch_record):

 t = {
k: tf.constant(batch_record[k], dtype=attr_dtypes[k])for k in extended_attrs + aggregated_attrs
}
return t
print("Null count: ", housing_all_df['Functional'].isnull().sum())
 print(housing_all_df['Functional'].value_counts())
 housing_all_df['Functional'].fillna(housing_all_df['Functional'].mode().values[0], inplace = True )<count_missing_values>
def debug_pred_batch_row_ids(row_ids, prev_batch_row_ids):

 assert len(set(row_ids)) == len(row_ids)
 assert row_ids == sorted(row_ids)
 if prev_batch_row_ids is not None:
assert row_ids[0] > prev_batch_row_ids[-1]
def debug_record_batch(record_batch):

 _tmp = defaultdict(list)
 for record in record_batch:
assert len(record.row_id)== 1
_tmp[record.user_id].append(record)
 for user_id, records in _tmp.items() :
task_container_ids_for_questions = [x.task_container_id[0] for x in records if x.content_type_id[0] == 0]
if len(task_container_ids_for_questions)> 0:
assert len(set(task_container_ids_for_questions)) == 1
row_ids_for_questions = [x.row_id[0] for x in records if x.content_type_id[0] == 0]
records_in_between = [x for x in records if row_ids_for_questions[0] <= x.row_id[0] <= row_ids_for_questions[-1]]
row_ids_in_between = [x.row_id[0] for x in records_in_between]
assert row_ids_for_questions == row_ids_in_between
assert row_ids_for_questions == [x.row_id[0] for x in records][-len(row_ids_for_questions):]