Kaggle competition: COVID19 Global Forecasting (Week 4)
train['prior_question_elapsed_time'] = train.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 valid['prior_question_elapsed_time'] = valid.prior_question_elapsed_time.fillna(prior_question_elapsed_time_mean)
 train['prior_question_had_explanation'] = train.prior_question_had_explanation.fillna(False ).astype('int8')
 valid['prior_question_had_explanation'] = valid.prior_question_had_explanation.fillna(False ).astype('int8' )<prepare_x_and_y>
train_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/train.csv")
 test_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-4/test.csv" )
TARGET = 'answered_correctly'
FEATS = ['answered_correctly_avg_u', 'answered_correctly_avg_c','answered_correctly_std_c','answered_correctly_sum_u','count_u', 'part','tag', 'prior_question_elapsed_time','time_diff', 'prior_question_had_explanation']
dro_cols = list(set(train.columns)- set(FEATS))
y_tr = train[TARGET]
y_va = valid[TARGET]
_=gc.collect()<create_dataframe>
train_df.apply(lambda col: col.isnull().value_counts() , axis=0 )
lgb_train = lgb.Dataset(train[FEATS], y_tr, categorical_feature = ['part', 'prior_question_had_explanation', 'tag'],free_raw_data=False)
 lgb_valid = lgb.Dataset(valid[FEATS], y_va, categorical_feature = ['part', 'prior_question_had_explanation', 'tag'], reference=lgb_train,free_raw_data=False)
 _=gc.collect()<import_modules>
test_df.apply(lambda col: col.isna().value_counts() , axis=0 )
import optuna<find_best_params>
train_df["Province_State"] = train_df["Province_State"].fillna("")
 test_df["Province_State"] = test_df["Province_State"].fillna("" )
def objective(trial):
params = {
'num_leaves': trial.suggest_int('num_leaves', 32, 512),
'max_bin': trial.suggest_int('max_bin', 700, 900),
'boosting' : 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': trial.suggest_int('max_depth', 4, 16),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 16),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 8),
'min_child_samples': trial.suggest_int('min_child_samples', 4, 80),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 1.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 1.0),
'early_stopping_rounds': 5
}
model = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_valid], verbose_eval=1000, num_boost_round=10000)
 val_pred = model.predict(valid[FEATS])
 score = roc_auc_score(y_va, val_pred)
 print(f"AUC = {score}")
 return score<find_best_params>
train_df["Date"] = pd.to_datetime(train_df["Date"])
 test_df["Date"] = pd.to_datetime(test_df["Date"] )
study = optuna.create_study(direction='maximize')
 study.optimize(objective, n_trials=1 )<find_best_params>
train_df["NewCases"] = train_df.groupby(["Country_Region", "Province_State"])["ConfirmedCases"].diff(periods=1)
 train_df["NewCases"] = train_df["NewCases"].fillna(0)
 train_df["NewCases"] = np.where(train_df["NewCases"] < 0, 0, train_df["NewCases"])
 train_df["NewFatalities"] = train_df.groupby(["Country_Region", "Province_State"])["Fatalities"].diff(periods=1)
 train_df["NewFatalities"] = train_df["NewFatalities"].fillna(0)
 train_df["NewFatalities"] = np.where(train_df["NewFatalities"] < 0, 0, train_df["NewFatalities"] )
print('Number of finished trials: {}'.format(len(study.trials)))
 print('Best trial:')
 trial = study.best_trial
print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))<init_hyperparams>
train_df["NewCases"] = np.log(train_df["NewCases"] + 1)
 train_df["NewFatalities"] = np.log(train_df["NewFatalities"] + 1 )
parameters = trial.params
parameters['objective'] = 'binary'
parameters['metric'] = 'auc'
parameters['early_stopping_rounds'] = 5
parameters['boosting'] = 'gbdt'<import_modules>
def preprocess_train(n_prev, n_next):
df = train_df.copy()
input_feats, output_feats = [], []
for i in range(1, n_prev+1):
for feat in ["NewCases", "NewFatalities"]:
df["{}_prev_{}".format(feat, i)] = df.groupby(["Country_Region", "Province_State"])[feat].shift(i)
 input_feats.append("{}_prev_{}".format(feat, i))
output_feats.extend(["NewCases", "NewFatalities"])
 for i in range(1, n_next):
for feat in ["NewCases", "NewFatalities"]:
df["{}_next_{}".format(feat, i)] = df.groupby(["Country_Region", "Province_State"])[feat].shift(-i)
 output_feats.append("{}_next_{}".format(feat, i))
df.dropna(inplace=True)
 const_df = pd.get_dummies(df[["Province_State", "Country_Region"]], drop_first=True)
 time_df = df[input_feats]
time_df = time_df.values.reshape(( df.shape[0],-1,2))
output_df = df[output_feats]
return const_df, time_df, output_df
import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Dense, Dropout, LSTM, Masking, Embedding, Concatenate, Input
from tensorflow.keras.layers import concatenate
from tensorflow.keras.metrics import AUC
from tensorflow.keras import backend as K
from tensorflow.keras.layers import Lambda
from tensorflow.keras.layers import multiply, Reshape
from tensorflow.keras.utils import Sequence<feature_engineering>
def preprocess_test(n_prev):
input_feats = []
append_df = pd.concat([train_df, test_df[test_df["Date"] == train_df["Date"].max() + timedelta(days=1)]])
 append_df.sort_values(["Country_Region", "Province_State", "Date"], ascending=[True, True, True], inplace=True)
 for i in range(1, n_prev+1):
for feat in ["NewCases", "NewFatalities"]:
append_df["{}_prev_{}".format(feat, i)] = append_df.groupby(["Country_Region", "Province_State"])[feat].shift(i)
 input_feats.append("{}_prev_{}".format(feat, i))
append_df = append_df[append_df["ForecastId"].notnull() ]
const_df = pd.get_dummies(append_df[["Province_State", "Country_Region"]], drop_first=True)
 time_df = append_df[input_feats]
time_df = time_df.values.reshape(( append_df.shape[0],-1,2))
return const_df, time_df
train.time_diff[train.time_diff >= 1e6] = 1e6
valid.time_diff[valid.time_diff >= 1e6] = 1e6
train.prior_question_elapsed_time = train.prior_question_elapsed_time/3e5
train.time_diff = train.time_diff/1e6
valid.prior_question_elapsed_time = valid.prior_question_elapsed_time/3e5
valid.time_diff = valid.time_diff/1e6<count_values>
n_next =(test_df["Date"].max() - train_df["Date"].max() ).days
n_next
<define_variables>
const_df, time_df, output_df = preprocess_train(n_next, n_next )
max_len = 31<categorify>
const_test_df, time_test_df = preprocess_test(n_next )
train_group = train.groupby('user_id')
 train_y = [frame['answered_correctly'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['answered_correctly'].to_numpy() [:, None]), max_len)]
train_y = np.reshape(pad_sequences(train_y, padding="pre"),(-1,max_len))
train_past_answer = [(frame['answered_correctly']+1 ).to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['answered_correctly'].to_numpy() [:, None]), max_len)]
train_past_answer = np.reshape(pad_sequences(train_past_answer, padding="pre"),(-1,max_len))
train_current_question = [(frame['content_id']+1 ).to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['content_id'].to_numpy() [:,None]), max_len)]
train_current_question = np.reshape(pad_sequences(train_current_question, padding="pre"),(-1,max_len))
train_prior_elaps_time = [frame['prior_question_elapsed_time'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['prior_question_elapsed_time'].to_numpy() [:,None]), max_len)]
train_prior_elaps_time = np.reshape(pad_sequences(train_prior_elaps_time, padding="pre", dtype = 'float32'),(-1,max_len))
train_time_diff = [frame['time_diff'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in train_group
for i in range(0, len(frame['time_diff'].to_numpy() [:,None]), max_len)]
train_time_diff = np.reshape(pad_sequences(train_time_diff, padding="pre", dtype = 'float32'),(-1,max_len))<split>
from keras.models import Model
from keras import layers
from keras import Input
train_answered_correcly = train_y[:,1:]
train_past_answered_correctly = train_past_answer[:,:-1]
train_past_question_answer = train_current_question[:,:-1] + train_y[:,:-1]*(train.content_id.max() +1)
 train_current_question = train_current_question[:,1:]
train_prior_elaps_time = train_prior_elaps_time[:,1:]
train_time_diff = train_time_diff[:,1:]<categorify>
time_input = Input(shape=(time_df.shape[1], time_df.shape[2]))
lstm = layers.LSTM(32 )(time_input)
 const_input = Input(shape=(const_df.shape[1],))
combine = layers.concatenate([lstm, const_input], axis=-1)
 output = layers.Dense(output_df.shape[1], activation='relu' )(combine)
 model = Model([time_input, const_input], output)
 model.compile(optimizer='adam',
loss='mean_squared_error')
 model.summary()
valid_group = valid.groupby('user_id')
 valid_y = [frame['answered_correctly'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['answered_correctly'].to_numpy() [:, None]), max_len)]
valid_y = np.reshape(pad_sequences(valid_y, padding="pre"),(-1,max_len))
valid_past_answer = [(frame['answered_correctly']+1 ).to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['answered_correctly'].to_numpy() [:, None]), max_len)]
valid_past_answer = np.reshape(pad_sequences(valid_past_answer, padding="pre"),(-1,max_len))
valid_current_question = [(frame['content_id']+1 ).to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['content_id'].to_numpy() [:,None]), max_len)]
valid_current_question = np.reshape(pad_sequences(valid_current_question, padding="pre"),(-1,max_len))
valid_prior_elaps_time = [frame['prior_question_elapsed_time'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['prior_question_elapsed_time'].to_numpy() [:,None]), max_len)]
valid_prior_elaps_time = np.reshape(pad_sequences(valid_prior_elaps_time, padding="pre", dtype = 'float32'),(-1,max_len,1))
valid_time_diff = [frame['time_diff'].to_numpy() [:, None].tolist() [i:i+max_len] for _, frame in valid_group
for i in range(0, len(frame['time_diff'].to_numpy() [:,None]), max_len)]
valid_time_diff = np.reshape(pad_sequences(valid_time_diff, padding="pre", dtype = 'float32'),(-1,max_len,1))<split>
model.fit([time_df, const_df], output_df, epochs=300, batch_size=128 )
valid_answered_correcly = valid_y[:,1:]
valid_past_answered_correctly = valid_past_answer[:,:-1]
valid_past_question_answer = valid_current_question[:,:-1] + valid_y[:,:-1]*(valid.content_id.max() +1)
 valid_current_question = valid_current_question[:,1:]
valid_prior_elaps_time = valid_prior_elaps_time[:,1:]
valid_time_diff = valid_time_diff[:,1:]<categorify>
output = model.predict([time_test_df, const_test_df])
 output.shape
def get_angles(pos, i, d_model):
angle_rates = 1 / np.power(10000,(2 *(i//2)) / np.float32(d_model))
return pos * angle_rates
def positional_encoding(position, d_model):
angle_rads = get_angles(np.arange(position)[:, np.newaxis],
np.arange(d_model)[np.newaxis, :],
d_model)
 angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])
 angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])
 pos_encoding = angle_rads[np.newaxis,...]
return tf.cast(pos_encoding, dtype=tf.float32)
 def create_padding_output_mask(seq):
seq = 1 - tf.cast(tf.math.equal(seq, 0), tf.float32)
 return seq[:, :, tf.newaxis]
def create_padding_mask(seq):
seq = tf.cast(tf.math.equal(seq, 0), tf.float32)
 return seq[:, tf.newaxis, tf.newaxis, :]
def create_look_ahead_mask(size):
mask = 1 - tf.linalg.band_part(tf.ones(( size, size)) , -1, 0)
 return mask
def scaled_dot_product_attention(q, k, v, mask):

 matmul_qk = tf.matmul(q, k, transpose_b=True)
 dk = tf.cast(tf.shape(k)[-1], tf.float32)
 scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
 if mask is not None:
scaled_attention_logits +=(mask * -1e9)
 attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
 output = tf.matmul(attention_weights, v)
 return output, attention_weights
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads):
super(MultiHeadAttention, self ).__init__()
self.num_heads = num_heads
self.d_model = d_model
assert d_model % self.num_heads == 0
self.depth = d_model // self.num_heads
self.wq = tf.keras.layers.Dense(d_model)
 self.wk = tf.keras.layers.Dense(d_model)
 self.wv = tf.keras.layers.Dense(d_model)
 self.dense = tf.keras.layers.Dense(d_model)
 def split_heads(self, x, batch_size):

 x = tf.reshape(x,(batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
 def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
q = self.wq(q)
 k = self.wk(k)
 v = self.wv(v)
 q = self.split_heads(q, batch_size)
 k = self.split_heads(k, batch_size)
 v = self.split_heads(v, batch_size)
 scaled_attention, attention_weights = scaled_dot_product_attention(
 q, k, v, mask)
 scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])
 concat_attention = tf.reshape(scaled_attention, 
(batch_size, -1, self.d_model))
output = self.dense(concat_attention)
 return output, attention_weights
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential([
tf.keras.layers.Dense(dff, activation='relu'),
tf.keras.layers.Dense(d_model)
 ])
 class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.2):
super(EncoderLayer, self ).__init__()
self.mha = MultiHeadAttention(d_model, num_heads)
 self.ffn = point_wise_feed_forward_network(d_model, dff)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = tf.keras.layers.Dropout(rate)
 self.dropout2 = tf.keras.layers.Dropout(rate)
 def call(self, x, e, training, mask):
attn_output, _ = self.mha(x, x, e, mask)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(e + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 out2 = self.layernorm2(out1 + ffn_output)
 return out2
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff,
maximum_position_encoding, rate=0.2):
super(Encoder, self ).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.pos_encoding = positional_encoding(maximum_position_encoding,
self.d_model)
 self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate)
 for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
 def call(self, x, e, training, mask):
seq_len = tf.shape(x)[1]
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x = self.enc_layers[i](x, e, training, mask)
 return x
class Transformer(tf.keras.Model):
def __init__(self, num_layers, en_d_model, en_num_heads, dff, pe_input, rate=0.2):
super(Transformer, self ).__init__()
self.encoder = Encoder(num_layers, en_d_model, en_num_heads, dff,
pe_input, rate)
 self.second_final_layer = tf.keras.layers.Dense(dff)
 self.final_layer = Dense(1,activation = 'sigmoid')
 def call(self, inp1, inp2, training, mask):
enc_output = self.encoder(inp1, inp2, training, mask)
 second_final_output = self.second_final_layer(enc_output)
 final_output = self.final_layer(second_final_output)
 return final_output<choose_model_class>
sub_test_df = test_df[test_df["Date"] > train_df["Date"].max() ]
sub_test_df = pd.concat([sub_test_df,
pd.DataFrame(output.reshape(( -1, 2)) , columns=["NewCases", "NewFatalities"], index=sub_test_df.index)],
axis=1)
 sub_test_df["NewCases"] = np.exp(sub_test_df["NewCases"])- 1
sub_test_df["NewFatalities"] = np.exp(sub_test_df["NewFatalities"])- 1
sub_test_df
num_layers = 2
d_model = 64
num_heads = 2
dff = 256
question_answer_pair_size = train_past_question_answer.max() +1
n_question = train_current_question.max() +1
pe_input = 30
def build(num_layers, d_model, num_heads, dff, question_answer_pair_size, n_question, pe_input):
masking_func = lambda inputs, previous_mask: previous_mask
en_input1 = Input(batch_shape =(None, None), name = 'question_answer_pair')
 en_input1_embed = Embedding(question_answer_pair_size, d_model )(en_input1)
 en_input2 = Input(batch_shape =(None, None, 1), name = 'other_feature1')
 en_input2_embed = Dense(d_model, input_shape =(None, None, 1))(en_input2)
 en_input3 = Input(batch_shape =(None, None, 1), name = 'other_feature2')
 en_input3_embed = Dense(d_model, input_shape =(None, None, 1))(en_input3)
 en_input_embed_sum = tf.math.add_n([en_input1_embed, en_input2_embed, en_input3_embed])
 en_input4 = Input(batch_shape =(None, None), name = 'current_question')
 en_input4_embed = Embedding(n_question, d_model )(en_input4)
 look_ahead_mask = create_look_ahead_mask(tf.shape(en_input_embed_sum)[1])
 padding_mask = create_padding_mask(en_input1)
 combined_mask = tf.maximum(look_ahead_mask, padding_mask)
 transformer = Transformer(num_layers, d_model, num_heads, dff, pe_input)
 final_output = transformer(en_input_embed_sum, en_input4_embed, True, combined_mask)
 output_mask = create_padding_output_mask(en_input1)
 output = multiply([final_output, output_mask])
 model = Model(inputs=[en_input1, en_input2, en_input3, en_input4], outputs=output)
 model.compile(optimizer = 'adam',
loss = 'binary_crossentropy',
metrics=['accuracy',AUC() ])
 return model
SAKT_model = build(num_layers, d_model, num_heads, dff, question_answer_pair_size, n_question, pe_input)
<train_model>
fixed_test_df = test_df[test_df["Date"] <= train_df["Date"].max() ].merge(train_df[train_df["Date"] >= test_df["Date"].min() ][["Province_State","Country_Region", "Date", "ConfirmedCases", "Fatalities"]],
how="left", on=["Province_State","Country_Region", "Date"])
 fixed_test_df
SAKT_model.fit([train_past_question_answer, train_prior_elaps_time, train_time_diff, train_current_question],train_answered_correcly,
validation_data=([valid_past_question_answer, valid_prior_elaps_time, valid_time_diff, valid_current_question], valid_answered_correcly), batch_size = 200,
epochs = 2, verbose = 1 )<categorify>
predict_df = pd.concat([sub_test_df, fixed_test_df] ).sort_values(["Country_Region", "Province_State", "Date"],
ascending=[True, True, True])
 predict_df
class EncoderLayer2(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.2):
super(EncoderLayer2, self ).__init__()
self.mha = MultiHeadAttention(d_model, num_heads)
 self.ffn = point_wise_feed_forward_network(d_model, dff)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = tf.keras.layers.Dropout(rate)
 self.dropout2 = tf.keras.layers.Dropout(rate)
 def call(self, x, training, mask):
attn_output, _ = self.mha(x, x, x, mask)
 attn_output = self.dropout1(attn_output, training=training)
 out1 = self.layernorm1(x + attn_output)
 ffn_output = self.ffn(out1)
 ffn_output = self.dropout2(ffn_output, training=training)
 out2 = self.layernorm2(out1 + ffn_output)
 return out2
class Encoder2(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff,
maximum_position_encoding, rate=0.2):
super(Encoder2, self ).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.pos_encoding = positional_encoding(maximum_position_encoding,
self.d_model)
 self.enc_layers = [EncoderLayer2(d_model, num_heads, dff, rate)
 for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
 def call(self, x, training, mask):
seq_len = tf.shape(x)[1]
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x = self.enc_layers[i](x, training, mask)
 return x
class DecoderLayer2(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.2):
super(DecoderLayer2, self ).__init__()
self.mha1 = MultiHeadAttention(d_model, num_heads)
 self.mha2 = MultiHeadAttention(d_model, num_heads)
 self.ffn = point_wise_feed_forward_network(d_model, dff)
 self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
 self.dropout1 = tf.keras.layers.Dropout(rate)
 self.dropout2 = tf.keras.layers.Dropout(rate)
 self.dropout3 = tf.keras.layers.Dropout(rate)
 def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)
 attn1 = self.dropout1(attn1, training=training)
 out1 = self.layernorm1(attn1 + x)
 attn2, attn_weights_block2 = self.mha2(
 enc_output, enc_output, out1, padding_mask)
 attn2 = self.dropout2(attn2, training=training)
 out2 = self.layernorm2(attn2 + out1)
 ffn_output = self.ffn(out2)
 ffn_output = self.dropout3(ffn_output, training=training)
 out3 = self.layernorm3(ffn_output + out2)
 return out3, attn_weights_block1, attn_weights_block2
class Decoder2(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff,
maximum_position_encoding, rate=0.2):
super(Decoder2, self ).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)
 self.dec_layers = [DecoderLayer2(d_model, num_heads, dff, rate)
 for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
 def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
seq_len = tf.shape(x)[1]
attention_weights = {}
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
 for i in range(self.num_layers):
x, block1, block2 = self.dec_layers[i](x, enc_output, training, look_ahead_mask, padding_mask)
 attention_weights['decoder_layer{}_block1'.format(i+1)] = block1
attention_weights['decoder_layer{}_block2'.format(i+1)] = block2
return x, attention_weights
class Transformer2(tf.keras.Model):
def __init__(self, num_layers, d_model, num_heads, dff, padding_length, rate=0.2):
super(Transformer2, self ).__init__()
self.encoder = Encoder2(num_layers, d_model, num_heads, dff, padding_length)
 self.decoder = Decoder2(num_layers, d_model, num_heads, dff, padding_length)
 self.second_final_layer = tf.keras.layers.Dense(dff)
 self.final_layer = Dense(1,activation = 'sigmoid')
 def call(self, inp1, inp2, training, en_combined_mask, de_look_ahead_mask, de_padding_mask):
enc_output = self.encoder(inp1, training, en_combined_mask)
 dec_output, attention_weights = self.decoder(
 inp2, enc_output, training, de_look_ahead_mask, de_padding_mask)
 second_final_output = self.second_final_layer(dec_output)
 final_output = self.final_layer(second_final_output)
 return final_output<choose_model_class>
predict_df = predict_df.reset_index()
for i in range(len(predict_df)) :
if pd.isnull(predict_df.iloc[i]["ConfirmedCases"]):
predict_df.loc[i, "ConfirmedCases"] = predict_df.iloc[i - 1]["ConfirmedCases"] + predict_df.iloc[i]["NewCases"]
if pd.isnull(predict_df.iloc[i]["Fatalities"]):
predict_df.loc[i, "Fatalities"] = predict_df.iloc[i - 1]["Fatalities"] + predict_df.iloc[i]["NewFatalities"]
predict_df
num_layers = 1
d_model = 64
num_heads = 2
dff = 256
n_question = train_current_question.max() +1
n_answer = 3
pe_input = 30
def build(num_layers, d_model, num_heads, dff, n_question, n_answer, pe_input):
en_input1 = Input(batch_shape =(None, None), name = 'current_question')
 en_input1_embed = Embedding(n_question, d_model )(en_input1)
 en_look_ahead_mask = create_look_ahead_mask(tf.shape(en_input1)[1])
 en_padding_mask = create_padding_mask(en_input1)
 en_combined_mask = tf.maximum(en_look_ahead_mask, en_padding_mask)
 de_input2 = Input(batch_shape =(None, None), name = 'past_answer')
 de_input2_embed = Embedding(n_answer, d_model )(de_input2)
 de_input3 = Input(batch_shape =(None, None, 1), name = 'other_feature1')
 de_input3_embed = Dense(d_model, input_shape =(None, None, 1))(de_input3)
 de_input4 = Input(batch_shape =(None, None, 1), name = 'other_feature2')
 de_input4_embed = Dense(d_model, input_shape =(None, None, 1))(de_input4)
 de_input = tf.math.add_n([de_input2_embed, de_input3_embed, de_input4_embed])
 transformer = Transformer2(num_layers, d_model, num_heads, dff, pe_input)
 final_output = transformer(en_input1_embed, de_input, True, en_combined_mask, en_combined_mask, en_combined_mask)
 output_mask = create_padding_output_mask(en_input1)
 model = Model(inputs=[en_input1, de_input2, de_input3, de_input4], outputs=final_output)
 model.compile(optimizer = 'adam',
loss = 'binary_crossentropy',
metrics=['accuracy',AUC() ])
 return model
SAINT_model = build(num_layers, d_model, num_heads, dff, n_question, n_answer, pe_input )<train_model>
predict_df[["ForecastId", "ConfirmedCases", "Fatalities"]].to_csv("submission.csv", index=False )
<concatenate><EOS> 
%matplotlib inline