Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<init_hyperparams>
warnings.filterwarnings('ignore')
 sub0 = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv', parse_dates=['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv', parse_dates=['Date'])
 train.shape, test.shape
pretrain_dir = None
one_fold = False
run_test = False
denoise = True
ae_epochs = 20
ae_epochs_each = 5
ae_batch_size = 32
epochs_list = [50, 30, 20, 20, 15, 15]
batch_size_list = [8, 16, 32, 64, 128, 256]
if pretrain_dir is not None:
for d in glob.glob(pretrain_dir + "*"):
shutil.copy(d, ".")
 %matplotlib inline<load_from_csv>
know = test[test.Date <= train.Date.max() ]
not_know = test[test.Date > train.Date.max() ]
know = know.merge(train, on=['Date','Country_Region','Province_State'], how='left')
 know.head()
train = pd.read_json("/kaggle/input/stanford-covid-vaccine/train.json",lines=True)
 if denoise:
train = train[train.signal_to_noise > 1].reset_index(drop = True)
 test = pd.read_json("/kaggle/input/stanford-covid-vaccine/test.json",lines=True)
 test_pub = test[test["seq_length"] == 107]
test_pri = test[test["seq_length"] == 130]
sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv")
 if run_test:
train = train[:30]
test_pub = test_pub[:30]
test_pri = test_pri[:30]
As = []
for id in tqdm(train["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As.append(a)
 As = np.array(As)
 As_pub = []
for id in tqdm(test_pub["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pub.append(a)
 As_pub = np.array(As_pub)
 As_pri = []
for id in tqdm(test_pri["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pri.append(a)
 As_pri = np.array(As_pri )<load_from_csv>
train['days'] =(train['Date'] - train.Date.min() ).dt.days
train['location'] = train['Country_Region'] + train['Province_State'].fillna('')
 not_know['days'] =(not_know['Date'] - train.Date.min() ).dt.days
not_know['location'] = not_know['Country_Region'] + not_know['Province_State'].fillna('' )
aug_df = pd.read_csv('.. /input/augmented-data-for-stanford-covid-vaccine/aug_data1.csv')
 def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train)
 test = aug_data(test)
 print(train.shape )<load_pretrained>
def get_param(loc):
_ = train[train.location == loc]
_['diff'] = _.ConfirmedCases.diff()
_['pct'] = _.ConfirmedCases.pct_change()
initial_speed = _.loc[_.ConfirmedCases.diff().argmax() ,'pct']
initial_mid = _.loc[_.ConfirmedCases.diff().argmax() , 'days']
initial_max = _.ConfirmedCases.max() * 2
return initial_speed, initial_mid, initial_max
get_param('Italy' )
As = []
for id in tqdm(train["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As.append(a)
 As = np.array(As)
 As_pub = []
for id in tqdm(test_pub["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pub.append(a)
 As_pub = np.array(As_pub)
 As_pri = []
for id in tqdm(test_pri["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pri.append(a)
 As_pri = np.array(As_pri )<concatenate>
loc_list = train.location.unique()
all_param = pd.DataFrame(index=loc_list, columns=['k','x_0','y_max'])
 for loc in loc_list:
_ = train[train.location == loc]
nn = not_know[not_know.location == loc]
initial_max = _.ConfirmedCases.max() *2
x = _.days
y1 = _.ConfirmedCases
try:
popt, pcov = opt.curve_fit(log_curve, x, y1, p0 = get_param(loc))
popt[2] = max(popt[2],get_param(loc)[2]/2)
 except RuntimeError:
popt = get_param(loc)
 y1_hat = log_curve(nn.days, *popt)
 not_know.loc[y1_hat.index, 'ConfirmedCases'] = y1_hat
all_param.loc[loc, 'k'] = popt[0]
all_param.loc[loc, 'x_0'] = popt[1]
all_param.loc[loc, 'y_max'] = popt[2]
print('Done!' )
targets = list(sub.columns[1:])
 print(targets)
 y_train = []
seq_len = train["seq_length"].iloc[0]
seq_len_target = train["seq_scored"].iloc[0]
ignore = -10000
ignore_length = seq_len - seq_len_target
for target in targets:
y = np.vstack(train[target])
 dummy = np.zeros([y.shape[0], ignore_length])+ ignore
y = np.hstack([y, dummy])
 y_train.append(y)
 y = np.stack(y_train, axis = 2)
 y.shape<categorify>
latest = train[train.Date == train.Date.max() ]
latest['DeathRate'] = latest['Fatalities'] / latest['ConfirmedCases']
not_know2 = not_know.merge(latest[['location','DeathRate']], on='location')
 not_know2['Fatalities'] = not_know2['ConfirmedCases'] * not_know2['DeathRate']
def get_structure_adj(train):
Ss = []
for i in tqdm(range(len(train))):
seq_length = train["seq_length"].iloc[i]
structure = train["structure"].iloc[i]
sequence = train["sequence"].iloc[i]
cue = []
a_structures = { 
("A", "U"): np.zeros([seq_length, seq_length]), 
("C", "G"): np.zeros([seq_length, seq_length]), 
("U", "G"): np.zeros([seq_length, seq_length]), 
("U", "A"): np.zeros([seq_length, seq_length]), 
("G", "C"): np.zeros([seq_length, seq_length]), 
("G", "U"): np.zeros([seq_length, seq_length]),
}
a_structure = np.zeros([seq_length, seq_length])
 for i in range(seq_length):
if structure[i] == "(":
cue.append(i)
 elif structure[i] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[i])][start, i] = 1
a_structures[(sequence[i], sequence[start])][i, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis = 2)
 a_strc = np.sum(a_strc, axis = 2, keepdims = True)
 Ss.append(a_strc)
 Ss = np.array(Ss)
 print(Ss.shape)
 return Ss
Ss = get_structure_adj(train)
 Ss_pub = get_structure_adj(test_pub)
 Ss_pri = get_structure_adj(test_pri )<data_type_conversions>
not_know2['ConfirmedCases'] = not_know2['ConfirmedCases'].round()
not_know2['Fatalities'] = not_know2['Fatalities'].round()
As = np.concatenate([As[:,:,:,None], Ss, Ds], axis = 3 ).astype(np.float32)
 As_pub = np.concatenate([As_pub[:,:,:,None], Ss_pub, Ds_pub], axis = 3 ).astype(np.float32)
 As_pri = np.concatenate([As_pri[:,:,:,None], Ss_pri, Ds_pri], axis = 3 ).astype(np.float32)
 del Ss, Ds, Ss_pub, Ds_pub, Ss_pri, Ds_pri
gc.collect()
As.shape, As_pub.shape, As_pri.shape<categorify>
not_know2.loc[not_know2.Date=='2020-04-30',
['location','ConfirmedCases','Fatalities']].sort_values('ConfirmedCases', ascending=False ).head(20 )
def return_ohe(n, i):
tmp = [0] * n
tmp[i] = 1
return tmp
def get_input(train):
mapping = {}
vocab = ["A", "G", "C", "U"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_node = np.stack(train["sequence"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = ["S", "M", "I", "B", "H", "E", "X"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_loop = np.stack(train["predicted_loop_type"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = [".", "(", ")"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_structure = np.stack(train["structure"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 X_node = np.concatenate([X_node, X_loop], axis = 2)
 a = np.sum(X_node *(2 ** np.arange(X_node.shape[2])[None, None, :]), axis = 2)
 vocab = sorted(set(a.flatten()))
print(vocab)
 ohes = []
for v in vocab:
ohes.append(a == v)
 ohes = np.stack(ohes, axis = 2)
 X_node = np.concatenate([X_node, ohes], axis = 2 ).astype(np.float32)
 print(X_node.shape)
 return X_node
X_node = get_input(train)
 X_node_pub = get_input(test_pub)
 X_node_pri = get_input(test_pri )<compute_test_metric>
sub1 = pd.concat([know[['ForecastId', 'ConfirmedCases','Fatalities']],
not_know2[['ForecastId', 'ConfirmedCases','Fatalities']]])
 sub1=sub1.sort_values('ForecastId' ).reset_index(drop=True)
 sub1.to_csv('submission.csv', index=False )
def mcrmse(t, p, seq_len_target = seq_len_target):
score = np.mean(np.sqrt(np.mean(( p - y_va)** 2, axis = 2)) [:, :seq_len_target])
 return score
def mcrmse_loss(t, y, seq_len_target = seq_len_target):
t = t[:, :seq_len_target]
y = y[:, :seq_len_target]
loss = tf.reduce_mean(tf.sqrt(tf.reduce_mean(( t - y)** 2, axis = 2)))
 return loss
def attention(x_inner, x_outer, n_factor, dropout):
x_Q = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_inner)
 x_K = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_V = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_KT = L.Permute(( 2, 1))(x_K)
 res = L.Lambda(lambda c: K.batch_dot(c[0], c[1])/ np.sqrt(n_factor))([x_Q, x_KT])
 res = tf.expand_dims(res, axis = 3)
 res = L.Conv2D(16, 3, 1, padding = "same", activation = "relu" )(res)
 res = L.Conv2D(1, 3, 1, padding = "same", activation = "relu" )(res)
 res = tf.squeeze(res, axis = 3)
 att = L.Lambda(lambda c: K.softmax(c, axis=-1))(res)
 att = L.Lambda(lambda c: K.batch_dot(c[0], c[1]))([att, x_V])
 return att
def multi_head_attention(x, y, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x, y, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x, y, n_factor_head, dropout)for i in range(n_head)]
att = L.Concatenate()(heads)
 att = L.Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x = L.Add()([x, att])
 x = L.LayerNormalization()(x)
 if dropout > 0:
x = L.Dropout(dropout )(x)
 return x
def res(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.LeakyReLU()(h)
 h = L.Dropout(rate )(h)
 return L.Add()([x, h])
 def forward(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.Dropout(rate )(h)
 h = L.LeakyReLU()(h)
 h = res(h, unit, kernel, rate)
 return h
def adj_attn(x, adj, unit, n = 2, rate = 0.1):
x_a = x
x_as = []
for i in range(n):
x_a = forward(x_a, unit)
 x_a = tf.matmul(adj, x_a)
 x_as.append(x_a)
 if n == 1:
x_a = x_as[0]
else:
x_a = L.Concatenate()(x_as)
 x_a = forward(x_a, unit)
 return x_a
def get_base(config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 adj_learned = L.Dense(1, "relu" )(adj)
 adj_all = L.Concatenate(axis = 3 )([adj, adj_learned])
 xs = []
xs.append(node)
 x1 = forward(node, 128, kernel = 3, rate = 0.0)
 x2 = forward(x1, 64, kernel = 6, rate = 0.0)
 x3 = forward(x2, 32, kernel = 15, rate = 0.0)
 x4 = forward(x3, 16, kernel = 30, rate = 0.0)
 x = L.Concatenate()([x1, x2, x3, x4])
 for unit in [64, 32, 16, 8]:
x_as = []
for i in range(adj_all.shape[3]):
x_a = adj_attn(x, adj_all[:, :, :, i], unit, rate = 0.0)
 x_as.append(x_a)
 x_c = forward(x, unit, kernel = 30)
 x = L.Concatenate()(x_as + [x_c])
 x = forward(x, unit)
 x = multi_head_attention(x, x, unit, 4, 0.0)
 xs.append(x)
 x = L.Concatenate()(xs)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 return model
def get_ae_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([L.SpatialDropout1D(0.3 )(node), adj])
 x = forward(x, 64, rate = 0.3)
 p = L.Dense(X_node.shape[2], "sigmoid" )(x)
 loss = - tf.reduce_mean(20 * node * tf.math.log(p + 1e-4)+(1 - node)* tf.math.log(1 - p + 1e-4))
model = tf.keras.Model(inputs = [node, adj], outputs = [loss])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = lambda t, y : y)
 return model
def get_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([node, adj])
 x = forward(x, 128, rate = 0.4)
 x = L.Dense(5, None )(x)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = mcrmse_loss)
 return model
def get_optimizer() :
adam = tf.optimizers.Adam()
return adam<train_model>
warnings.filterwarnings('ignore')
 sub0 = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv', parse_dates=['Date'])
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv', parse_dates=['Date'])
 train.shape, test.shape
<load_pretrained>
know = test[test.Date <= train.Date.max() ]
not_know = test[test.Date > train.Date.max() ]
know = know.merge(train, on=['Date','Country_Region','Province_State'], how='left')
 know.head()
config = {}
base = get_base(config)
 model = get_model(base, config )<define_variables>
train['days'] =(train['Date'] - train.Date.min() ).dt.days
train['location'] = train['Country_Region'] + train['Province_State'].fillna('')
 not_know['days'] =(not_know['Date'] - train.Date.min() ).dt.days
not_know['location'] = not_know['Country_Region'] + not_know['Province_State'].fillna('' )
ls.. /input/epoch50/<predict_on_test>
def get_param(loc):
_ = train[train.location == loc]
_['diff'] = _.ConfirmedCases.diff()
_['pct'] = _.ConfirmedCases.pct_change()
initial_speed = _.loc[_.ConfirmedCases.diff().argmax() ,'pct']
initial_mid = _.loc[_.ConfirmedCases.diff().argmax() , 'days']
initial_max = _.ConfirmedCases.max() * 2
return initial_speed, initial_mid, initial_max
get_param('Italy' )
p_pub = 0
p_pri = 0
for i in range(2):
model.load_weights(f".. /input/augd43/model{i}")
 p_pub += model.predict([X_node_pub, As_pub])/ 2
p_pri += model.predict([X_node_pri, As_pri])/ 2
if one_fold:
p_pub *= 2
p_pri *= 2
break
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]<save_to_csv>
loc_list = train.location.unique()
all_param = pd.DataFrame(index=loc_list, columns=['k','x_0','y_max'])
 for loc in loc_list:
_ = train[train.location == loc]
nn = not_know[not_know.location == loc]
initial_max = _.ConfirmedCases.max() *2
x = _.days
y1 = _.ConfirmedCases
try:
popt, pcov = opt.curve_fit(log_curve, x, y1, p0 = get_param(loc))
popt[2] = max(popt[2],get_param(loc)[2]/2)
 except RuntimeError:
popt = get_param(loc)
 y1_hat = log_curve(nn.days, *popt)
 not_know.loc[y1_hat.index, 'ConfirmedCases'] = y1_hat
all_param.loc[loc, 'k'] = popt[0]
all_param.loc[loc, 'x_0'] = popt[1]
all_param.loc[loc, 'y_max'] = popt[2]
print('Done!' )
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df.to_csv("submission.csv", index = False )<init_hyperparams>
latest = train[train.Date == train.Date.max() ]
latest['DeathRate'] = latest['Fatalities'] / latest['ConfirmedCases']
not_know2 = not_know.merge(latest[['location','DeathRate']], on='location')
 not_know2['Fatalities'] = not_know2['ConfirmedCases'] * not_know2['DeathRate']
pretrain_dir = None
one_fold = False
run_test = False
denoise = True
ae_epochs = 20
ae_epochs_each = 5
ae_batch_size = 32
epochs_list = [50, 30, 20, 20, 15, 15]
batch_size_list = [8, 16, 32, 64, 128, 256]
if pretrain_dir is not None:
for d in glob.glob(pretrain_dir + "*"):
shutil.copy(d, ".")
 %matplotlib inline<load_from_csv>
not_know2['ConfirmedCases'] = not_know2['ConfirmedCases'].round()
not_know2['Fatalities'] = not_know2['Fatalities'].round()
train = pd.read_json("/kaggle/input/stanford-covid-vaccine/train.json",lines=True)
 if denoise:
train = train[train.signal_to_noise > 1].reset_index(drop = True)
 test = pd.read_json("/kaggle/input/stanford-covid-vaccine/test.json",lines=True)
 test_pub = test[test["seq_length"] == 107]
test_pri = test[test["seq_length"] == 130]
sub = pd.read_csv("/kaggle/input/stanford-covid-vaccine/sample_submission.csv")
 if run_test:
train = train[:30]
test_pub = test_pub[:30]
test_pri = test_pri[:30]
As = []
for id in tqdm(train["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As.append(a)
 As = np.array(As)
 As_pub = []
for id in tqdm(test_pub["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pub.append(a)
 As_pub = np.array(As_pub)
 As_pri = []
for id in tqdm(test_pri["id"]):
a = np.load(f"/kaggle/input/stanford-covid-vaccine/bpps/{id}.npy")
 As_pri.append(a)
 As_pri = np.array(As_pri )<concatenate>
not_know2.loc[not_know2.Date=='2020-04-30',
['location','ConfirmedCases','Fatalities']].sort_values('ConfirmedCases', ascending=False ).head(20 )
targets = list(sub.columns[1:])
 print(targets)
 y_train = []
seq_len = train["seq_length"].iloc[0]
seq_len_target = train["seq_scored"].iloc[0]
ignore = -10000
ignore_length = seq_len - seq_len_target
for target in targets:
y = np.vstack(train[target])
 dummy = np.zeros([y.shape[0], ignore_length])+ ignore
y = np.hstack([y, dummy])
 y_train.append(y)
 y = np.stack(y_train, axis = 2)
 y.shape<categorify>
sub1 = pd.concat([know[['ForecastId', 'ConfirmedCases','Fatalities']],
not_know2[['ForecastId', 'ConfirmedCases','Fatalities']]])
 sub1=sub1.sort_values('ForecastId' ).reset_index(drop=True)
 sub1.to_csv('submission.csv', index=False )