Kaggle competition: COVID19 Global Forecasting (Week 4)
As = np.concatenate([As[:,:,:,None], Ss, Ds], axis = 3 ).astype(np.float32)
 As_pub = np.concatenate([As_pub[:,:,:,None], Ss_pub, Ds_pub], axis = 3 ).astype(np.float32)
 As_pri = np.concatenate([As_pri[:,:,:,None], Ss_pri, Ds_pri], axis = 3 ).astype(np.float32)
 del Ss, Ds, Ss_pub, Ds_pub, Ss_pri, Ds_pri
As.shape, As_pub.shape, As_pri.shape<categorify>
warnings.filterwarnings("ignore" )
def return_ohe(n, i):
tmp = [0] * n
tmp[i] = 1
return tmp
def get_input(train):
mapping = {}
vocab = ["A", "G", "C", "U"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_node = np.stack(train["sequence"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = ["S", "M", "I", "B", "H", "E", "X"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_loop = np.stack(train["predicted_loop_type"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 mapping = {}
vocab = [".", "(", ")"]
for i, s in enumerate(vocab):
mapping[s] = return_ohe(len(vocab), i)
 X_structure = np.stack(train["structure"].apply(lambda x : list(map(lambda y : mapping[y], list(x)))))
 X_node = np.concatenate([X_node, X_loop], axis = 2)
 a = np.sum(X_node *(2 ** np.arange(X_node.shape[2])[None, None, :]), axis = 2)
 vocab = sorted(set(a.flatten()))
print(vocab)
 ohes = []
for v in vocab:
ohes.append(a == v)
 ohes = np.stack(ohes, axis = 2)
 X_node = np.concatenate([X_node, ohes], axis = 2 ).astype(np.float32)
 print(X_node.shape)
 return X_node
X_node = get_input(train)
 X_node_pub = get_input(test_pub)
 X_node_pri = get_input(test_pri )<compute_test_metric>
df_train=pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 df_test=pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 df_sub=pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 print(df_train.shape)
 print(df_test.shape)
 print(df_sub.shape )
def mcrmse(t, p, seq_len_target = seq_len_target):
score = np.mean(np.sqrt(np.mean(( p - y_va)** 2, axis = 2)) [:, :seq_len_target])
 return score
def mcrmse_loss(t, y, seq_len_target = seq_len_target):
t = t[:, :seq_len_target]
y = y[:, :seq_len_target]
loss = tf.reduce_mean(tf.sqrt(tf.reduce_mean(( t - y)** 2, axis = 2)))
 return loss
def attention(x_inner, x_outer, n_factor, dropout):
x_Q = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_inner)
 x_K = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_V = L.Conv1D(n_factor, 1, activation='linear',
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(x_outer)
 x_KT = L.Permute(( 2, 1))(x_K)
 res = L.Lambda(lambda c: K.batch_dot(c[0], c[1])/ np.sqrt(n_factor))([x_Q, x_KT])
 res = tf.expand_dims(res, axis = 3)
 res = L.Conv2D(16, 3, 1, padding = "same", activation = "relu" )(res)
 res = L.Conv2D(1, 3, 1, padding = "same", activation = "relu" )(res)
 res = tf.squeeze(res, axis = 3)
 att = L.Lambda(lambda c: K.softmax(c, axis=-1))(res)
 att = L.Lambda(lambda c: K.batch_dot(c[0], c[1]))([att, x_V])
 return att
def multi_head_attention(x, y, n_factor, n_head, dropout):
if n_head == 1:
att = attention(x, y, n_factor, dropout)
 else:
n_factor_head = n_factor // n_head
heads = [attention(x, y, n_factor_head, dropout)for i in range(n_head)]
att = L.Concatenate()(heads)
 att = L.Dense(n_factor,
kernel_initializer='glorot_uniform',
bias_initializer='glorot_uniform',
)(att)
 x = L.Add()([x, att])
 x = L.LayerNormalization()(x)
 if dropout > 0:
x = L.Dropout(dropout )(x)
 return x
def res(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.LeakyReLU()(h)
 h = L.Dropout(rate )(h)
 return L.Add()([x, h])
 def forward(x, unit, kernel = 3, rate = 0.1):
h = L.Conv1D(unit, kernel, 1, padding = "same", activation = None )(x)
 h = L.LayerNormalization()(h)
 h = L.Dropout(rate )(h)
 h = L.LeakyReLU()(h)
 h = res(h, unit, kernel, rate)
 return h
def adj_attn(x, adj, unit, n = 2, rate = 0.1):
x_a = x
x_as = []
for i in range(n):
x_a = forward(x_a, unit)
 x_a = tf.matmul(adj, x_a)
 x_as.append(x_a)
 if n == 1:
x_a = x_as[0]
else:
x_a = L.Concatenate()(x_as)
 x_a = forward(x_a, unit)
 return x_a
def get_base(config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 adj_learned = L.Dense(1, "relu" )(adj)
 adj_all = L.Concatenate(axis = 3 )([adj, adj_learned])
 xs = []
xs.append(node)
 x1 = forward(node, 128, kernel = 3, rate = 0.0)
 x2 = forward(x1, 64, kernel = 6, rate = 0.0)
 x3 = forward(x2, 32, kernel = 15, rate = 0.0)
 x4 = forward(x3, 16, kernel = 30, rate = 0.0)
 x = L.Concatenate()([x1, x2, x3, x4])
 for unit in [64, 32, 16, 8]:
x_as = []
for i in range(adj_all.shape[3]):
x_a = adj_attn(x, adj_all[:, :, :, i], unit, rate = 0.0)
 x_as.append(x_a)
 x_c = forward(x, unit, kernel = 30)
 x = L.Concatenate()(x_as + [x_c])
 x = forward(x, unit)
 x = multi_head_attention(x, x, unit, 4, 0.0)
 xs.append(x)
 x = L.Concatenate()(xs)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 return model
def get_ae_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([L.SpatialDropout1D(0.3 )(node), adj])
 x = forward(x, 64, rate = 0.3)
 p = L.Dense(X_node.shape[2], "sigmoid" )(x)
 loss = - tf.reduce_mean(20 * node * tf.math.log(p + 1e-4)+(1 - node)* tf.math.log(1 - p + 1e-4))
model = tf.keras.Model(inputs = [node, adj], outputs = [loss])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = lambda t, y : y)
 return model
def get_model(base, config):
node = tf.keras.Input(shape =(None, X_node.shape[2]), name = "node")
 adj = tf.keras.Input(shape =(None, None, As.shape[3]), name = "adj")
 x = base([node, adj])
 x = forward(x, 128, rate = 0.4)
 x = L.Dense(5, None )(x)
 model = tf.keras.Model(inputs = [node, adj], outputs = [x])
 opt = get_optimizer()
model.compile(optimizer = opt, loss = mcrmse_loss)
 return model
def get_optimizer() :
adam = tf.optimizers.Adam()
return adam<train_model>
print(f"Unique Countries: {len(df_train.Country_Region.unique())}" )
config = {}
if ae_epochs > 0:
base = get_base(config)
 ae_model = get_ae_model(base, config)
 for i in range(ae_epochs//ae_epochs_each):
print(f"------ {i} ------")
 print("--- train ---")
 ae_model.fit([X_node, As], [X_node[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- public ---")
 ae_model.fit([X_node_pub, As_pub], [X_node_pub[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 print("--- private ---")
 ae_model.fit([X_node_pri, As_pri], [X_node_pri[:,0]],
epochs = ae_epochs_each,
batch_size = ae_batch_size)
 gc.collect()
print("****** save ae model ******")
 base.save_weights("./base_ae" )<split>
print(f"Unique Regions: {df_train.shape[0]/len(df_train.Date.unique())}" )
cv = StratifiedKFold(5, shuffle=True, random_state=42)
 scores = []
preds = np.zeros([len(X_node), X_node.shape[1], 5])
 for i,(tr_idx, va_idx)in enumerate(cv.split(X_node, train["SN_filter"])) :
print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 print(f"------ fold {i} start -----")
 X_node_tr = X_node[tr_idx]
X_node_va = X_node[va_idx]
As_tr = As[tr_idx]
As_va = As[va_idx]
y_tr = y[tr_idx]
y_va = y[va_idx]
base = get_base(config)
 if ae_epochs > 0:
print("****** load ae model ******")
 base.load_weights("./base_ae")
 model = get_model(base, config)
 if pretrain_dir is not None:
d = f"./model{i}"
print(f"--- load from {d} ---")
 model.load_weights(d)
 for epochs, batch_size in zip(epochs_list, batch_size_list):
print(f"epochs : {epochs}, batch_size : {batch_size}")
 model.fit([X_node_tr, As_tr], [y_tr],
validation_data=([X_node_va, As_va], [y_va]),
epochs = epochs,
batch_size = batch_size, validation_freq = 3)
 model.save_weights(f"./model{i}")
 p = model.predict([X_node_va, As_va])
 scores.append(mcrmse(y_va, p))
print(f"fold {i}: mcrmse {scores[-1]}")
 preds[va_idx] = p
if one_fold:
break
pd.to_pickle(preds, "oof.pkl" )<predict_on_test>
df_train.Country_Region.value_counts()
p_pub = 0
p_pri = 0
for i in range(5):
model.load_weights(f"./model{i}")
 p_pub += model.predict([X_node_pub, As_pub])/ 5
p_pri += model.predict([X_node_pri, As_pri])/ 5
if one_fold:
p_pub *= 5
p_pri *= 5
break
for i, target in enumerate(targets):
test_pub[target] = [list(p_pub[k, :, i])for k in range(p_pub.shape[0])]
test_pri[target] = [list(p_pri[k, :, i])for k in range(p_pri.shape[0])]<save_to_csv>
print(f"Number of rows without Country_Region : {df_train.Country_Region.isna().sum() }" )
preds_ls = []
for df, preds in [(test_pub, p_pub),(test_pri, p_pri)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=targets)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls)
 preds_df.to_csv("submission.csv", index = False)
 preds_df.head()<set_options>
df_train["UniqueRegion"]=df_train.Country_Region
df_train.UniqueRegion[df_train.Province_State.isna() ==False]=df_train.Province_State+" , "+df_train.Country_Region
df_train[df_train.Province_State.isna() ==False]
pd.set_option('max_columns', 50)
 plt.style.use('bmh')
 color_cycle = cycle(plt.rcParams['axes.prop_cycle'].by_key() ['color'] )<load_from_csv>
df_train.drop(labels=["Id","Province_State","Country_Region"], axis=1, inplace=True )
osj = os.path.join
INPUT_DIR = '.. /input/m5-forecasting-accuracy/'
train = pd.read_csv(osj(INPUT_DIR, 'sales_train_validation.csv'))
price = pd.read_csv(osj(INPUT_DIR, 'sell_prices.csv'))
calender = pd.read_csv(osj(INPUT_DIR, 'calendar.csv'))
sample_submit = pd.read_csv(osj(INPUT_DIR, 'sample_submission.csv'))<data_type_conversions>
test_dates=list(df_test.Date.unique())
 print(f"Period :{len(df_test.Date.unique())} days")
 print(f"From : {df_test.Date.min() } To : {df_test.Date.max() }" )
calender['date'] = pd.to_datetime(calender['date'] )<define_variables>
print(f"Total Regions : {df_test.shape[0]/43}" )
d_cols = [c for c in train.columns if 'd_' in c]<merge>
df_test["UniqueRegion"]=df_test.Country_Region
df_test.UniqueRegion[df_test.Province_State.isna() ==False]=df_test.Province_State+" , "+df_test.Country_Region
df_test.drop(labels=["Province_State","Country_Region"], axis=1, inplace=True )
past_sales = train.set_index('id')[d_cols].T.merge(calender.set_index('d')['date'],
left_index=True,
right_index=True,
validate='1:1' ).set_index('date' )<create_dataframe>
len(df_test.UniqueRegion.unique() )
def make_total_sales_lowess(p):
total_sales = pd.DataFrame(p.sum(axis=1), columns=['total sales'])
 total_sales_noXmas = total_sales.drop(index=[dt.datetime(2011,12,25),
dt.datetime(2012,12,25),
dt.datetime(2013,12,25),
dt.datetime(2014,12,25),
dt.datetime(2015,12,25)])
 df = total_sales_noXmas
lowess = sm.nonparametric.lowess(df['total sales'], df.index, frac=.3)
 lowess_x = list(zip(*lowess)) [0]
lowess_y = list(zip(*lowess)) [1]
f = interp1d(lowess_x, lowess_y, bounds_error=False)
 new_lowess_x = df.index
new_lowess_y = f(new_lowess_x)
 total_sales_lowess = total_sales_noXmas
total_sales_lowess['lowess'] = new_lowess_y
total_sales_lowess['total sales-lowess'] = total_sales_lowess['total sales'] - total_sales_lowess['lowess']
return total_sales_lowess<groupby>
only_train_dates=set(train_dates)-set(test_dates)
 print("Only train dates : ",len(only_train_dates))
intersection_dates=set(test_dates)&set(train_dates)
 print("Intersection dates : ",len(intersection_dates))
only_test_dates=set(test_dates)-set(train_dates)
 print("Only Test dates : ",len(only_test_dates))
total_sales_lowess = make_total_sales_lowess(past_sales )<load_from_csv>
df_test_temp=pd.DataFrame()
df_test_temp["Date"]=df_test.Date
df_test_temp["ConfirmedCases"]=0.0
df_test_temp["Fatalities"]=0.0
df_test_temp["UniqueRegion"]=df_test.UniqueRegion
df_test_temp["Delta"]=1.0
submission = pd.read_csv('.. /input/m5-three-shades-of-dark-darker-magic/submission_v1.csv' )<feature_engineering>
%%time
final_df=pd.DataFrame(columns=["Date","ConfirmedCases","Fatalities","UniqueRegion"])
 for region in df_train.UniqueRegion.unique() :
df_temp=df_train[df_train.UniqueRegion==region].reset_index()
df_temp["Delta"]=1.0
size_train=df_temp.shape[0]
for i in range(1,df_temp.shape[0]):
if(df_temp.ConfirmedCases[i-1]>0):
df_temp.Delta[i]=df_temp.ConfirmedCases[i]/df_temp.ConfirmedCases[i-1]
n=7
delta_list=df_temp.tail(n ).Delta
delta_avg=df_temp.tail(n ).Delta.mean()
death_rate=df_temp.tail(1 ).Fatalities.sum() /df_temp.tail(1 ).ConfirmedCases.sum()
df_test_app=df_test_temp[df_test_temp.UniqueRegion==region]
df_test_app=df_test_app[df_test_app.Date>df_temp.Date.max() ]
X=np.arange(1,n+1 ).reshape(-1,1)
 Y=delta_list
model=LinearRegression()
model.fit(X,Y)
 df_temp=pd.concat([df_temp,df_test_app])
 df_temp=df_temp.reset_index()
for i in range(size_train, df_temp.shape[0]):
n=n+1
damper=df_temp.Delta[i-5]
pred=max(1,model.predict(np.array([n] ).reshape(-1,1)) [0])
 df_temp.Delta[i]=(damper+pred+delta_avg)/3
for i in range(size_train, df_temp.shape[0]):
df_temp.ConfirmedCases[i]=round(df_temp.ConfirmedCases[i-1]*df_temp.Delta[i],0)
 df_temp.Fatalities[i]=round(death_rate*df_temp.ConfirmedCases[i],0)
 size_test=df_temp.shape[0]-df_test_temp[df_test_temp.UniqueRegion==region].shape[0]
df_temp=df_temp.iloc[size_test:,:]
df_temp=df_temp[["Date","ConfirmedCases","Fatalities","UniqueRegion","Delta"]]
final_df=pd.concat([final_df,df_temp], ignore_index=True)
 final_df.shape
teams_now = 3215
teams_before = 3186
submission.loc[submission['id'].str.contains('CA_1'), 'F1'] *= 1.009
submission.loc[submission['id'].str.contains('CA_2'), 'F1'] *= 1.02
submission.loc[submission['id'].str.contains('CA_3'), 'F1'] *= 1.009
submission.loc[submission['id'].str.contains('CA_4'), 'F1'] *= 1.011
submission.loc[submission['id'].str.contains('TX_1'), 'F1'] *= 1.01
submission.loc[submission['id'].str.contains('TX_2'), 'F1'] *= 1.011
submission.loc[submission['id'].str.contains('TX_3'), 'F1'] *= 1.011
submission.loc[submission['id'].str.contains('WI_1'), 'F1'] *= 1.015
submission.loc[submission['id'].str.contains('WI_2'), 'F1'] *= 1.02
submission.loc[submission['id'].str.contains('WI_3'), 'F1'] *= 1.015
submission['F1'].sum()<feature_engineering>
df_sub.Fatalities=final_df.Fatalities
df_sub.ConfirmedCases=final_df.ConfirmedCases
df_sub.to_csv("submission.csv", index=None )