Kaggle competition: COVID19 Global Forecasting (Week 4)
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
return np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
 )<define_variables>
import datetime
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import mean_squared_log_error
from math import sqrt
from sklearn.preprocessing import PolynomialFeatures
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
train_inputs = preprocess_inputs(train)
 train_labels = np.array(train[target_cols].values.tolist() ).transpose(( 0, 2, 1))<compute_test_metric>
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv');
df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv');
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=len(target_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score<choose_model_class>
df_train['Date'] = pd.to_datetime(df_train['Date'], format = '%Y-%m-%d');
df_test['Date'] = pd.to_datetime(df_test['Date'], format = '%Y-%m-%d');
def gru_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def lstm_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def build_model(gru=1, seq_len=107, pred_len=68, dropout=0.4,
embed_dim=100, hidden_dim=128, layers=3):
inputs = tf.keras.layers.Input(shape=(seq_len, 3))
embed = tf.keras.layers.Embedding(input_dim=len(token2int), output_dim=embed_dim )(inputs)
 reshaped = tf.reshape(
 embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
hidden = tf.keras.layers.SpatialDropout1D (.2 )(reshaped)
 if gru==1:
hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif gru==0:
hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif gru==3:
hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif gru==4:
hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif gru==5:
hidden = lstm_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = tf.keras.layers.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 adam = tf.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead = tfa.optimizers.Lookahead(adam, sync_period=6)
 ranger = tfa.optimizers.Lookahead(radam, sync_period=6)
 model.compile(optimizer=adam, loss=mcrmse)
 return model<define_variables>
df_train['Country_Region'].value_counts()
FOLDS = 5
EPOCHS = 100
REPEATS = 1
BATCH_SIZE = 64
VERBOSE = 2
SEED = 34<define_variables>
grp_obj = df_train.groupby(['Country_Region','Province_State']);
df = pd.DataFrame(grp_obj.agg({'Date':['min','max','count']}))
df
public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df )<define_search_model>
print(df[('Date','min')].value_counts())
 print(df[('Date','max')].value_counts())
 print(df[('Date','count')].value_counts() )
lr_callback = tf.keras.callbacks.ReduceLROnPlateau()<train_model>
grp_obj = df_test.groupby(['Country_Region','Province_State']);
df = pd.DataFrame(grp_obj.agg({'Date':['min','max','count']}))
print(df[('Date','count')].value_counts() )
gru_histories = []
gru_private_preds = np.zeros(( private_df.shape[0], 130, 5))
gru_public_preds = np.zeros(( public_df.shape[0], 107, 5))
rskf = RepeatedStratifiedKFold(FOLDS, n_repeats = REPEATS, random_state = SEED)
 for f,(train_index, val_index)in enumerate(rskf.split(train_inputs, train['SN_filter'])) :
sv_gru = tf.keras.callbacks.ModelCheckpoint(f'gru-{f}.h5')
 train_ = train_inputs[train_index]
train_labs = train_labels[train_index]
val_ = train_inputs[val_index]
val_labs = train_labels[val_index]
gru = build_model(gru=1)
 history = gru.fit(train_, train_labs,
validation_data=(val_,val_labs),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=[lr_callback,sv_gru],
verbose = VERBOSE)
 gru_histories.append(history)
 gru_short = build_model(gru=1, seq_len=107, pred_len=107)
 gru_short.load_weights(f'gru-{f}.h5')
 gru_public_pred = gru_short.predict(public_inputs)/ FOLDS
gru_long = build_model(gru=1, seq_len=130, pred_len=130)
 gru_long.load_weights(f'gru-{f}.h5')
 gru_private_pred = gru_long.predict(private_inputs)/ FOLDS * REPEATS
gru_public_preds += gru_public_pred
gru_private_preds += gru_private_pred
del gru_short, gru_long<init_hyperparams>
def combine_country_province(df):
df.loc[:,'Province_State'] = df['Province_State'].fillna("")
 df.loc[:,'Region'] = df['Country_Region'] + " " + df['Province_State']
df.loc[:,'Region'] = df.loc[:,'Region'].str.strip() ;
df = df.drop(
 labels=['Province_State','Country_Region'],
axis='columns',
inplace=False 
)
 return df;
df_train = combine_country_province(df_train);
df_test = combine_country_province(df_test);
df_train
lstm_histories = []
lstm_private_preds = np.zeros(( private_df.shape[0], 130, 5))
lstm_public_preds = np.zeros(( public_df.shape[0], 107, 5))
rskf = RepeatedStratifiedKFold(FOLDS, n_repeats = REPEATS, random_state = SEED)
 for f,(train_index, val_index)in enumerate(rskf.split(train_inputs, train['SN_filter'])) :
sv_gru = tf.keras.callbacks.ModelCheckpoint(f'lstm-{f}.h5')
 train_ = train_inputs[train_index]
train_labs = train_labels[train_index]
val_ = train_inputs[val_index]
val_labs = train_labels[val_index]
lstm = build_model(gru=0)
 history = lstm.fit(
 train_, train_labs,
validation_data=(val_,val_labs),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=[lr_callback,sv_gru],
verbose = VERBOSE)
 lstm_histories.append(history)
 lstm_short = build_model(gru=0, seq_len=107, pred_len=107)
 lstm_short.load_weights(f'lstm-{f}.h5')
 lstm_public_pred = lstm_short.predict(public_inputs)/ FOLDS
lstm_long = build_model(gru=0, seq_len=130, pred_len=130)
 lstm_long.load_weights(f'lstm-{f}.h5')
 lstm_private_pred = lstm_long.predict(private_inputs)/ FOLDS * REPEATS
lstm_public_preds += lstm_public_pred
lstm_private_preds += lstm_private_pred
del lstm_short, lstm_long<find_best_model_class>
def days_since_dataset(df):
df['Date2'] = df['Date'] - datetime.datetime.strptime('22012020', "%d%m%Y")
 df['Date2'] = df['Date2'].dt.days;
return df['Date2'];
df_train['Date2'] = days_since_dataset(df_train)
 df_test['Date2'] = days_since_dataset(df_test)
 df_train
hyb1_histories = []
hyb1_private_preds = np.zeros(( private_df.shape[0], 130, 5))
hyb1_public_preds = np.zeros(( public_df.shape[0], 107, 5))
rskf = RepeatedStratifiedKFold(FOLDS, n_repeats = REPEATS, random_state = SEED)
 for f,(train_index, val_index)in enumerate(rskf.split(train_inputs, train['SN_filter'])) :
sv_gru = tf.keras.callbacks.ModelCheckpoint(f'hyb1-{f}.h5')
 train_ = train_inputs[train_index]
train_labs = train_labels[train_index]
val_ = train_inputs[val_index]
val_labs = train_labels[val_index]
lstm = build_model(gru=3)
 history = lstm.fit(
 train_, train_labs,
validation_data=(val_,val_labs),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=[lr_callback,sv_gru],
verbose = VERBOSE)
 hyb1_histories.append(history)
 hyb1_short = build_model(gru=3, seq_len=107, pred_len=107)
 hyb1_short.load_weights(f'hyb1-{f}.h5')
 hyb1_public_pred = hyb1_short.predict(public_inputs)/ FOLDS
hyb1_long = build_model(gru=3, seq_len=130, pred_len=130)
 hyb1_long.load_weights(f'hyb1-{f}.h5')
 hyb1_private_pred = hyb1_long.predict(private_inputs)/ FOLDS * REPEATS
hyb1_public_preds += hyb1_public_pred
hyb1_private_preds += hyb1_private_pred
del hyb1_short, hyb1_long<find_best_model_class>
def err_func(y_true,y_pred):
msle = mean_squared_log_error(y_true, y_pred);
return sqrt(msle )
hyb2_histories = []
hyb2_private_preds = np.zeros(( private_df.shape[0], 130, 5))
hyb2_public_preds = np.zeros(( public_df.shape[0], 107, 5))
rskf = RepeatedStratifiedKFold(FOLDS, n_repeats = REPEATS, random_state = SEED)
 for f,(train_index, val_index)in enumerate(rskf.split(train_inputs, train['SN_filter'])) :
sv_gru = tf.keras.callbacks.ModelCheckpoint(f'hyb2-{f}.h5')
 train_ = train_inputs[train_index]
train_labs = train_labels[train_index]
val_ = train_inputs[val_index]
val_labs = train_labels[val_index]
lstm = build_model(gru=5)
 history = lstm.fit(
 train_, train_labs,
validation_data=(val_,val_labs),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=[lr_callback,sv_gru],
verbose = VERBOSE)
 hyb2_histories.append(history)
 hyb2_short = build_model(gru=5, seq_len=107, pred_len=107)
 hyb2_short.load_weights(f'hyb2-{f}.h5')
 hyb2_public_pred = hyb2_short.predict(public_inputs)/ FOLDS
hyb2_long = build_model(gru=5, seq_len=130, pred_len=130)
 hyb2_long.load_weights(f'hyb2-{f}.h5')
 hyb2_private_pred = hyb2_long.predict(private_inputs)/ FOLDS * REPEATS
hyb2_public_preds += hyb2_public_pred
hyb2_private_preds += hyb2_private_pred
del hyb2_short, hyb2_long<split>
degree = 4
poly = PolynomialFeatures(degree = degree, include_bias=False)
 model1 = LinearRegression()
model2 = LinearRegression()
public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df )<prepare_output>
preds_gru = []
for df, preds in [(public_df, gru_public_preds),(private_df, gru_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_gru.append(single_df)
 preds_gru_df = pd.concat(preds_gru)
 preds_gru_df.head()<prepare_output>
X_cols = ['Date2'];
y1_col = ['ConfirmedCases']
y2_col = ['Fatalities']
all_pred_train = pd.DataFrame() ;
all_pred_test = pd.DataFrame() ;
regions = df_train['Region'].unique() ;
train = df_train.copy() ;
test = df_test.copy() ;
for idx, region in enumerate(regions):
scaler = StandardScaler() ;
this_region_train = train['Region'] == region;
this_region_test = test['Region'] == region;
X0_train_iter = train.loc[this_region_train,X_cols];
y1_train_iter = train.loc[this_region_train,y1_col];
y2_train_iter = train.loc[this_region_train,y2_col];
X0_test_iter = test.loc[this_region_test,X_cols];
X0_train_iter = poly.fit_transform(X0_train_iter);
X0_test_iter = poly.fit_transform(X0_test_iter);
X0_train_iter = scaler.fit_transform(X0_train_iter)
 X0_test_iter = scaler.transform(X0_test_iter)
 model1.fit(X0_train_iter, y1_train_iter);
y1_train_iter_pred = model1.predict(X0_train_iter);
y1_test_iter_pred = model1.predict(X0_test_iter);
model2.fit(X0_train_iter, y2_train_iter);
y2_train_iter_pred = model2.predict(X0_train_iter);
y2_test_iter_pred = model2.predict(X0_test_iter);
pred_iter_train = pd.DataFrame({
'Id': train.loc[this_region_train,'Id'],
'ConfirmedCases': y1_train_iter_pred.reshape(-1),
'Fatalities': y2_train_iter_pred.reshape(-1)
 })
 all_pred_train = pd.concat([all_pred_train, pred_iter_train], axis = 0);
pred_iter_test = pd.DataFrame({
'ForecastId': test.loc[this_region_test,'ForecastId'],
'ConfirmedCases': y1_test_iter_pred.reshape(-1),
'Fatalities': y2_test_iter_pred.reshape(-1)
 })
 all_pred_test = pd.concat([all_pred_test, pred_iter_test], axis = 0);
print(all_pred_train)
 print(all_pred_test )
preds_lstm = []
for df, preds in [(public_df, lstm_public_preds),(private_df, lstm_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_lstm.append(single_df)
 preds_lstm_df = pd.concat(preds_lstm)
 preds_lstm_df.head()<prepare_output>
all_pred_test = all_pred_test.astype('int')
preds_hyb1 = []
for df, preds in [(public_df, hyb1_public_preds),(private_df, hyb1_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_hyb1.append(single_df)
 preds_hyb1_df = pd.concat(preds_hyb1)
 preds_hyb1_df.head()<prepare_output>
answer = pd.merge(df_test,all_pred_test, left_on = 'ForecastId',right_on = 'ForecastId');
train_max_date = train['Date2'].max()
test_min_date = test['Date2'].min()
for idx, region in enumerate(regions):
sel1 =(( train['Date2'] >= test_min_date)&(train['Region'] == region)) ;
to_paste = train.loc[sel1,['ConfirmedCases','Fatalities']].copy() ;
sel2 =(( answer['Date2'] <= train_max_date)&(answer['Region'] == region))
answer.loc[sel2,['ConfirmedCases','Fatalities']] = to_paste.loc[:,['ConfirmedCases','Fatalities']].values;
preds_hyb2 = []
for df, preds in [(public_df, hyb2_public_preds),(private_df, hyb2_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_hyb2.append(single_df)
 preds_hyb2_df = pd.concat(preds_hyb2)
 preds_hyb2_df.head()<prepare_output>
answer = answer.loc[:,['ForecastId','ConfirmedCases','Fatalities']]
answer = answer.astype('int');
answer.to_csv("submission.csv", index = False);
answer.dtypes
blend_preds_df = pd.DataFrame()
blend_preds_df['id_seqpos'] = preds_gru_df['id_seqpos']
blend_preds_df['reactivity'] = 0.25*preds_gru_df['reactivity'] + 0.25*preds_lstm_df['reactivity'] + 0.25*preds_hyb1_df['reactivity'] + 0.25*preds_hyb2_df['reactivity']
blend_preds_df['deg_Mg_pH10'] = 0.25*preds_gru_df['deg_Mg_pH10'] + 0.25*preds_lstm_df['deg_Mg_pH10'] + 0.25*preds_hyb1_df['deg_Mg_pH10'] + 0.25*preds_hyb2_df['deg_Mg_pH10']
blend_preds_df['deg_pH10'] = 0.25*preds_gru_df['deg_pH10'] + 0.25*preds_lstm_df['deg_pH10'] + 0.25*preds_hyb1_df['deg_pH10'] + 0.25*preds_hyb2_df['deg_pH10']
blend_preds_df['deg_Mg_50C'] = 0.25*preds_gru_df['deg_Mg_50C'] + 0.25*preds_lstm_df['deg_Mg_50C'] + 0.25*preds_hyb1_df['deg_Mg_50C'] + 0.25*preds_hyb2_df['deg_Mg_50C']
blend_preds_df['deg_50C'] = 0.25*preds_gru_df['deg_50C'] + 0.25*preds_lstm_df['deg_50C'] + 0.25*preds_hyb1_df['deg_50C'] + 0.25*preds_hyb2_df['deg_50C']<merge>