Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<set_options>
EPOCHS = 60
BATCH_SIZE = 64
SEED = 34
TTA = True
FOLD_N = 5
sns.set_style('darkgrid')
 %matplotlib inline
warnings.filterwarnings('ignore')
<load_from_csv>
def seed_everything(seed = SEED):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything()
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 df_train['Date'] = pd.to_datetime(df_train['Date'], format = '%Y-%m-%d')
 df_test['Date'] = pd.to_datetime(df_test['Date'], format = '%Y-%m-%d')
 stamp = dt.datetime(2020, 4, 14)
 df_train = df_train[df_train['Date']<=stamp]
df_test = df_test[df_test['Date']<=stamp]
df_train['Days'] = df_train['Date']-df_train['Date'].min()
df_train.Province_State.fillna('ZZZ', inplace=True)
 df_test.Province_State.fillna('ZZZ', inplace=True)
 df_train['Days'] = df_train['Days'].astype('timedelta64[D]')
<load_from_csv>
def pandas_list_to_array(df):
return np.transpose(
 np.array(df.values.tolist()), 
(0, 2, 1)
)
 def preprocess_inputs(df, token2int, cols=['sequence', 'structure', 'predicted_loop_type']):
base_feature = np.transpose(df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values.tolist() , 
(0, 2, 1))
bpps_sum_feature = np.array(df['bpps_sum'].tolist())[:, :, np.newaxis]
bpps_max_feature = np.array(df['bpps_max'].tolist())[:, :, np.newaxis]
bpps_nb_feature = np.array(df['bpps_nb'].tolist())[:, :, np.newaxis]
return np.concatenate([base_feature, bpps_sum_feature, bpps_max_feature, bpps_nb_feature], axis=2)
 def augment_data(augment_df, df):
target_df = df.copy()
new_df = augment_df[augment_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id', 'sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id', 'cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
dict_geo = {}
df_mid = df_train.groupby(['Country_Region','Province_State'],as_index=False)
 print(len(df_mid))
df_metadata = pd.read_csv('/kaggle/input/covid19-forecasting-metadata/region_metadata.csv')
 df_metadata.Province_State.fillna('ZZZ', inplace=True)
 dict_key = df_metadata.groupby(['Country_Region','Province_State'],as_index=False)
 for name,group in df_mid:
i = df_metadata[(df_metadata['Country_Region']== name[0])&(df_metadata['Province_State']== name[1])].index.tolist() [0]
lat = df_metadata.loc[i,'lat']
long = df_metadata.loc[i,'lon']
pop = df_metadata.loc[i,'population']
area = df_metadata.loc[i,'area']
dict_geo.update({name:[lat,long,pop,area]} )<feature_engineering>
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return tf.keras.backend.sqrt(mse)
 target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C', 'deg_pH10', 'deg_50C']
def mcrmse(y_actual, y_pred, num_scored=len(target_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
def add_daily_measures(df):
df.loc[0,'Daily Cases'] = df.loc[0,'ConfirmedCases']
df.loc[0,'Daily Deaths'] = df.loc[0,'Fatalities']
for i in range(1,len(df)) :
df.loc[i,'Daily Cases'] = df.loc[i,'ConfirmedCases'] - df.loc[i-1,'ConfirmedCases']
df.loc[i,'Daily Deaths'] = df.loc[i,'Fatalities'] - df.loc[i-1,'Fatalities']
if df.loc[i,'Daily Deaths']<0:
df.loc[i,'Daily Deaths'] = df.loc[i-1,'Daily Deaths']
if df.loc[i,'Daily Cases']<0:
df.loc[i,'Daily Cases'] = df.loc[i-1,'Daily Cases']
df.loc[i,'lastconfirm'] = df.loc[i-1,'ConfirmedCases']
df.loc[i,'lastdeath'] = df.loc[i-1,'Fatalities']
df.loc[0,'Daily Cases'] = 0
df.loc[0,'Daily Deaths'] = 0
df.loc[0,'lastconfirm'] = 0
df.loc[0,'lastdeath'] = 0
return df<feature_engineering>
class GRU_Layer(Layer):
def __init__(self, hidden_dim, dropout):
super(GRU_Layer, self ).__init__()
self.custom_layers = Bidirectional(GRU(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def call(self, inputs):
return self.custom_layers(inputs)
 class LSTM_Layer(Layer):
def __init__(self, hidden_dim, dropout):
super(LSTM_Layer, self ).__init__()
self.custom_layers = Bidirectional(LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def call(self, inputs):
return self.custom_layers(inputs )
def train_dev_split(df, days):
date = df['Date'].max() - dt.timedelta(days=days)
 return df,df[df['Date'] <= date], df[df['Date'] > date]
def add_more_daily_measures(df):
for i in range(4,len(df)) :
df.loc[i,'Daily Cases 1'] = df.loc[i-1,'Daily Cases']
df.loc[i,'Daily Deaths 1'] = df.loc[i-1,'Daily Deaths']
df.loc[i,'Daily Cases 2'] = df.loc[i-2,'Daily Cases']
df.loc[i,'Daily Deaths 2'] = df.loc[i-2,'Daily Deaths']
df.loc[i,'Daily Cases 3'] = df.loc[i-3,'Daily Cases']
df.loc[i,'Daily Deaths 3'] = df.loc[i-3,'Daily Deaths']
df.loc[0,'Daily Cases 1'] = 0
df.loc[0,'Daily Deaths 1'] = 0
df.loc[0,'Daily Cases 2'] = 0
df.loc[0,'Daily Deaths 2'] = 0
df.loc[0,'Daily Cases 3'] = 0
df.loc[0,'Daily Deaths 3'] = 0
df.loc[1,'Daily Cases 1'] = df.loc[0,'Daily Cases']
df.loc[1,'Daily Deaths 1'] = df.loc[0,'Daily Deaths']
df.loc[1,'Daily Cases 2'] = 0
df.loc[1,'Daily Deaths 2'] = 0
df.loc[1,'Daily Cases 3'] = 0
df.loc[1,'Daily Deaths 3'] = 0
df.loc[2,'Daily Cases 1'] = df.loc[1,'Daily Cases']
df.loc[2,'Daily Deaths 1'] = df.loc[1,'Daily Deaths']
df.loc[2,'Daily Cases 2'] = df.loc[0,'Daily Cases']
df.loc[2,'Daily Deaths 2'] = df.loc[0,'Daily Deaths']
df.loc[2,'Daily Cases 3'] = 0
df.loc[2,'Daily Deaths 3'] = 0
return df<feature_engineering>
class MIX_LSTM_GRU(tf.keras.Model):
def __init__(self, seq_len=107, pred_len=68, dropout=0.5, embed_dim=100,
hidden_dim=256, model_type=0, token2int=None):
super(MIX_LSTM_GRU, self ).__init__()
self.token2int = token2int
self.embed = L.Embedding(input_dim=len(self.token2int), output_dim=embed_dim)
 self.dense = L.Dense(5, activation='linear')
 self.pred_len = pred_len
self.hidden_dim = hidden_dim
self.dropout = dropout
self.type = model_type
def build(self, inputs):
self.gru_layers = []
self.lstm_layers = []
for i in range(2):
self.gru_layers.append(GRU_Layer(self.hidden_dim, self.dropout))
self.lstm_layers.append(LSTM_Layer(self.hidden_dim, self.dropout))
def call(self, inputs):
categorical_features = inputs[:, :, :3]
numerical_features = inputs[:, :, 3:]
embed = self.embed(categorical_features)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = tf.keras.layers.concatenate([reshaped, numerical_features], axis=2)
 if self.type == 0:
hidden = self.gru_layers[0](reshaped)
 hidden = self.gru_layers[1](hidden)
 elif self.type == 1:
hidden = self.lstm_layers[0](reshaped)
 hidden = self.gru_layers[0](hidden)
 elif self.type == 2:
hidden = self.gru_layers[1](reshaped)
 hidden = self.lstm_layers[1](hidden)
 elif self.type == 3:
hidden = self.lstm_layers[0](reshaped)
 hidden = self.lstm_layers[1](hidden)
 truncated = hidden[:, :self.pred_len]
out = self.dense(truncated)
 return out
df_list = []
grouped = df_train.groupby(['Country_Region','Province_State'],as_index=False)
 for name,group in grouped:
df_tmp = df_train[(df_train["Country_Region"]==name[0])&(df_train["Province_State"] == name[1])].reset_index(drop = True)
 df_tmp = add_daily_measures(df_tmp)
 df_tmp = add_more_daily_measures(df_tmp)
 df_tmp['Lat'] = dict_geo[name][0]
df_tmp['Long'] = dict_geo[name][1]
df_tmp['population'] = dict_geo[name][2]
df_tmp['area'] = dict_geo[name][3]
df_tmp['popdensity'] = dict_geo[name][2]/dict_geo[name][3]
df_list.append(df_tmp)
 df_concat = pd.concat(df_list,ignore_index=True)
 df_train = df_concat
df_train,df_past,df_dev = train_dev_split(df_train,13)
 columns =['Daily Cases 3','Daily Cases 2','Daily Cases 1','Daily Deaths 3','Daily Deaths 2','Daily Deaths 1','Lat','Long','population','area','popdensity','lastconfirm','lastdeath','Days','Province_State', 'Country_Region','ConfirmedCases','Fatalities','Daily Cases','Daily Deaths']
df_train = df_train[columns]
df_dev = df_dev[columns]<prepare_x_and_y>
def read_bpps_sum(df, root_path):
bpps_arr = []
for obj_id in df.id.to_list() :
bppm = np.load(os.path.join(root_path, obj_id + '.npy')).sum(axis=1)
 bpps_arr.append(bppm)
 return bpps_arr
def read_bpps_max(df, root_path):
bpps_arr = []
for obj_id in df.id.to_list() :
bppm = np.load(os.path.join(root_path, obj_id + '.npy'))
bpps_arr.append(np.max(bppm, axis=1))
return bpps_arr
def read_bpps_nb(df, root_path, bpps_mean, bpps_std):
bpps_arr = []
for obj_id in df.id.to_list() :
bpps = np.load(os.path.join(root_path, obj_id + '.npy'))
bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_mean)/ bpps_std
bpps_arr.append(bpps_nb)
 return bpps_arr
def calc_bpps_mean_std(df, root_path):
bpps_arr = []
for obj_id in df.id.to_list() :
bpps = np.load(os.path.join(root_path, obj_id + '.npy'))
bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_arr.append(bpps_nb)
 return np.mean(bpps_arr), np.std(bpps_arr)
 def add_bpp_feature(train, test, bpps_dir):
mean, std = calc_bpps_mean_std(train, bpps_dir)
 train['bpps_sum'] = read_bpps_sum(train, bpps_dir)
 test['bpps_sum'] = read_bpps_sum(test, bpps_dir)
 train['bpps_max'] = read_bpps_max(train, bpps_dir)
 test['bpps_max'] = read_bpps_max(test, bpps_dir)
 train['bpps_nb'] = read_bpps_nb(train, bpps_dir, mean, std)
 test['bpps_nb'] = read_bpps_nb(test, bpps_dir, mean, std)
 return train, test
columns_test = ['Daily Cases 3','Daily Cases 2','Daily Cases 1','Daily Deaths 3','Daily Deaths 2','Daily Deaths 1','Lat','Long','population','area','popdensity','lastconfirm','lastdeath','Days']
def generate_test_feature(df):
alpha = []
beta = []
df_target = pd.DataFrame(columns = columns_test)
 df_adjust = df.tail(7)
 df_alpha = df_adjust['Daily Cases'].values
for i in range(6):
alpha.append(df_alpha[i+1]-df_alpha[i])
 df_beta = df_adjust['Daily Deaths'].values
for i in range(6):
beta.append(df_beta[i+1]-df_beta[i])
 df_tmp = df.tail(1)
 df_target['Daily Cases 3'] = df_tmp['Daily Cases 2']
df_target['Daily Cases 2'] = df_tmp['Daily Cases 1']
df_target['Daily Cases 1'] = df_tmp['Daily Cases']
df_target['Daily Deaths 3'] = df_tmp['Daily Deaths 2']
df_target['Daily Deaths 2'] = df_tmp['Daily Deaths 1']
df_target['Daily Deaths 1'] = df_tmp['Daily Deaths']
df_target['Lat'] = df_tmp['Lat']
df_target['Long'] = df_tmp['Long']
df_target['population'] = df_tmp['population']
df_target['area'] = df_tmp['area']
df_target['popdensity'] = df_tmp['popdensity']
df_target['lastconfirm'] = df_tmp['ConfirmedCases']
df_target['lastdeath'] = df_tmp['Fatalities']
df_target['Days'] = df_tmp['Days']+1
return df_target,sum(alpha),sum(beta )<drop_column>
def group_by_kmean_reactivity(train, token2int):
kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train, token2int)[:, :, 0])
 train['cluster_id'] = kmeans_model.labels_
return train
d = {'ForecastId':[], 'ConfirmedCases':[], 'Fatalities':[]}
submission_confirm = []
submission_death = []
china_train = df_train[(df_train['Country_Region']=='China')&(df_train['Days']>50)]
china_death = china_train.drop(['Country_Region','Province_State','ConfirmedCases','Fatalities','Daily Cases'], axis=1)
 china_death = china_death.values
china_confirm = china_train.drop(['Country_Region','Province_State','ConfirmedCases','Fatalities','Daily Deaths 3','Daily Deaths 2','Daily Deaths 1','Daily Deaths'], axis=1)
 china_confirm = china_confirm.values
df_train_noch = df_train[(df_train['Days']>50)&(df_train['Country_Region']!='China')]
train_death = df_train_noch.drop(['Country_Region','Province_State','ConfirmedCases','Fatalities','Daily Cases'], axis=1)
 print(train_death.columns)
 train_death = train_death.values
train_death = np.append(china_death,train_death,axis = 0)
 train_confirm = df_train_noch.drop(['Country_Region','Province_State','ConfirmedCases','Fatalities','Daily Deaths 3','Daily Deaths 2','Daily Deaths 1','Daily Deaths'], axis=1)
 print(train_confirm.columns)
 train_confirm = train_confirm.values
train_confirm = np.append(china_confirm,train_confirm,axis = 0)
 X_train_confirm, y_train_confirm = train_confirm[:,:-1], train_confirm[:,-1:]
X_train_death, y_train_death = train_death[:,:-1], train_death[:,-1:]
model1 = XGBRegressor(n_estimators = 1000,learning_rate = 0.1,tree_method='gpu_hist')
 model1.fit(X_train_confirm, y_train_confirm[:,0])
 model2 = XGBRegressor(n_estimators = 1000,learning_rate = 0.1,tree_method='gpu_hist')
 model2.fit(X_train_death, y_train_death[:,0])
 for country in df_train.Country_Region.unique() :
if country=='China':
alpha = 0.1
else:
alpha = 1
df_train1 = df_train[df_train["Country_Region"]==country]
for state in df_train1.Province_State.unique() :
df_train2 = df_train1[df_train1["Province_State"]==state]
df_train3 = df_train2.drop(['Country_Region','Province_State'], axis=1)
 df_overlay = df_dev[(df_dev["Country_Region"]==country)&(df_dev["Province_State"]==state)]
submission_confirm =submission_confirm + df_overlay['ConfirmedCases'].values.tolist()
submission_death =submission_death + df_overlay['Fatalities'].values.tolist()
for i in range(30):
df_target,confirm_fact,deaths_fact = generate_test_feature(df_train3)
 df_target_death = df_target[columns_test]
df_target_confirm = df_target[columns_test].drop(['Daily Deaths 3','Daily Deaths 2','Daily Deaths 1'], axis=1)
 target_confirm = df_target_confirm.values
target_death = df_target_death.values
if confirm_fact<0:
y_pred1 = abs(model1.predict(target_confirm)) *0.85*alpha
else:
y_pred1 = abs(model1.predict(target_confirm)) *alpha
if deaths_fact<0:
y_pred2 = abs(model2.predict(target_death)) *0.8*alpha
else:
y_pred2 = abs(model2.predict(target_death)) *alpha
submission_confirm.append(target_death[0][11]+int(y_pred1))
submission_death.append(target_death[0][12]+int(y_pred2))
target = target_death.flatten()
target = np.append(target,target[11]+int(y_pred1))
target = np.append(target,target[12]+int(y_pred2))
target = np.append(target,int(y_pred1))
target = np.append(target,int(y_pred2))
target = np.reshape(target,(1, 18))
last_index = df_train3.tail(1 ).index.values[0]
df_train3.loc[last_index+1] = target[0]
d['ConfirmedCases'] = submission_confirm
d['Fatalities'] = submission_death
a = np.arange(1,len(submission_death)+1,1)
 d['ForecastId'] = a 
<save_to_csv>
def train_mix_lstm_gru(train, public_df, private_df, target_cols, token2int, model_type=0, FOLD_N=FOLD_N):
Ver='MIX_LSTM_GRU_' + str(model_type)
 gkf = GroupKFold(n_splits=FOLD_N)
 public_inputs = preprocess_inputs(public_df, token2int)
 private_inputs = preprocess_inputs(private_df, token2int)
 holdouts = []
holdout_preds = []
for cv,(tr_idx, vl_idx)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
trn = train.iloc[tr_idx]
x_trn = preprocess_inputs(trn, token2int)
 y_trn = np.array(trn[target_cols].values.tolist() ).transpose(( 0, 2, 1))
w_trn = np.log(trn.signal_to_noise+1.1)/2
val = train.iloc[vl_idx]
x_val_all = preprocess_inputs(val, token2int)
 val = val[val.SN_filter == 1]
x_val = preprocess_inputs(val, token2int)
 y_val = np.array(val[target_cols].values.tolist() ).transpose(( 0, 2, 1))
model = MIX_LSTM_GRU(model_type=model_type, token2int=token2int)
 model.compile(tf.keras.optimizers.Adam() , loss=mcrmse)
 model_short = MIX_LSTM_GRU(seq_len=107, pred_len=107, model_type=model_type, token2int=token2int)
 model_long = MIX_LSTM_GRU(seq_len=130, pred_len=130, model_type=model_type, token2int=token2int)
 history = model.fit(
 x_trn, y_trn,
validation_data =(x_val, y_val),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
sample_weight=w_trn,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.ModelCheckpoint(f'model{Ver}_cv{cv}.h5')
 ] 
)
 fig = px.line(
 history.history, y=['loss', 'val_loss'],
labels={'index': 'epoch', 'value': 'Mean Squared Error'},
title='Training History')
 fig.show()
model.load_weights(f'model{Ver}_cv{cv}.h5')
 model_short.compile(optimizer=tf.optimizers.Adam() , loss=mcrmse)
 model_short.train_on_batch(tf.zeros(( 1,107,6)) , tf.zeros(( 1,107,5)))
 model_short.load_weights(f'model{Ver}_cv{cv}.h5')
 model_long.compile(optimizer=tf.optimizers.Adam() , loss=mcrmse)
 model_long.train_on_batch(tf.zeros(( 1,130,6)) , tf.zeros(( 1,130,5)))
 model_long.load_weights(f'model{Ver}_cv{cv}.h5')
 holdouts.append(train.iloc[vl_idx])
 holdout_preds.append(model.predict(x_val_all))
if cv == 0:
public_preds = model_short.predict(public_inputs)/FOLD_N
private_preds = model_long.predict(private_inputs)/FOLD_N
else:
public_preds += model_short.predict(public_inputs)/FOLD_N
private_preds += model_long.predict(private_inputs)/FOLD_N
return holdouts, holdout_preds, public_df, public_preds, private_df, private_preds
print(model1.feature_importances_)
 print(model2.feature_importances_)
 df_submit = pd.DataFrame(d)
 print(df_submit.head())
 df_submit.to_csv(r'submission.csv', index=False )<load_from_csv>
data_dir = '/kaggle/input/stanford-covid-vaccine/'
bpps_dir = '/kaggle/input/stanford-covid-vaccine/bpps/'
train = pd.read_json(data_dir + 'train.json', lines=True)
 test = pd.read_json(data_dir + 'test.json', lines=True)
 sample_df = pd.read_csv(data_dir + 'sample_submission.csv')
 aug_df = pd.read_csv('/kaggle/input/openvaccine-augment/aug_data.csv')
 token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
train, test = add_bpp_feature(train, test, bpps_dir)
 train = group_by_kmean_reactivity(train, token2int)
 if TTA:
train = augment_data(aug_df, train)
 test = augment_data(aug_df, test)
 target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C', 'deg_pH10', 'deg_50C']
train_inputs = preprocess_inputs(train, token2int)
 train_labels = pandas_list_to_array(train[target_cols])
 public_df = test.query("seq_length == 107")
 private_df = test.query("seq_length == 130" )
def read_data() :
data = pd.read_csv(".. /input/datatrain4/train4.csv")
 data = data.values
return data<load_from_disk>
val_df, val_preds, test_df, test_preds = [], [], [], []
debug = False
if debug:
nmodel = 1
else:
nmodel = 4
for i in range(nmodel):
holdouts, holdout_preds, public_df, public_preds, private_df, private_preds = train_mix_lstm_gru(train,
public_df, private_df,
target_cols,
token2int,model_type=i)
 val_df += holdouts
val_preds += holdout_preds
test_df.append(public_df)
 test_df.append(private_df)
 test_preds.append(public_preds)
 test_preds.append(private_preds )
data = read_data()<feature_engineering>
preds_ls = []
for df, preds in zip(test_df, test_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
preds_ls = []
for df, preds in zip(val_df, val_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
single_df['SN_filter'] = df[df['id'] == uid].SN_filter.values[0]
preds_ls.append(single_df)
 holdouts_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
<feature_engineering><EOS> 
submission = preds_df[['id_seqpos', 'reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']]
submission.to_csv(f'submission.csv', index=False)
 print(f'wrote to submission.csv' )