Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<data_type_conversions>
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
def allocate_gpu_memory(gpu_number=0):
physical_devices = tf.config.experimental.list_physical_devices('GPU')
 if physical_devices:
try:
print("Found {} GPU(s)".format(len(physical_devices)))
 tf.config.set_visible_devices(physical_devices[gpu_number], 'GPU')
 tf.config.experimental.set_memory_growth(physical_devices[gpu_number], True)
 print("
except RuntimeError as e:
print(e)
 else:
print("Not enough GPU hardware devices available")
 allocate_gpu_memory()
Ver='GRU_LSTM1'
aug_data = '.. /input/augmented-data/aug_data1.csv'
debug = False
date = pd.to_datetime(train["Date"])
 datet = pd.to_datetime(test["Date"])
 print(date)
 train['Date'] = pd.to_datetime(train.Date)
 test['Date'] = pd.to_datetime(test.Date )<train_model>
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def build_model(seq_len=107, pred_len=68, dropout=0.5, embed_dim=100*2, hidden_dim=256*2, type=0):
inputs = L.Input(shape=(seq_len, 6))
categorical_feat_dim = 3
categorical_fea = inputs[:, :, :categorical_feat_dim]
numerical_fea = inputs[:, :, 3:]
embed = L.Embedding(input_dim=len(token2int), output_dim=embed_dim )(categorical_fea)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = L.concatenate([reshaped, numerical_fea], axis=2)
 reshaped = L.LayerNormalization()(reshaped)
 if type == 0:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 1:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 2:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif type == 3:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = L.Dense(5, activation='linear', name='reactivity' )(truncated)
 out2 = L.Dense(5, activation='linear', name='error' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=[out, out2])
 losses = { "reactivity": mcrmse, "error": mcrmse }
loss_weights = {"reactivity": 0.95, "error": 0.05}
model.compile(tf.keras.optimizers.Adam() , loss=losses, loss_weights=loss_weights)
 return model
ldate = int(len(date))
ldatet = int(len(datet))
print("Length of training- date is", ldate)
 print("Length of test- date is", ldatet )<feature_engineering>
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
pred_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
pred_cols_error = ['reactivity_error', 'deg_error_Mg_pH10', 'deg_error_pH10', 'deg_error_Mg_50C', 'deg_error_50C']
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea], 2)
 def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=len(pred_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
train['month'] = train['Date'].dt.month
train['day'] = train['Date'].dt.day
test['month'] = test['Date'].dt.month
test['day'] = test['Date'].dt.day<count_unique_values>
train = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json', lines=True)
 RANDOM_SEED=46
RANDOM_SEED_KMEANS=110
KMEANS_CLUSTERS=100
train = train[train.signal_to_noise > 0.05]
train = train.sample(random_state=RANDOM_SEED, frac=1 )
print("Datatrain")
 traindays = train['Date'].nunique()
print("Number of Country_Region: ", train['Country_Region'].nunique())
 print("Number of Province_State: ", train['Province_State'].nunique())
 print("Number of Days: ", traindays)
 notrain = train['Id'].nunique()
print("Number of datapoints in train:", notrain)
 lotrain = int(notrain/traindays)
 print("L Trains:", lotrain)
<count_unique_values>
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test )
print("Datatest")
 testdays = test['Date'].nunique()
print("Number of Days: ", testdays)
 notest = test['ForecastId'].nunique()
print("Number of datapoints in test:", notest)
 lotest = int(notest/testdays)
 print("L Test:", lotest )<define_variables>
kmeans_model = KMeans(n_clusters=KMEANS_CLUSTERS, random_state=RANDOM_SEED_KMEANS ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_
zt = datet[0]
daycount = []
for i in range(0,lotrain):
for j in range(1,traindays+1):
daycount.append(j)
 print(daycount)
 print(zt )<define_variables>
!pip install -q umap-learn
for i in range(traindays):
if(zt == date[i]):
zx = i
print(zx)
 daytest = []
for i in range(0,lotest):
for j in range(1,testdays+1):
jr = zx + j
daytest.append(jr)
 print(daytest )<prepare_output>
mapper = umap.UMAP().fit(preprocess_inputs(train)[:,:,0])
 embedding = mapper.transform(preprocess_inputs(train)[:,:,0] )
train.insert(8,"DayCount",daycount,False)
 test.insert(6,"DayCount",daytest,False )<define_variables>
aug_df = pd.read_csv(aug_data)
 display(aug_df.head() )
traincount = int(len(train["Date"]))
testcount = int(len(test["Date"]))
print(traincount,testcount )<categorify>
def add_aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = add_aug_data(train)
 test = add_aug_data(test )
train.Province_State = train.Province_State.fillna(0)
 empty = 0
for i in range(0,traincount):
if(train.Province_State[i] == empty):
train.Province_State[i] = train.Country_Region[i]<categorify>
if debug:
train = train[:200]
test = test[:200]
test.Province_State = test.Province_State.fillna(0)
 empty = 0
for i in range(0,testcount):
if(test.Province_State[i] == empty):
test.Province_State[i] = test.Country_Region[i]<categorify>
class TrainingPlot(tf.keras.callbacks.Callback):
def __init__(self, nrows=1, ncols=2, figsize=(10, 5), title=None, save_file=None, old_logs_path=None):
self.nrows=nrows
self.ncols=ncols
self.figsize=figsize
self.title=title
self.old_logs_path=old_logs_path
self.old_logs = []
self.old_log_file_names = []
if self.old_logs_path is not None:
p = pathlib.Path(self.old_logs_path)
 self.old_logs_files = p.parent.glob(p.name)
 for f in self.old_logs_files:
try:
self.old_logs.append(pd.read_csv(f))
self.old_log_file_names.append(pathlib.Path(f ).stem)
 except:
continue
if save_file:
self.save_file = save_file
self.metrics = []
self.logs = []
def add(self, row, col, name, color=None, vmin=None, vmax=None, show_min=False, show_max=False):
self.metrics.append({'row': row, 'col': col, 'name': name, 'color': color, 'vmin': vmin, 'vmax': vmax, 'show_min': show_min, 'show_max': show_max})
 def on_train_begin(self, logs={}):
self.logs = []
for m in self.metrics:
m['values'] = []
def on_epoch_end(self, epoch, logs={}):
self.logs.append(logs)
 clear_output(wait=True)
 plt.style.use("seaborn")
 fig, ax = plt.subplots(self.nrows, self.ncols, figsize=self.figsize)
 if len(ax.shape)== 1:
ax = np.expand_dims(ax, axis=0)
 if self.title:
fig.suptitle(self.title)
 for m in self.metrics:
if logs.get(m['name'])is None:
v = m['values']
v.append(np.nan)
 continue
a = ax[m['row'], m['col']]
if m['name'] == 'off':
a.axis('off')
 continue
v = m['values']
v.append(logs.get(m['name']))
for i, old_log in enumerate(self.old_logs):
if m['name'] not in old_log:
continue
old_values = old_log[m['name']].values
a.plot(np.arange(len(old_values)) , old_values,
'-',
color=m['color'],
alpha=0.2, lw=1)
 a.plot(np.arange(len(v)) , v, '-o', color=m['color'], label=m['name'], lw=1, markersize=3)
 a.set_xlabel('Epoch
yname = m['name']
if yname.startswith('val_'):
yname = m['name'][4:]
a.set_ylabel(yname, size=14)
 xdist = a.get_xlim() [1] - a.get_xlim() [0]
ydist = a.get_ylim() [1] - a.get_ylim() [0]
if ydist is not None and xdist is not None:
if m['show_max']:
x = np.argmax(v)
 y = np.max(v)
 a.scatter(x, y, s=200, color=m['color'], alpha=0.5)
 a.text(x-0.03*xdist, y-0.13*ydist, f'{round(y, 4)}', size=14)
 if m['show_min']:
x = np.argmin(v)
 y = np.min(v)
 a.scatter(x, y, s=200, color=m['color'], alpha=0.5)
 a.text(x-0.03*xdist, y+0.05*ydist, f'{round(y, 4)}', size=14)
 if m['vmin'] is not None:
a.set_ylim(m['vmin'], m['vmax'])
 a.legend()
plt.show()
if self.save_file:
fig.savefig(self.save_file)
 def create_plot(label, csv_file_mask='*.csv'):
plot = TrainingPlot(nrows=1, ncols=2, figsize=(20, 5), title=label,
save_file=f'{label}.png',
old_logs_path=csv_file_mask 
)
 plot.add(0, 0, 'loss', 'green')
 plot.add(0, 0, 'val_loss', 'red', show_min=True)
 plot.add(0, 1, 'lr', 'black', vmin=0)
 return plot
def cvs_callback(filename):
return tf.keras.callbacks.CSVLogger(filename)
label = preprocessing.LabelEncoder()
train.Country_Region = label.fit_transform(train.Country_Region)
 train.Province_State = label.fit_transform(train.Province_State )<categorify>
def train_and_predict(type = 0, FOLD_N = 5):
gkf = GroupKFold(n_splits=FOLD_N)
 public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 holdouts = []
holdout_preds = []
for cv,(tr_idx, vl_idx)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
trn = train.iloc[tr_idx]
x_trn = preprocess_inputs(trn)
 y_trn = np.array(trn[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
y_trn_err = np.log(np.array(trn[pred_cols_error].values.tolist() ).transpose(( 0, 2, 1)))
 w_trn = np.log(trn.signal_to_noise+1.1)/2
val = train.iloc[vl_idx]
x_val_all = preprocess_inputs(val)
 val = val[val.SN_filter == 1]
x_val = preprocess_inputs(val)
 y_val = np.array(val[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
y_val_err = np.log(np.array(val[pred_cols_error].values.tolist() ).transpose(( 0, 2, 1)))
 model = build_model(type=type)
 model_short = build_model(seq_len=107, pred_len=107,type=type)
 model_long = build_model(seq_len=130, pred_len=130,type=type)
 history = model.fit(
 x_trn, {"reactivity": y_trn, "error": y_trn_err},
validation_data =(x_val, {"reactivity": y_val, "error": y_val_err}),
batch_size=32,
epochs=90,
sample_weight=w_trn,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.ModelCheckpoint(f'model-t{type}-v{Ver}-f{cv}.h5'),
create_plot(f'train-{cv}', csv_file_mask='train-*.csv'),
cvs_callback(f'train-{cv}.csv')
 ] 
)
 model.load_weights(f'model-t{type}-v{Ver}-f{cv}.h5')
 model_short.load_weights(f'model-t{type}-v{Ver}-f{cv}.h5')
 model_long.load_weights(f'model-t{type}-v{Ver}-f{cv}.h5')
 holdouts.append(train.iloc[vl_idx])
 holdout_preds.append(model.predict(x_val_all)[0])
 if cv == 0:
public_preds = model_short.predict(public_inputs)[0]/FOLD_N
private_preds = model_long.predict(private_inputs)[0]/FOLD_N
else:
public_preds += model_short.predict(public_inputs)[0]/FOLD_N
private_preds += model_long.predict(private_inputs)[0]/FOLD_N
return holdouts, holdout_preds, public_df, public_preds, private_df, private_preds
test.Country_Region = label.fit_transform(test.Country_Region)
 test.Province_State = label.fit_transform(test.Province_State)
<rename_columns>
!rm *.csv
train.rename({'Id': 'ForecastId'}, axis=1, inplace=True )<drop_column>
val_df, val_preds, test_df, test_preds = [], [], [], []
if debug:
nmodel = 1
else:
nmodel = 4
for i in range(nmodel):
holdouts, holdout_preds, public_df, public_preds, private_df, private_preds = train_and_predict(i)
 val_df += holdouts
val_preds += holdout_preds
test_df.append(public_df)
 test_df.append(private_df)
 test_preds.append(public_preds)
 test_preds.append(private_preds )
forecastid = test['ForecastId']
cases = train.ConfirmedCases
fatalities = train.Fatalities
train = train.drop(['ForecastId', 'Date','ConfirmedCases','Fatalities'], axis = 1)
 test = test.drop(['ForecastId', 'Date'], axis = 1 )<train_on_grid>
preds_ls = []
for df, preds in zip(test_df, test_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=pred_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
preds_ls = []
for df, preds in zip(val_df, val_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=pred_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
single_df['SN_filter'] = df[df['id'] == uid].SN_filter.values[0]
preds_ls.append(single_df)
 holdouts_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
lr_model = LinearRegression()
lr_modelfat = LinearRegression()
lr_model.fit(train, cases)
 lr_modelfat.fit(train,fatalities )<compute_train_metric>
holdouts_df.to_csv(f'holdouts.csv', index=False )
ypredtr = lr_model.predict(train)
 ypredtrft = lr_modelfat.predict(train)
 err = mean_squared_error(ypredtr,cases)
 err1 = mean_squared_error(ypredtrft,fatalities)
 print(err,err1 )<choose_model_class>
submission = preds_df[['id_seqpos', 'reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']]
submission.to_csv(f'submission.csv', index=False)
 print(f'wrote to submission.csv' )
regr_cs = XGBRegressor(n_estimators = 2500 , gamma = 0, learning_rate = 0.04, random_state = 42 , max_depth = 23)
 regr_ft = XGBRegressor(n_estimators = 2500 , gamma = 0, learning_rate = 0.04, random_state = 42 , max_depth = 23 )<compute_train_metric>
def print_mse(prd):
val = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 val_data = []
for mol_id in val['id'].unique() :
sample_data = val.loc[val['id'] == mol_id]
sample_seq_length = sample_data.seq_length.values[0]
for i in range(68):
sample_dict = {
'id_seqpos' : sample_data['id'].values[0] + '_' + str(i),
'reactivity_gt' : sample_data['reactivity'].values[0][i],
'deg_Mg_pH10_gt' : sample_data['deg_Mg_pH10'].values[0][i],
'deg_Mg_50C_gt' : sample_data['deg_Mg_50C'].values[0][i],
}
val_data.append(sample_dict)
 val_data = pd.DataFrame(val_data)
 val_data = val_data.merge(prd, on='id_seqpos')
 rmses = []
mses = []
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
rmse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean() **.5
mse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean()
rmses.append(rmse)
 mses.append(mse)
 print(col, rmse, mse)
 print(np.mean(rmses), np.mean(mses))
regr_cs.fit(train,cases)
 ypred = regr_cs.predict(train)
 err = mean_squared_error(ypred,cases)
 print("Training - Mean Squared Error is: ",err )<predict_on_test>
print_mse(holdouts_df )
<create_dataframe><EOS> 
print_mse(holdouts_df[holdouts_df.SN_filter == 1] )