Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<define_variables>
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
def allocate_gpu_memory(gpu_number=0):
physical_devices = tf.config.experimental.list_physical_devices('GPU')
 if physical_devices:
try:
print("Found {} GPU(s)".format(len(physical_devices)))
 tf.config.set_visible_devices(physical_devices[gpu_number], 'GPU')
 tf.config.experimental.set_memory_growth(physical_devices[gpu_number], True)
 print("
except RuntimeError as e:
print(e)
 else:
print("Not enough GPU hardware devices available")
 allocate_gpu_memory()
Ver='GRU_LSTM1'
aug_data = '.. /input/openvaccine-augmentation-data/aug_data1.csv'
debug = False
preds_str = []
for filepath, distance, neighbour_ in zip(test_file_names, distances_test, neighbors_test):
sample_result = []
sample_classes = []
for d, n in zip(distance, neighbour_):
train_file = train_files[n].split(os.sep)[-1]
class_train = file_id_mapping[train_file]
sample_classes.append(class_train)
 sample_result.append(( class_train, d))
if "new_whale" not in sample_classes:
sample_result.append(( "new_whale", 0.1))
sample_result.sort(key=lambda x: x[1])
 sample_result = sample_result[:5]
preds_str.append(" ".join([x[0] for x in sample_result]))<save_to_csv>
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def build_model(seq_len=107, pred_len=68, dropout=0.5, embed_dim=100, hidden_dim=256, type=0):
inputs = L.Input(shape=(seq_len, 6))
categorical_feat_dim = 3
categorical_fea = inputs[:, :, :categorical_feat_dim]
numerical_fea = inputs[:, :, 3:]
embed = L.Embedding(input_dim=len(token2int), output_dim=embed_dim )(categorical_fea)
 reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
reshaped = L.concatenate([reshaped, numerical_fea], axis=2)
 if type == 0:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 1:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif type == 2:
hidden = gru_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 elif type == 3:
hidden = lstm_layer(hidden_dim, dropout )(reshaped)
 hidden = lstm_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = L.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 model.compile(tf.keras.optimizers.Adam() , loss=mcrmse)
 return model
df = pd.DataFrame(preds_str, columns=["Id"])
 df['Image'] = [x.split(os.sep)[-1] for x in test_file_names]
df.to_csv("sub.csv", index=False )<load_from_csv>
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
pred_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
base_fea = np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
)
 bpps_sum_fea = np.array(df['bpps_sum'].to_list())[:,:,np.newaxis]
bpps_max_fea = np.array(df['bpps_max'].to_list())[:,:,np.newaxis]
bpps_nb_fea = np.array(df['bpps_nb'].to_list())[:,:,np.newaxis]
return np.concatenate([base_fea,bpps_sum_fea,bpps_max_fea,bpps_nb_fea], 2)
 def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=len(pred_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
train_images = glob(".. /input/train/*jpg")
 test_images = glob(".. /input/test/*jpg")
 df = pd.read_csv(".. /input/train.csv")
 df["Image"] = df["Image"].map(lambda x : ".. /input/train/"+x)
 ImageToLabelDict = dict(zip(df["Image"], df["Id"]))<prepare_x_and_y>
train = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('.. /input/stanford-covid-vaccine/test.json', lines=True )
SIZE = 64
def ImportImage(filename):
img = Image.open(filename ).convert("LA" ).resize(( SIZE,SIZE))
return np.array(img)[:,:,0]
train_img = np.array([ImportImage(img)for img in train_images])
 x = train_img<count_values>
def read_bpps_sum(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).max(axis=1))
return bpps_arr
def read_bpps_max(df):
bpps_arr = []
for mol_id in df.id.to_list() :
bpps_arr.append(np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy" ).sum(axis=1))
return bpps_arr
def read_bpps_nb(df):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps_arr = []
for mol_id in df.id.to_list() :
bpps = np.load(f".. /input/stanford-covid-vaccine/bpps/{mol_id}.npy")
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
bpps_arr.append(bpps_nb)
 return bpps_arr
train['bpps_sum'] = read_bpps_sum(train)
 test['bpps_sum'] = read_bpps_sum(test)
 train['bpps_max'] = read_bpps_max(train)
 test['bpps_max'] = read_bpps_max(test)
 train['bpps_nb'] = read_bpps_nb(train)
 test['bpps_nb'] = read_bpps_nb(test )
print("%d training images" %x.shape[0])
 print("Nbr of samples/class\tNbr of classes")
 for index, val in df["Id"].value_counts().value_counts().sort_index().iteritems() :
print("%d\t\t\t%d" %(index,val))<categorify>
kmeans_model = KMeans(n_clusters=200, random_state=110 ).fit(preprocess_inputs(train)[:,:,0])
 train['cluster_id'] = kmeans_model.labels_
class LabelOneHotEncoder() :
def __init__(self):
self.ohe = OneHotEncoder()
self.le = LabelEncoder()
def fit_transform(self, x):
features = self.le.fit_transform(x)
 return self.ohe.fit_transform(features.reshape(-1,1))
def transform(self, x):
return self.ohe.transform(self.la.transform(x.reshape(-1,1)))
 def inverse_tranform(self, x):
return self.le.inverse_transform(self.ohe.inverse_tranform(x))
def inverse_labels(self, x):
return self.le.inverse_transform(x)
 y = list(map(ImageToLabelDict.get, train_images))
lohe = LabelOneHotEncoder()
y_cat = lohe.fit_transform(y )<count_values>
aug_df = pd.read_csv(aug_data)
 display(aug_df.head() )
WeightFunction = lambda x : 1./x**0.75
ClassLabel2Index = lambda x : lohe.le.inverse_tranform([[x]])
 CountDict = dict(df["Id"].value_counts())
 class_weight_dic = { lohe.le.transform([image_name])[0] : WeightFunction(count)for image_name, count in CountDict.items() }
del CountDict<choose_model_class>
def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train)
 test = aug_data(test )
batch_size = 128
num_classes = len(y_cat.toarray() [0])
 epochs = 9
print('x_train shape:', x_train.shape)
 print(x_train.shape[0], 'train samples')
 model = Sequential()
model.add(Conv2D(48, kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
model.add(Conv2D(48,(3, 3), activation='sigmoid'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Conv2D(48,(5, 5), activation='sigmoid'))
model.add(MaxPooling2D(pool_size=(3, 3)))
 model.add(Dropout(0.33))
model.add(Flatten())
 model.add(Dense(36, activation='sigmoid'))
model.add(Dropout(0.33))
model.add(Dense(36, activation='sigmoid'))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta() ,
metrics=['accuracy'])
 model.summary()
model.fit_generator(image_gen.flow(x_train, y_train.toarray() , batch_size=batch_size),
steps_per_epoch= x_train.shape[0]//batch_size,
epochs=epochs,
verbose=1,
class_weight=class_weight_dic)
 Training accuracy: {1:.4f}'.format(*score))<save_to_csv>
if debug:
train = train[:200]
test = test[:200]
with open("sample_submission.csv","w")as f:
with warnings.catch_warnings() :
f.write("Image,Id
")
 warnings.filterwarnings("ignore",category=DeprecationWarning)
 for image in test_images:
img = ImportImage(image)
 x = img.astype("float32")
 x = image_gen.standardize(x.reshape(1,SIZE,SIZE))
y = model.predict_proba(x.reshape(1,SIZE,SIZE,1))
predicted_args = np.argsort(y)[0][::-1][:5]
predicted_tags = lohe.inverse_labels(predicted_args)
 image = split(image)[-1]
predicted_tags = " ".join(predicted_tags)
 f.write("%s,%s
" %(image, predicted_tags))<import_modules>
def train_and_predict(type = 0, FOLD_N = 5):
gkf = GroupKFold(n_splits=FOLD_N)
 public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df)
 holdouts = []
holdout_preds = []
for cv,(tr_idx, vl_idx)in enumerate(gkf.split(train, train['reactivity'], train['cluster_id'])) :
trn = train.iloc[tr_idx]
x_trn = preprocess_inputs(trn)
 y_trn = np.array(trn[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
w_trn = np.log(trn.signal_to_noise+1.1)/2
val = train.iloc[vl_idx]
x_val_all = preprocess_inputs(val)
 val = val[val.SN_filter == 1]
x_val = preprocess_inputs(val)
 y_val = np.array(val[pred_cols].values.tolist() ).transpose(( 0, 2, 1))
model = build_model(type=type)
 model_short = build_model(seq_len=107, pred_len=107,type=type)
 model_long = build_model(seq_len=130, pred_len=130,type=type)
 history = model.fit(
 x_trn, y_trn,
validation_data =(x_val, y_val),
batch_size=64,
epochs=60,
sample_weight=w_trn,
callbacks=[
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.ModelCheckpoint(f'model{Ver}_cv{cv}.h5')
 ] 
)
 fig = px.line(
 history.history, y=['loss', 'val_loss'],
labels={'index': 'epoch', 'value': 'Mean Squared Error'},
title='Training History')
 fig.show()
model.load_weights(f'model{Ver}_cv{cv}.h5')
 model_short.load_weights(f'model{Ver}_cv{cv}.h5')
 model_long.load_weights(f'model{Ver}_cv{cv}.h5')
 holdouts.append(train.iloc[vl_idx])
 holdout_preds.append(model.predict(x_val_all))
if cv == 0:
public_preds = model_short.predict(public_inputs)/FOLD_N
private_preds = model_long.predict(private_inputs)/FOLD_N
else:
public_preds += model_short.predict(public_inputs)/FOLD_N
private_preds += model_long.predict(private_inputs)/FOLD_N
return holdouts, holdout_preds, public_df, public_preds, private_df, private_preds
<set_options>
val_df, val_preds, test_df, test_preds = [], [], [], []
if debug:
nmodel = 1
else:
nmodel = 4
for i in range(nmodel):
holdouts, holdout_preds, public_df, public_preds, private_df, private_preds = train_and_predict(i)
 val_df += holdouts
val_preds += holdout_preds
test_df.append(public_df)
 test_df.append(private_df)
 test_preds.append(public_preds)
 test_preds.append(private_preds )
warnings.simplefilter(action='ignore')
<load_from_csv>
preds_ls = []
for df, preds in zip(test_df, test_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=pred_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
preds_ls = []
for df, preds in zip(val_df, val_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=pred_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
single_df['SN_filter'] = df[df['id'] == uid].SN_filter.values[0]
preds_ls.append(single_df)
 holdouts_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
nbeats_pred01_df = pd.read_csv('.. /input/previous-results/nbeats_toplvl_forecasts1.csv')
 nbeats_pred02_df = pd.read_csv('.. /input/previous-results/nbeats_toplvl_forecasts2.csv')
<define_variables>
submission = preds_df[['id_seqpos', 'reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']]
submission.to_csv(f'submission.csv', index=False)
 print(f'wrote to submission.csv' )
BUILD_ENSEMBLE = True<load_from_csv>
def print_mse(prd):
val = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 val_data = []
for mol_id in val['id'].unique() :
sample_data = val.loc[val['id'] == mol_id]
sample_seq_length = sample_data.seq_length.values[0]
for i in range(68):
sample_dict = {
'id_seqpos' : sample_data['id'].values[0] + '_' + str(i),
'reactivity_gt' : sample_data['reactivity'].values[0][i],
'deg_Mg_pH10_gt' : sample_data['deg_Mg_pH10'].values[0][i],
'deg_Mg_50C_gt' : sample_data['deg_Mg_50C'].values[0][i],
}
val_data.append(sample_dict)
 val_data = pd.DataFrame(val_data)
 val_data = val_data.merge(prd, on='id_seqpos')
 rmses = []
mses = []
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
rmse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean() **.5
mse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean()
rmses.append(rmse)
 mses.append(mse)
 print(col, rmse, mse)
 print(np.mean(rmses), np.mean(mses))
if BUILD_ENSEMBLE:
pred_01_df = pd.read_csv('.. /input/previous-results/90_submission_v1.csv')
 pred_02_df = pd.read_csv('.. /input/previous-results/93_submission_v2.csv')
 pred_03_df = pd.read_csv('.. /input/previous-results/95_submission_v3.csv')
 pred_04_df = pd.read_csv('.. /input/previous-results/97_submission_v4.csv')
 pred_05_df = pd.read_csv('.. /input/previous-results/99_submission_v5.csv')
 avg_pred =(np.array(pred_01_df.values[:,1:])
 + np.array(pred_02_df.values[:,1:])
 + np.array(pred_03_df.values[:,1:])
 + np.array(pred_04_df.values[:,1:])
 + np.array(pred_05_df.values[:,1:])
)/5.0
valid_pred_df = pd.DataFrame(avg_pred, columns=pred_01_df.columns[1:])
 submission_pred_df = pd.concat([pred_01_df['id'],valid_pred_df],axis=1)
 else:
print('Should not submit single distibution')
<load_from_csv>
print_mse(holdouts_df )
<concatenate><EOS> 
print_mse(holdouts_df[holdouts_df.SN_filter == 1] )