Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<categorify>
warnings.filterwarnings('ignore')
classEncoder.classes_[trainLabels[:5]]<categorify>
def seed_everything(seed = 34):
os.environ['PYTHONHASHSEED']=str(seed)
 tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 seed_everything()
y = classEncoder.fit_transform(y)
 y[:5]<import_modules>
train = pd.read_json('/kaggle/input/stanford-covid-vaccine/train.json', lines=True)
 test = pd.read_json('/kaggle/input/stanford-covid-vaccine/test.json', lines=True)
 sample_sub = pd.read_csv('/kaggle/input/stanford-covid-vaccine/sample_submission.csv' )
from sklearn.preprocessing import StandardScaler<normalization>
print(f"Samples with signal_to_noise greater than 1: {len(train.loc[(train['signal_to_noise'] > 1)])}")
 print(f"Samples with SN_filter = 1: {len(train.loc[(train['SN_filter'] == 1)])}" )
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X )<import_modules>
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
from sklearn.model_selection import train_test_split<split>
token2int = {x:i for i, x in enumerate('().ACGUBEHIMSX')}
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size = 0.3, random_state = 99)
 X_train.shape, X_test.shape, y_train.shape, y_test.shape<import_modules>
def preprocess_inputs(df, cols=['sequence', 'structure', 'predicted_loop_type']):
return np.transpose(
 np.array(
 df[cols] 
.applymap(lambda seq: [token2int[x] for x in seq])
.values 
.tolist() 
), 
(0, 2, 1)
 )
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV<define_search_space>
def rmse(y_actual, y_pred):
mse = tf.keras.losses.mean_squared_error(y_actual, y_pred)
 return K.sqrt(mse)
 def mcrmse(y_actual, y_pred, num_scored=len(target_cols)) :
score = 0
for i in range(num_scored):
score += rmse(y_actual[:, :, i], y_pred[:, :, i])/ num_scored
return score
params = {'C':[1, 10, 50, 100, 500, 1000, 2000],
'tol': [0.001, 0.0001, 0.005]
}<choose_model_class>
train_inputs = preprocess_inputs(train)
 train_labels = np.array(train[target_cols].values.tolist() ).transpose(( 0, 2, 1))
lr = LogisticRegression(solver='newton-cg', multi_class='multinomial')
<choose_model_class>
def gru_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.GRU(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def lstm_layer(hidden_dim, dropout):
return tf.keras.layers.Bidirectional(
 tf.keras.layers.LSTM(hidden_dim,
dropout=dropout,
return_sequences=True,
kernel_initializer='orthogonal'))
def build_model(gru=False, lstm=False, seq_len=107, pred_len=68, dropout=0.4,
embed_dim=100, hidden_dim=128, layers=3):
inputs = tf.keras.layers.Input(shape=(seq_len, 3))
embed = tf.keras.layers.Embedding(input_dim=len(token2int), output_dim=embed_dim )(inputs)
 reshaped = tf.reshape(
 embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
hidden = tf.keras.layers.SpatialDropout1D (.2 )(reshaped)
 if gru:
for _ in range(layers):
hidden = gru_layer(hidden_dim, dropout )(hidden)
 elif lstm:
for _ in range(layers):
hidden = lstm_layer(hidden_dim, dropout )(hidden)
 truncated = hidden[:, :pred_len]
out = tf.keras.layers.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 adam = tf.optimizers.Adam()
radam = tfa.optimizers.RectifiedAdam()
lookahead = tfa.optimizers.Lookahead(adam, sync_period=6)
 ranger = tfa.optimizers.Lookahead(radam, sync_period=6)
 model.compile(optimizer=adam, loss=mcrmse)
 return model
LogLoss = make_scorer(log_loss, greater_is_better=False, needs_proba=True )<choose_model_class>
FOLDS = 4
EPOCHS = 120
REPEATS = 1
BATCH_SIZE = 64
VERBOSE = 2
SEED = 34
clf = GridSearchCV(lr, params, scoring = LogLoss, refit = 'True', n_jobs = 1, cv = 5)
<train_model>
public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df )
lr.fit(X_train, y_train )<train_model>
if tf.config.list_physical_devices('GPU')is not None:
print('Training on GPU' )
clf_fit = clf.fit(X_scaled, y )<compute_test_metric>
lr_callback = tf.keras.callbacks.ReduceLROnPlateau()
print('Logistic Regression Score {}'.format(lr.score(X_test, y_test)) )<train_model>
gru_histories = []
gru_private_preds = np.zeros(( private_df.shape[0], 130, 5))
gru_public_preds = np.zeros(( public_df.shape[0], 107, 5))
rskf = RepeatedStratifiedKFold(FOLDS, n_repeats = REPEATS, random_state = SEED)
 for f,(train_index, val_index)in enumerate(rskf.split(train_inputs, train['SN_filter'])) :
sv_gru = tf.keras.callbacks.ModelCheckpoint(f'gru-{f}.h5')
 train_ = train_inputs[train_index]
train_labs = train_labels[train_index]
val_ = train_inputs[val_index]
val_labs = train_labels[val_index]
gru = build_model(gru=True)
 history = gru.fit(train_, train_labs,
validation_data=(val_,val_labs),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=[lr_callback,sv_gru],
verbose = VERBOSE)
 gru_histories.append(history)
 gru_short = build_model(gru=True, seq_len=107, pred_len=107)
 gru_short.load_weights(f'gru-{f}.h5')
 gru_public_pred = gru_short.predict(public_inputs)/ FOLDS
gru_long = build_model(gru=True, seq_len=130, pred_len=130)
 gru_long.load_weights(f'gru-{f}.h5')
 gru_private_pred = gru_long.predict(private_inputs)/ FOLDS * REPEATS
gru_public_preds += gru_public_pred
gru_private_preds += gru_private_pred
del gru_short, gru_long
print('Grid Search Score {}'.format(clf_fit.score(X_scaled, y)) )<find_best_params>
print(f" GRU mean fold MCRMSE: {np.mean([min(history.history['val_loss'])for history in gru_histories])}" )
print("best params: " + str(clf.best_params_))
print("best estimator: " + str(clf.best_estimator_))
clf.cv_results_['mean_test_score'] 
<load_from_csv>
lstm_histories = []
lstm_private_preds = np.zeros(( private_df.shape[0], 130, 5))
lstm_public_preds = np.zeros(( public_df.shape[0], 107, 5))
rskf = RepeatedStratifiedKFold(FOLDS, n_repeats = REPEATS, random_state = SEED)
 for f,(train_index, val_index)in enumerate(rskf.split(train_inputs, train['SN_filter'])) :
sv_gru = tf.keras.callbacks.ModelCheckpoint(f'lstm-{f}.h5')
 train_ = train_inputs[train_index]
train_labs = train_labels[train_index]
val_ = train_inputs[val_index]
val_labs = train_labels[val_index]
lstm = build_model(lstm=True)
 history = lstm.fit(
 train_, train_labs,
validation_data=(val_,val_labs),
batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=[lr_callback,sv_gru],
verbose = VERBOSE)
 lstm_histories.append(history)
 lstm_short = build_model(lstm=True, seq_len=107, pred_len=107)
 lstm_short.load_weights(f'lstm-{f}.h5')
 lstm_public_pred = lstm_short.predict(public_inputs)/ FOLDS
lstm_long = build_model(lstm=True, seq_len=130, pred_len=130)
 lstm_long.load_weights(f'lstm-{f}.h5')
 lstm_private_pred = lstm_long.predict(private_inputs)/ FOLDS * REPEATS
lstm_public_preds += lstm_public_pred
lstm_private_preds += lstm_private_pred
del lstm_short, lstm_long
test = pd.read_csv('.. /input/leaf-classification/test.csv.zip')
 test_ids = test.pop('id')
 x_test = test.values
scaler = StandardScaler()
x_test = scaler.fit_transform(x_test)
 y_pred = clf.predict_proba(x_test )<load_from_csv>
print(f" LSTM mean fold validation MCRMSE: {np.mean([min(history.history['val_loss'])for history in lstm_histories])}" )
sample_sub = pd.read_csv('.. /input/leaf-classification/sample_submission.csv.zip', nrows = 5)
 sample_sub<save_to_csv>
public_df = test.query("seq_length == 107" ).copy()
private_df = test.query("seq_length == 130" ).copy()
public_inputs = preprocess_inputs(public_df)
 private_inputs = preprocess_inputs(private_df )
subm = pd.DataFrame(y_pred, index=test_ids, columns = classEncoder.classes_)
 subm.to_csv('Submission_LogisticReg_v1.csv' )<set_options>
preds_gru = []
for df, preds in [(public_df, gru_public_preds),(private_df, gru_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_gru.append(single_df)
 preds_gru_df = pd.concat(preds_gru)
 preds_gru_df.head()
start = time.time()
%pylab inline
data = pd.read_csv('.. /input/train.csv')
 parent_data = data.copy()
ID = data.pop('id')
 data.shape
data.describe()
y = data.pop('species')
 y = LabelEncoder().fit(y ).transform(y)
 print(y.shape)
 X = preprocessing.MinMaxScaler().fit(data ).transform(data)
 X = StandardScaler().fit(data ).transform(data)
 print(X.shape)
 X
y_cat = to_categorical(y)
 print(y_cat.shape)
 sss = StratifiedShuffleSplit(n_splits=10, test_size=0.1,random_state=12345)
 train_index, val_index = next(iter(sss.split(X, y)))
 x_train, x_val = X[train_index], X[val_index]
y_train, y_val = y_cat[train_index], y_cat[val_index]
print("x_train dim: ",x_train.shape)
 print("x_val dim: ",x_val.shape)
 print()
model1 = Sequential()
model1.add(Dense(600,input_dim=192, init='uniform', activation='relu'))
model1.add(Dropout(0.3))
model1.add(Dense(600, activation='sigmoid'))
model1.add(Dropout(0.3))
model1.add(Dense(99, activation='softmax'))
model1.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"])
 early_stopping = EarlyStopping(monitor='val_loss', patience=300)
 history = model1.fit(x_train, y_train,batch_size=192,nb_epoch=2500 ,verbose=0,
validation_data=(x_val, y_val),callbacks=[early_stopping])
 print('val_acc: ',max(history.history['val_acc']))
print('val_loss: ',min(history.history['val_loss']))
print('train_acc: ',max(history.history['acc']))
print('train_loss: ',min(history.history['loss']))
print("train/val loss ratio: ", min(history.history['loss'])/min(history.history['val_loss']))
plt.semilogy(history.history['loss'])
 plt.semilogy(history.history['val_loss'])
 plt.title('model1 loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()
plt.plot(history.history['acc'])
 plt.plot(history.history['val_acc'])
 plt.title('model1 accuracy')
 plt.ylabel('accuracy')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()
model2 = Sequential()
model2.add(Dense(1024,input_dim=192, init='glorot_normal', activation='relu'))
model2.add(Dropout(0.2))
model2.add(Dense(512, activation='sigmoid'))
model2.add(Dropout(0.2))
model2.add(Dense(99, activation='softmax'))
model2.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"])
 early_stopping = EarlyStopping(monitor='val_loss', patience=300)
 history = model2.fit(x_train, y_train,batch_size=192,nb_epoch=2500 ,verbose=0,
validation_data=(x_val, y_val),callbacks=[early_stopping])
 print('val_acc: ',max(history.history['val_acc']))
print('val_loss: ',min(history.history['val_loss']))
print('train_acc: ',max(history.history['acc']))
print('train_loss: ',min(history.history['loss']))
print("train/val loss ratio: ", min(history.history['loss'])/min(history.history['val_loss']))
plt.semilogy(history.history['loss'])
 plt.semilogy(history.history['val_loss'])
 plt.title('model2 loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()
plt.plot(history.history['acc'])
 plt.plot(history.history['val_acc'])
 plt.title('model2 accuracy')
 plt.ylabel('accuracy')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()
model3 = Sequential()
model3.add(Dense(600,input_dim=192, init='uniform', activation='relu'))
model3.add(Dropout(0.3))
model3.add(Dense(600, activation='sigmoid'))
model3.add(Dropout(0.3))
model3.add(Dense(99, activation='softmax'))
model3.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"])
 early_stopping = EarlyStopping(monitor='val_loss', patience=300)
 history = model3.fit(x_train, y_train,batch_size=192,nb_epoch=2500 ,verbose=0,
validation_data=(x_val, y_val),callbacks=[early_stopping])
 print('val_acc: ',max(history.history['val_acc']))
print('val_loss: ',min(history.history['val_loss']))
print('train_acc: ',max(history.history['acc']))
print('train_loss: ',min(history.history['loss']))
print("train/val loss ratio: ", min(history.history['loss'])/min(history.history['val_loss']))
plt.semilogy(history.history['loss'])
 plt.semilogy(history.history['val_loss'])
 plt.title('model3 loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()
plt.plot(history.history['acc'])
 plt.plot(history.history['val_acc'])
 plt.title('model3 accuracy')
 plt.ylabel('accuracy')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()
model4 = Sequential()
model4.add(Dense(768,input_dim=192, init='uniform', activation='relu'))
model4.add(Dropout(0.3))
model3.add(Dense(384, activation='sigmoid'))
model3.add(Dropout(0.3))
model4.add(Dense(99, activation='softmax'))
model4.compile(loss='categorical_crossentropy',optimizer='rmsprop', metrics = ["accuracy"])
 early_stopping = EarlyStopping(monitor='val_loss', patience=600)
 history = model4.fit(x_train, y_train,batch_size=192,nb_epoch=2500 ,verbose=0,
validation_data=(x_val, y_val),callbacks=[early_stopping])
 print('val_acc: ',max(history.history['val_acc']))
print('val_loss: ',min(history.history['val_loss']))
print('train_acc: ',max(history.history['acc']))
print('train_loss: ',min(history.history['loss']))
print("train/val loss ratio: ", min(history.history['loss'])/min(history.history['val_loss']))
plt.semilogy(history.history['loss'])
 plt.semilogy(history.history['val_loss'])
 plt.title('model4 loss')
 plt.ylabel('loss')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()
plt.plot(history.history['acc'])
 plt.plot(history.history['val_acc'])
 plt.title('model4 accuracy')
 plt.ylabel('accuracy')
 plt.xlabel('epoch')
 plt.legend(['train', 'test'], loc='upper left')
 plt.show()
test = pd.read_csv('.. /input/test.csv')
 index = test.pop('id')
 test = preprocessing.MinMaxScaler().fit(test ).transform(test)
 test = StandardScaler().fit(test ).transform(test)
 yPred1 = model1.predict_proba(test)
 yPred2 = model2.predict_proba(test)
 yPred3 = model3.predict_proba(test)
 yPred4 = model4.predict_proba(test)
 yPred =(yPred1 + yPred2 + yPred3 + yPred4)/ 4.0
yPred = pd.DataFrame(yPred,index=index,columns=sort(parent_data.species.unique()))
yPred
fp = open('submission_nn_kernel.csv','w')
 fp.write(yPred.to_csv())
 end = time.time()
print()
print(round(( end-start),2), "seconds" )<load_from_csv>
preds_lstm = []
for df, preds in [(public_df, lstm_public_preds),(private_df, lstm_private_preds)]:
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_lstm.append(single_df)
 preds_lstm_df = pd.concat(preds_lstm)
 preds_lstm_df.head()
data = pd.read_csv('.. /input/train.csv')
 ID = data.pop('id')
 or_data = data.copy()<import_modules>
blend_preds_df = pd.DataFrame()
blend_preds_df['id_seqpos'] = preds_gru_df['id_seqpos']
blend_preds_df['reactivity'] =.5*preds_gru_df['reactivity'] +.5*preds_lstm_df['reactivity']
blend_preds_df['deg_Mg_pH10'] =.5*preds_gru_df['deg_Mg_pH10'] +.5*preds_lstm_df['deg_Mg_pH10']
blend_preds_df['deg_pH10'] =.5*preds_gru_df['deg_pH10'] +.5*preds_lstm_df['deg_pH10']
blend_preds_df['deg_Mg_50C'] =.5*preds_gru_df['deg_Mg_50C'] +.5*preds_lstm_df['deg_Mg_50C']
blend_preds_df['deg_50C'] =.5*preds_gru_df['deg_50C'] +.5*preds_lstm_df['deg_50C']
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.cross_validation import train_test_split
from sklearn.model_selection import GridSearchCV<import_modules>
submission = sample_sub[['id_seqpos']].merge(blend_preds_df, on=['id_seqpos'])
 submission.head()
<categorify><EOS> 
submission.to_csv('submission.csv', index=False)
 print('Submission saved' )