Kaggle competition: OpenVaccine: COVID-19 mRNA Vaccine Degradation Prediction
<SOS> metric: MWCRMSE Kaggle data source: stanford-covid-vaccine<choose_model_class>
SEED = 101
np.random.seed(SEED)
 tf.random.set_seed(SEED)
 DATA_DIR = Path(".. /input/stanford-covid-vaccine/")
 BPPS_DIR = DATA_DIR / "bpps"
train = pd.read_json(DATA_DIR / "train.json", lines=True)
 test = pd.read_json(DATA_DIR / "test.json", lines=True)
 aug_df = pd.read_csv('/kaggle/input/how-to-generate-augmentation-data/aug_data.csv')
 bppm_paths = list(BPPS_DIR.glob("*.npy"))
debug = False
TPU = False
def capsule() :
K.clear_session()
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(rate=0.2 )(x)
 x = Bidirectional(CuDNNGRU(100, return_sequences=True,
kernel_initializer=glorot_normal(seed=12300), recurrent_initializer=orthogonal(gain=1.0, seed=10000)) )(x)
 x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True )(x)
 x = Flatten()(x)
 x = Dense(100, activation="relu", kernel_initializer=glorot_normal(seed=12300))(x)
 x = Dropout(0.12 )(x)
 x = BatchNormalization()(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam() ,)
 return model<split>
alphabet = 'AGCU().MXBISHE'
alphabet_rna = 'AGCU'
alphabet_struc = '() XXX.'
alphabet_loop = 'MXB...I...S.H.E'
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
non_target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
marked_target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']
def get_bppm(id_):
return np.load(BPPS_DIR / f"{id_}.npy")
 def get_bpps_nb(id_):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps = get_bppm(id_)
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
return bpps_nb
def mk_pair_map(structure, type='pm'):
pm = np.full(len(structure), 0, dtype=int)
 pd = np.full(len(structure), 0, dtype=int)
 queue = []
for i, s in enumerate(structure):
if s == "(":
queue.append(i)
 elif s == ")":
j = queue.pop()
pm[i] = j
pm[j] = i
pd[i] = i-j
pd[j] = i-j
if type == 'pm':
return pm
elif type == 'pd':
return pd
def get_structure_adj(seq_length, structure, sequence):
Ss = []
cue = []
a_structures = OrderedDict([ 
(( "A", "U"), np.zeros([seq_length, seq_length])) , 
(( "C", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "A"), np.zeros([seq_length, seq_length])) , 
(( "G", "C"), np.zeros([seq_length, seq_length])) , 
(( "G", "U"), np.zeros([seq_length, seq_length])) ,
])
 for j in range(seq_length):
if structure[j] == "(":
cue.append(j)
 elif structure[j] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[j])][start, j] = 1
a_structures[(sequence[j], sequence[start])][j, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis=2)
 a_strc = np.sum(a_strc, axis=2, keepdims=False)
 return a_strc
def preprocess_data(data):
data = data.loc[data['SN_filter'] == 1].copy()
data = data.reset_index(drop=True)
 return data
def step(seq_length):
data = list(range(int(seq_length)))
 newList = []
newList = [x / seq_length for x in data]
return newList
def preprocess_features(data):
data['seq'] = data.apply(lambda x: integer_encoder(x['sequence'], alphabet_rna), axis=1)
 data['struc'] = data.apply(lambda x: integer_encoder(x['structure'], alphabet_struc), axis=1)
 data['loop'] = data.apply(lambda x: integer_encoder(x['predicted_loop_type'], alphabet_loop), axis=1)
 data['step'] = data.apply(lambda x: step(x['seq_length']), axis=1)
 data['pair_dist'] = data.structure.apply(mk_pair_map, type='pd')
 data['pair_map'] = data.structure.apply(mk_pair_map, type='pm')
 data['bppm_max'] = data.apply(lambda x: get_bppm(x['id'] ).max(0), axis=1)
 data['bppm_sum'] = data.apply(lambda x: get_bppm(x['id'] ).sum(0), axis=1)
 data['bppm_nb'] = data.apply(lambda x: get_bpps_nb(x['id']), axis=1)
 data['adj_struc'] = data.apply(lambda x: get_structure_adj(x['seq_length'], x['structure'], x['sequence'] ).sum(0), axis=1)
 a = np.array(data['seq'].values.tolist())[:,:,newaxis]
b = np.array(data['struc'].values.tolist())[:,:,newaxis]
c = np.array(data['loop'].values.tolist())[:,:,newaxis]
d = np.array(data['adj_struc'].values.tolist())[:,:,newaxis]
f = np.array(data['bppm_max'].values.tolist())[:,:,newaxis]
g = np.array(data['bppm_sum'].values.tolist())[:,:,newaxis]
h = np.array(data['bppm_nb'].values.tolist())[:,:,newaxis]
features_all=np.concatenate(( a,b,c,d,f,g,h), axis = 2)
 return features_all
def preprocess_labels(data):
labels = data[target_cols].copy()
return np.array(labels.values.tolist())
 def integer_encoder(my_string, alphabet):
data = my_string.lower()
alphabet = alphabet.lower()
char_to_int = dict(( c, i)for i, c in enumerate(alphabet))
integer_encoded = [char_to_int[char] for char in data]
return np.array(integer_encoded )
def f1_smart(y_true, y_pred):
args = np.argsort(y_pred)
 tp = y_true.sum()
fs =(tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)
 res_idx = np.argmax(fs)
 return 2 * fs[res_idx],(y_pred[args[res_idx]] + y_pred[args[res_idx + 1]])/ 2
kfold = StratifiedKFold(n_splits=5, random_state=10, shuffle=True)
 bestscore = []
y_test = np.zeros(( X_test.shape[0],))
for i,(train_index, valid_index)in enumerate(kfold.split(X, Y)) :
X_train, X_val, Y_train, Y_val = X[train_index], X[valid_index], Y[train_index], Y[valid_index]
filepath="weights_best.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.6, patience=1, min_lr=0.0001, verbose=2)
 earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')
 callbacks = [checkpoint, reduce_lr]
model = capsule()
if i == 0:print(model.summary())
 model.fit(X_train, Y_train, batch_size=512, epochs=6, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks, 
)
 model.load_weights(filepath)
 y_pred = model.predict([X_val], batch_size=1024, verbose=2)
 y_test += np.squeeze(model.predict([X_test], batch_size=1024, verbose=2)) /5
f1, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))
print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))
bestscore.append(threshold )<save_to_csv>
def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train)
 test = aug_data(test )
y_test = y_test.reshape(( -1, 1))
pred_test_y =(y_test>np.mean(bestscore)).astype(int)
 sub['prediction'] = pred_test_y
sub.to_csv("submission.csv", index=False )<set_options>
train_features = preprocess_features(preprocess_data(train))
train_labels = preprocess_labels(preprocess_data(train)).transpose(0,2,1 )
%matplotlib inline
warnings.filterwarnings('ignore')
 tqdm_notebook().pandas() 
<import_modules>
import keras
import keras.backend as K
import tensorflow as tf
import tensorflow.keras.layers as L
from keras.models import Sequential
from keras.layers import Dropout, Embedding, LSTM, Dense, Bidirectional, Activation, Flatten, GRU
from keras.layers import BatchNormalization, SpatialDropout1D, InputLayer, Reshape, Lambda
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.optimizers import SGD, Adam, Adadelta, RMSprop
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold, KFold, GroupKFold
from sklearn.cluster import KMeans
import nltk 
<define_variables>
def MCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1 )
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 seed_everything()<load_from_csv>
if TPU:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )
path = '.. /input/'
df_train = pd.read_csv(path+'train.csv')
 df_test = pd.read_csv(path+'test.csv')
 print("Number of data points in training data:", df_train.shape[0])
 print("Number of data points in test data:", df_test.shape[0] )<categorify>
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def conv1d_layer(filters, kernel_size):
return L.Conv1D(filters=filters, kernel_size=kernel_size, padding='valid')
 def build_model(seq_len=107, pred_len=68, dropout1=0.0, dropout2=0.3, embed_dim=230, hidden_dim1=220, hidden_dim2=330,
type=0, filters=255, kernel_size=5):
lr = 0.0010953574938066576
inputs = L.Input(shape=(seq_len, train_features.shape[2]))
integer_fea_seq = inputs[:, :, :1]
embed_seq = L.Embedding(input_dim=len(alphabet_rna)+1, output_dim=embed_dim )(integer_fea_seq)
 reshaped_seq = tf.reshape(embed_seq, shape=(-1, embed_seq.shape[1], embed_seq.shape[2] * embed_seq.shape[3]))
integer_fea_struc = inputs[:, :, 1:2]
embed_struc = L.Embedding(input_dim=len(alphabet_struc)+1, output_dim=embed_dim )(integer_fea_struc)
 reshaped_struc = tf.reshape(embed_struc, shape=(-1, embed_struc.shape[1], embed_struc.shape[2] * embed_struc.shape[3]))
integer_fea_loop = inputs[:, :, 2:3]
embed_loop = L.Embedding(input_dim=len(alphabet_loop)+1, output_dim=embed_dim )(integer_fea_loop)
 reshaped_loop = tf.reshape(embed_loop, shape=(-1, embed_loop.shape[1], embed_loop.shape[2] * embed_loop.shape[3]))
float_fea = inputs[:, :, 3:]
concat = L.concatenate([reshaped_seq, reshaped_struc, reshaped_loop, float_fea], axis=2)
 if type == 0:
hidden = lstm_layer(hidden_dim1, dropout1 )(concat)
 hidden = gru_layer(hidden_dim2, dropout2 )(hidden)
 elif type == 1:
hidden = gru_layer(hidden_dim1, dropout1 )(concat)
 hidden = gru_layer(hidden_dim2, dropout2 )(hidden)
 elif type == 2:
hidden = gru_layer(hidden_dim1, dropout1 )(concat)
 hidden = lstm_layer(hidden_dim2, dropout2 )(hidden)
 elif type == 3:
hidden = lstm_layer(hidden_dim1, dropout1 )(concat)
 hidden = lstm_layer(hidden_dim2, dropout2 )(hidden)
 truncated = hidden[:, :pred_len]
out = L.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 model.compile(tf.keras.optimizers.Adam(lr=lr), loss=MCRMSE)
 return model
def clean_tag(text):
if '[math]' in text:
text = re.sub('\[math\].*?math\]', '[formula]', text)
 if 'http' in text or 'www' in text:
text = re.sub('(?:(?:https?|ftp):\/\/)?[\w/\-?=%.]+\.[\w/\-?=%.]+', '[url]', text)
 return text<define_variables>
if debug:
train = train[:20]
test = test[:20]
contraction_mapping = {"We'd": "We had", "That'd": "That had", "AREN'T": "Are not", "HADN'T": "Had not", "Could've": "Could have", "LeT's": "Let us", "How'll": "How will", "They'll": "They will", "DOESN'T": "Does not", "HE'S": "He has", "O'Clock": "Of the clock", "Who'll": "Who will", "What'S": "What is", "Ain't": "Am not", "WEREN'T": "Were not", "Y'all": "You all", "Y'ALL": "You all", "Here's": "Here is", "It'd": "It had", "Should've": "Should have", "I'M": "I am", "ISN'T": "Is not", "Would've": "Would have", "He'll": "He will", "DON'T": "Do not", "She'd": "She had", "WOULDN'T": "Would not", "She'll": "She will", "IT's": "It is", "There'd": "There had", "It'll": "It will", "You'll": "You will", "He'd": "He had", "What'll": "What will", "Ma'am": "Madam", "CAN'T": "Can not", "THAT'S": "That is", "You've": "You have", "She's": "She is", "Weren't": "Were not", "They've": "They have", "Couldn't": "Could not", "When's": "When is", "Haven't": "Have not", "We'll": "We will", "That's": "That is", "We're": "We are", "They're": "They' are", "You'd": "You would", "How'd": "How did", "What're": "What are", "Hasn't": "Has not", "Wasn't": "Was not", "Won't": "Will not", "There's": "There is", "Didn't": "Did not", "Doesn't": "Does not", "You're": "You are", "He's": "He is", "SO's": "So is", "We've": "We have", "Who's": "Who is", "Wouldn't": "Would not", "Why's": "Why is", "WHO's": "Who is", "Let's": "Let us", "How's": "How is", "Can't": "Can not", "Where's": "Where is", "They'd": "They had", "Don't": "Do not", "Shouldn't":"Should not", "Aren't":"Are not", "ain't": "is not", "What's": "What is", "It's": "It is", "Isn't":"Is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def clean_contractions(text):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(" ")])
 return text<define_variables>
if debug:
n_clusters = 20
else:
n_clusters = 200
kmeans_model = KMeans(n_clusters=n_clusters, random_state=110 ).fit(preprocess_features(train)[:,:,2])
 train['cluster_id'] = kmeans_model.labels_
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′',
'█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓',
'—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━',
'⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○',
'◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐',
'├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\x97', '↺',
'∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊',
'┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´',
'↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\x93', '≧', '］',
'\x94', '∀', '♛', '\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％',
'╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭',
'┊', '╯', '☾', '➔', '∴', '\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐',
'▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\x91', '➚', '◡', '╰', '\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒',
'☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜',
'‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢',
'≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥',
'⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ',
'✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦',
'◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ',
'⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋',
'､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞',
'◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ',
'✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱',
'☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹',
'⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚',
'☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛',
'⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙',
'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘',
'˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛',
'☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑',
'✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ',
'⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘',
'⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊',
'⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷',
'◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝',
'⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢',
'≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']
def clean_punct(x):
for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x<string_transform>
batch_size = 64
def train_and_predict(type = 0, FOLD_N = 5, Ver=1):
if debug:
FOLD_N = 2
gkf = GroupKFold(n_splits=FOLD_N)
 test_107 = test.query("seq_length == 107" ).copy()
test_130 = test.query("seq_length == 130" ).copy()
inputs_107 = preprocess_features(test_107)
 inputs_130 = preprocess_features(test_130)
 holdouts = []
holdout_preds = []
for cv,(train_index, test_index)in enumerate(gkf.split(train, train['deg_Mg_pH10'], train['cluster_id'])) :
trn = train.iloc[train_index].copy()
X_train = preprocess_features(trn)
 y_train = np.array(trn[target_cols].values.tolist() ).transpose(( 0, 2, 1))
val = train.iloc[test_index].copy()
x_val_all = preprocess_features(val)
 val = val[val.SN_filter == 1]
X_test = preprocess_features(val)
 y_test = np.array(val[target_cols].values.tolist() ).transpose(( 0, 2, 1))
sample_weight = np.log(trn.signal_to_noise+1.1)*2
if TPU:
with tpu_strategy.scope() :
model = build_model(type=type)
 model_107 = build_model(seq_len=107, pred_len=107,type=type)
 model_130 = build_model(seq_len=130, pred_len=130,type=type)
 else:
model = build_model(type=type)
 model_107 = build_model(seq_len=107, pred_len=107,type=type)
 model_130 = build_model(seq_len=130, pred_len=130,type=type)
 history = model.fit(
 X_train, y_train,
validation_data =(X_test, y_test),
batch_size=batch_size,
epochs=105,
sample_weight=sample_weight,
callbacks=[
tf.keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=7, restore_best_weights=True),
tf.keras.callbacks.ReduceLROnPlateau() ,
tf.keras.callbacks.ModelCheckpoint(f'model{Ver}_cv{cv}.h5', save_weights_only=True)
 ] 
)
 model.load_weights(f'model{Ver}_cv{cv}.h5')
 model_107.load_weights(f'model{Ver}_cv{cv}.h5')
 model_130.load_weights(f'model{Ver}_cv{cv}.h5')
 holdouts.append(train.iloc[test_index].copy())
 holdout_preds.append(model.predict(x_val_all))
if cv == 0:
preds_107 = model_107.predict(inputs_107)/FOLD_N
preds_130 = model_130.predict(inputs_130)/FOLD_N
else:
preds_107 += model_107.predict(inputs_107)/FOLD_N
preds_130 += model_130.predict(inputs_130)/FOLD_N
return holdouts, holdout_preds, test_107, preds_107, test_130, preds_130
def data_cleaning(x):
x = clean_tag(x)
 x = clean_contractions(x)
 x = clean_punct(x)
 return x<categorify>
val_df, val_preds, test_df, test_preds = [], [], [], []
n_model = 4
for i in range(n_model):
holdouts, holdout_preds, test_107, preds_107, test_130, preds_130 = train_and_predict(i)
 val_df += holdouts
val_preds += holdout_preds
test_df.append(test_107)
 test_df.append(test_130)
 test_preds.append(preds_107)
 test_preds.append(preds_130 )
df_train['preprocessed_question_text'] = df_train['question_text'].progress_map(lambda x: data_cleaning(x))
df_test['preprocessed_question_text'] = df_test['question_text'].progress_map(lambda x: data_cleaning(x))<feature_engineering>
preds_ls = []
for df, preds in zip(test_df, test_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
preds_ls.append(single_df)
 preds_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
preds_ls = []
for df, preds in zip(val_df, val_preds):
for i, uid in enumerate(df.id):
single_pred = preds[i]
single_df = pd.DataFrame(single_pred, columns=target_cols)
 single_df['id_seqpos'] = [f'{uid}_{x}' for x in range(single_df.shape[0])]
single_df['SN_filter'] = df[df['id'] == uid].SN_filter.values[0]
preds_ls.append(single_df)
 holdouts_df = pd.concat(preds_ls ).groupby('id_seqpos' ).mean().reset_index()
spell_model = gensim.models.KeyedVectors.load_word2vec_format('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')
 words = spell_model.index2word
w_rank = {}
for i, word in enumerate(words):
w_rank[word] = i
WORDS = w_rank
def words(text): return re.findall(r'\w+', text.lower())
 def P(word):
"Probability of `word`"
return - WORDS.get(word, 0)
 def correction(word):
"Most probable spelling correction for word."
return max(candidates(word), key=P)
 def candidates(word):
"Generate possible spelling corrections for word."
return(known([word])or known(edits1(word)) or [word])
 def known(words):
"The subset of `words` that appear in the dictionary of WORDS."
return set(w for w in words if w in WORDS)
 def edits1(word):
"All edits that are one edit away from `word`."
letters = 'abcdefghijklmnopqrstuvwxyz'
splits = [(word[:i], word[i:])for i in range(len(word)+ 1)]
deletes = [L + R[1:] for L, R in splits if R]
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
inserts = [L + c + R for L, R in splits for c in letters]
return set(deletes + transposes + replaces + inserts)
 def edits2(word):
"All edits that are two edits away from `word`"
return(e2 for e1 in edits1(word)for e2 in edits1(e1))
def singlify(word):
return "".join([letter for i, letter in enumerate(word)if i == 0 or letter != word[i-1]] )<define_search_model>
submission = preds_df[['id_seqpos', 'reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']]
submission.to_csv(f'submission.csv', index=False)
 print(f'wrote to submission.csv' )
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer('english' )<load_pretrained>
def print_mse(prd):
val = pd.read_json('.. /input/stanford-covid-vaccine/train.json', lines=True)
 val_data = []
for mol_id in val['id'].unique() :
sample_data = val.loc[val['id'] == mol_id]
sample_seq_length = sample_data.seq_length.values[0]
for i in range(68):
sample_dict = {
'id_seqpos' : sample_data['id'].values[0] + '_' + str(i),
'reactivity_gt' : sample_data['reactivity'].values[0][i],
'deg_Mg_pH10_gt' : sample_data['deg_Mg_pH10'].values[0][i],
'deg_Mg_50C_gt' : sample_data['deg_Mg_50C'].values[0][i],
}
val_data.append(sample_dict)
 val_data = pd.DataFrame(val_data)
 val_data = val_data.merge(prd, on='id_seqpos')
 rmses = []
mses = []
for col in ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']:
rmse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean() **.5
mse =(( val_data[col] - val_data[col+'_gt'])** 2 ).mean()
rmses.append(rmse)
 mses.append(mse)
 print(col, rmse, mse)
 print(np.mean(rmses), np.mean(mses))
def load_embeddings(word_dict, lemma_dict, embedding):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 if embedding == 'glove':
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
else:
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 for key in tqdm_notebook(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
if len(key)> 1:
word = correction(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words<concatenate>
print_mse(holdouts_df )
<feature_engineering><EOS> 
print_mse(holdouts_df[holdouts_df.SN_filter == 1] )