Kaggle competition: Mechanisms of Action (MoA) Prediction
test_107_pred = []
test_130_pred = []
for i, _ in enumerate(folds):
best_model_path = f'fold_{i}/checkpoints/best.pth'
runner = SupervisedRunner(device='cuda', input_key=input_keys, input_target_key=input_citerion_keys)
 dataset = CompDataset(test[test['seq_length']==107], bbps_folder, device='cuda', train_mode=False)
 dataloader = DataLoader(dataset, batch_size,)
 prediction = runner.predict_loader(loader=dataloader, model=model_init() , resume=best_model_path)
 prediction = np.concatenate([b['logits'].cpu().numpy() for b in prediction])[:,:,[0,1,3,2,4]]
test_107_pred.append(prediction)
 dataset = CompDataset(test[test['seq_length']==130], bbps_folder, device='cuda', train_mode=False)
 dataloader = DataLoader(dataset, batch_size,)
 prediction = runner.predict_loader(loader=dataloader, model=model_init() , resume=best_model_path)
 prediction = np.concatenate([b['logits'].cpu().numpy() for b in prediction])[:,:,[0,1,3,2,4]]
test_130_pred.append(prediction)
 test_107_pred_oof = np.mean(test_107_pred, axis=0)
 test_130_pred_oof = np.mean(test_130_pred, axis=0)
 sample_sub = pd.read_csv(f'{path}sample_submission.csv')
 sample_sub_ids = sample_sub['id_seqpos'].map(lambda x: x[:12])
 ssi_107 = sample_sub_ids.isin(test[test['seq_length']==107]['id'])
 ssi_130 = sample_sub_ids.isin(test[test['seq_length']==130]['id'])
 sample_sub.loc[ssi_107, 'reactivity':] = test_107_pred_oof.reshape(-1, 5)
 sample_sub.loc[ssi_130, 'reactivity':] = test_130_pred_oof.reshape(-1, 5)
 sample_sub.to_csv(f"submission.csv", index=False)
 sample_sub<load_from_csv>
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1 )
SEED = 101
np.random.seed(SEED)
 tf.random.set_seed(SEED)
 DATA_DIR = Path(".. /input/stanford-covid-vaccine/")
 BPPS_DIR = DATA_DIR / "bpps"
train = pd.read_json(DATA_DIR / "train.json", lines=True)
 test = pd.read_json(DATA_DIR / "test.json", lines=True)
 aug_df = pd.read_csv('/kaggle/input/how-to-generate-augmentation-data/aug_data.csv')
 bppm_paths = list(BPPS_DIR.glob("*.npy"))
debug = False
TPU = False<define_variables>
def preprocessor_labelsmooth() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_features = train_features.drop(['sig_id'],axis=1)
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['cp_type'],axis=1)
 train_features = train_features.iloc[non_ctl_idx]
public_test_features = pd.read_csv('/kaggle/input/moapublictest/test_features.csv')
 public_test_features = public_test_features.drop('sig_id',axis=1)
 test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop('sig_id',axis=1)
 cat = 'cp_dose'
le = preprocessing.LabelEncoder()
le.fit(train_features[cat])
 train_features[cat] = le.transform(train_features[cat])
 public_test_features[cat] = le.transform(public_test_features[cat])
 test_features[cat] = le.transform(test_features[cat])
 scaler = preprocessing.MinMaxScaler()
scaler.fit(train_features.append(public_test_features.drop('cp_type',axis=1)))
 data_train = scaler.transform(train_features)
 data_test = scaler.transform(test_features.drop('cp_type',axis=1))
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return data_train, data_test, cs, gs
def preprocessor_autoencoder(data_test,cs,gs):
cells_test = data_test[:,cs]
genes_test = data_test[:,gs]
cells_autoencoder = tf.keras.models.load_model(
 '.. /input/moaaemodels/CellsAE')
 genes_autoencoder = tf.keras.models.load_model(
 '.. /input/moaaemodels/GenesAE')
 ae_cells_test = cells_autoencoder.encoder(cells_test ).numpy()
ae_genes_test = genes_autoencoder.encoder(genes_test ).numpy()
data_test = np.concatenate(( data_test[:,~(cs+gs)],ae_genes_test,ae_cells_test),axis=1)
 return data_test
def preprocessor_resnet() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return train_features,test_features,cs,gs,predictors
def preprocessor_resnet_fold(train,test,cs,gs):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test
alphabet = 'AGCU().MXBISHE'
alphabet_rna = 'AGCU'
alphabet_struc = '() XXX.'
alphabet_loop = 'MXB...I...S.H.E'
target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
non_target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_pH10', 'deg_Mg_50C', 'deg_50C']
marked_target_cols = ['reactivity', 'deg_Mg_pH10', 'deg_Mg_50C']
def get_bppm(id_):
return np.load(BPPS_DIR / f"{id_}.npy")
 def get_bpps_nb(id_):
bpps_nb_mean = 0.077522
bpps_nb_std = 0.08914
bpps = get_bppm(id_)
 bpps_nb =(bpps > 0 ).sum(axis=0)/ bpps.shape[0]
bpps_nb =(bpps_nb - bpps_nb_mean)/ bpps_nb_std
return bpps_nb
def mk_pair_map(structure, type='pm'):
pm = np.full(len(structure), 0, dtype=int)
 pd = np.full(len(structure), 0, dtype=int)
 queue = []
for i, s in enumerate(structure):
if s == "(":
queue.append(i)
 elif s == ")":
j = queue.pop()
pm[i] = j
pm[j] = i
pd[i] = i-j
pd[j] = i-j
if type == 'pm':
return pm
elif type == 'pd':
return pd
def get_structure_adj(seq_length, structure, sequence):
Ss = []
cue = []
a_structures = OrderedDict([ 
(( "A", "U"), np.zeros([seq_length, seq_length])) , 
(( "C", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "G"), np.zeros([seq_length, seq_length])) , 
(( "U", "A"), np.zeros([seq_length, seq_length])) , 
(( "G", "C"), np.zeros([seq_length, seq_length])) , 
(( "G", "U"), np.zeros([seq_length, seq_length])) ,
])
 for j in range(seq_length):
if structure[j] == "(":
cue.append(j)
 elif structure[j] == ")":
start = cue.pop()
a_structures[(sequence[start], sequence[j])][start, j] = 1
a_structures[(sequence[j], sequence[start])][j, start] = 1
a_strc = np.stack([a for a in a_structures.values() ], axis=2)
 a_strc = np.sum(a_strc, axis=2, keepdims=False)
 return a_strc
def preprocess_data(data):
data = data.loc[data['SN_filter'] == 1].copy()
data = data.reset_index(drop=True)
 return data
def step(seq_length):
data = list(range(int(seq_length)))
 newList = []
newList = [x / seq_length for x in data]
return newList
def preprocess_features(data):
data['seq'] = data.apply(lambda x: integer_encoder(x['sequence'], alphabet_rna), axis=1)
 data['struc'] = data.apply(lambda x: integer_encoder(x['structure'], alphabet_struc), axis=1)
 data['loop'] = data.apply(lambda x: integer_encoder(x['predicted_loop_type'], alphabet_loop), axis=1)
 data['step'] = data.apply(lambda x: step(x['seq_length']), axis=1)
 data['pair_dist'] = data.structure.apply(mk_pair_map, type='pd')
 data['pair_map'] = data.structure.apply(mk_pair_map, type='pm')
 data['bppm_max'] = data.apply(lambda x: get_bppm(x['id'] ).max(0), axis=1)
 data['bppm_sum'] = data.apply(lambda x: get_bppm(x['id'] ).sum(0), axis=1)
 data['bppm_nb'] = data.apply(lambda x: get_bpps_nb(x['id']), axis=1)
 data['adj_struc'] = data.apply(lambda x: get_structure_adj(x['seq_length'], x['structure'], x['sequence'] ).sum(0), axis=1)
 a = np.array(data['seq'].values.tolist())[:,:,newaxis]
b = np.array(data['struc'].values.tolist())[:,:,newaxis]
c = np.array(data['loop'].values.tolist())[:,:,newaxis]
d = np.array(data['adj_struc'].values.tolist())[:,:,newaxis]
f = np.array(data['bppm_max'].values.tolist())[:,:,newaxis]
g = np.array(data['bppm_sum'].values.tolist())[:,:,newaxis]
h = np.array(data['bppm_nb'].values.tolist())[:,:,newaxis]
features_all=np.concatenate(( a,b,c,d,f,g,h), axis = 2)
 return features_all
def preprocess_labels(data):
labels = data[target_cols].copy()
return np.array(labels.values.tolist())
 def integer_encoder(my_string, alphabet):
data = my_string.lower()
alphabet = alphabet.lower()
char_to_int = dict(( c, i)for i, c in enumerate(alphabet))
integer_encoded = [char_to_int[char] for char in data]
return np.array(integer_encoded )<merge>
model_paths = ['.. /input/moalabelsmoothmodels/LabelSmoothed','.. /input/moaaemodels/AutoEncoded','.. /input/moa-multi-input-resnet-model/TwoHeads']
n_models = 3
n_labels = labels_train.shape[1]
n_test = test_features.shape[0]
n_train = train_features.shape[0]
y_val = np.zeros(( n_models, n_train, n_labels))
y_pred = np.zeros(( n_models, n_test, n_labels))
def aug_data(df):
target_df = df.copy()
new_df = aug_df[aug_df['id'].isin(target_df['id'])]
del target_df['structure']
del target_df['predicted_loop_type']
new_df = new_df.merge(target_df, on=['id','sequence'], how='left')
 df['cnt'] = df['id'].map(new_df[['id','cnt']].set_index('id' ).to_dict() ['cnt'])
 df['log_gamma'] = 100
df['score'] = 1.0
df = df.append(new_df[df.columns])
 return df
train = aug_data(train)
 test = aug_data(test )<categorify>
p_min = 0.001
p_max = 0.999
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
train_features = preprocess_features(preprocess_data(train))
train_labels = preprocess_labels(preprocess_data(train)).transpose(0,2,1 )<import_modules>
n_seeds = 6
n_folds = 5
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 data_train, data_test, cs, gs = preprocessor_labelsmooth()
data_train_ae = preprocessor_autoencoder(data_train,cs,gs)
 data_test_ae = preprocessor_autoencoder(data_test,cs,gs)
 for seed in seeds:
fold = 0
mskf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in mskf.split(data_train,labels_train):
y_test = labels_train[test]
for i in range(2):
model_path = model_paths[i]+'_seed_'+str(seed)+'_fold_'+str(fold)
 model = tf.keras.models.load_model(model_path,custom_objects={'logloss':logloss})
 if i==1:
X_test = data_train_ae[test]
y_pred[i] += model.predict(data_test_ae)/(n_folds*n_seeds)
 else:
X_test = data_train[test]
y_pred[i] += model.predict(data_test)/(n_folds*n_seeds)
 y_val[i,test] += model.predict(X_test)/n_seeds
fold += 1
tf.keras.backend.clear_session()
del model, X_test, y_test
gc.collect()
import keras
import keras.backend as K
import tensorflow as tf
import tensorflow.keras.layers as L
from keras.models import Sequential
from keras.layers import Dropout, Embedding, LSTM, Dense, Bidirectional, Activation, Flatten, GRU
from keras.layers import BatchNormalization, SpatialDropout1D, InputLayer, Reshape, Lambda
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.optimizers import SGD, Adam, Adadelta, RMSprop
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold, KFold, GroupKFold
from sklearn.cluster import KMeans<compute_test_metric>
n_seeds = 5
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 10
train_features,test_features,cs,gs,predictors = preprocessor_resnet()
for seed in seeds:
fold = 0
kf = KFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in kf.split(train_features):
_, X_test = preprocessor_resnet_fold(train_features.iloc[train].values,
train_features.iloc[test].values,cs,gs)
 _,data_test = preprocessor_resnet_fold(train_features.iloc[train].values,
test_features.drop('cp_type',axis=1 ).values,cs,gs)
 X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
model_path = model_paths[2]+'_seed_'+str(seed)+'_fold_'+str(fold)
 model = tf.keras.models.load_model(model_path,custom_objects={'logloss':logloss})
 y_val[2,test] += model.predict([X_test,X_test_2])/n_seeds
y_pred[2] += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 fold += 1
tf.keras.backend.clear_session()
del model, X_test,X_test_2,data_test,data_test_2
gc.collect()
def MCRMSE(y_true, y_pred):
colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=1)
 return tf.reduce_mean(tf.sqrt(colwise_mse), axis=1 )<choose_model_class>
def blended_preds(y_pred,ws):
return(y_pred*ws[:,np.newaxis,np.newaxis] ).sum(axis=0)
 def blended_oof(ws):
y_vals = blended_preds(y_val,ws)
 loss = logloss(tf.constant(labels_train,dtype=tf.float32),tf.constant(y_vals,dtype=tf.float32))
return loss.numpy() +(( 1-np.sum(ws)) **2 )
if TPU:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu )<choose_model_class>
opt = dual_annealing(blended_oof,bounds=(( 0,1),(0,1),(0,1)))
 ws = opt.x
print('The weights are ' + str(ws))
print('Sum of weights is ' + str(ws.sum()))
print('OOF score is ' + str(opt.fun))
def gru_layer(hidden_dim, dropout):
return L.Bidirectional(L.GRU(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def lstm_layer(hidden_dim, dropout):
return L.Bidirectional(L.LSTM(hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer = 'orthogonal'))
def conv1d_layer(filters, kernel_size):
return L.Conv1D(filters=filters, kernel_size=kernel_size, padding='valid')
 def build_model(seq_len=107, pred_len=68, dropout1=0.0, dropout2=0.3, embed_dim=230, hidden_dim1=220, hidden_dim2=330,
type=0, filters=255, kernel_size=5):
lr = 0.0010953574938066576
inputs = L.Input(shape=(seq_len, train_features.shape[2]))
integer_fea_seq = inputs[:, :, :1]
embed_seq = L.Embedding(input_dim=len(alphabet_rna)+1, output_dim=embed_dim )(integer_fea_seq)
 reshaped_seq = tf.reshape(embed_seq, shape=(-1, embed_seq.shape[1], embed_seq.shape[2] * embed_seq.shape[3]))
integer_fea_struc = inputs[:, :, 1:2]
embed_struc = L.Embedding(input_dim=len(alphabet_struc)+1, output_dim=embed_dim )(integer_fea_struc)
 reshaped_struc = tf.reshape(embed_struc, shape=(-1, embed_struc.shape[1], embed_struc.shape[2] * embed_struc.shape[3]))
integer_fea_loop = inputs[:, :, 2:3]
embed_loop = L.Embedding(input_dim=len(alphabet_loop)+1, output_dim=embed_dim )(integer_fea_loop)
 reshaped_loop = tf.reshape(embed_loop, shape=(-1, embed_loop.shape[1], embed_loop.shape[2] * embed_loop.shape[3]))
float_fea = inputs[:, :, 3:]
concat = L.concatenate([reshaped_seq, reshaped_struc, reshaped_loop, float_fea], axis=2)
 if type == 0:
hidden = lstm_layer(hidden_dim1, dropout1 )(concat)
 hidden = gru_layer(hidden_dim2, dropout2 )(hidden)
 elif type == 1:
hidden = gru_layer(hidden_dim1, dropout1 )(concat)
 hidden = gru_layer(hidden_dim2, dropout2 )(hidden)
 elif type == 2:
hidden = gru_layer(hidden_dim1, dropout1 )(concat)
 hidden = lstm_layer(hidden_dim2, dropout2 )(hidden)
 elif type == 3:
hidden = lstm_layer(hidden_dim1, dropout1 )(concat)
 hidden = lstm_layer(hidden_dim2, dropout2 )(hidden)
 truncated = hidden[:, :pred_len]
out = L.Dense(5, activation='linear' )(truncated)
 model = tf.keras.Model(inputs=inputs, outputs=out)
 model.compile(tf.keras.optimizers.Adam(lr=lr), loss=MCRMSE)
 return model<split>
y_val = blended_preds(y_val,ws)
 y_pred = blended_preds(y_pred,ws )
<train_model><EOS> 
p_min = 0.001
p_max = 0.999
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )