Kaggle competition: Mechanisms of Action (MoA) Prediction
stopword = stopwords.words('english' )<feature_engineering>
pd.set_option('display.max_rows', 50)
 pd.set_option('display.max_columns', 50)
 KAGGLE = True
TRAIN = True
INFERENCE = True
PATH = '.. /input/lish-moa/' if KAGGLE else None
print(PATH)
 test_features = pd.read_csv(PATH + 'test_features.csv')
 train_features = pd.read_csv(PATH + 'train_features.csv')
 train_targets_scored = pd.read_csv(PATH + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(PATH + 'train_targets_nonscored.csv')
 sample_submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 drug_ids = pd.read_csv(PATH + 'train_drug.csv' )
data['q_len'] = data['question_text'].str.len()
data['q_words'] = data['question_text'].apply(lambda row: len(row.split(" ")))
 data['u_chars'] = data['question_text'].apply(lambda row: sum(1 for c in row if c.isupper()))
data['l_chars'] = data['question_text'].apply(lambda row: sum(1 for c in row if c.islower()))
data['n_stopwords'] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word in stopword))
data['n_cap_words'] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word.isupper()))
data['n_diff_words'] = data['question_text'].apply(lambda row: len(set(row.split(" "))))
data['avg_word_len'] = data['question_text'].apply(lambda row: sum(len(i)for i in row.split(" ")) /len(row.split(" ")))
 data['n_numerical_words'] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word.isnumeric()))
data["nb_simley"] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if re.findall(r"^(:\(|:\)) +$", word)))
 data["nb_symbols"] = data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if re.findall(r"[@_!
data["nb_punct"] = data['question_text'].apply(lambda row: sum(1 for c in row if(c=="'" or c==';' or c=="/" or c=='.')))
 print(data.shape)
<feature_engineering>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
test_data['q_len'] = test_data['question_text'].str.len()
test_data['q_words'] = test_data['question_text'].apply(lambda row: len(row.split(" ")))
 test_data['u_chars'] = test_data['question_text'].apply(lambda row: sum(1 for c in row if c.isupper()))
test_data['l_chars'] = test_data['question_text'].apply(lambda row: sum(1 for c in row if c.islower()))
test_data['n_stopwords'] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word in stopword))
test_data['n_cap_words'] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word.isupper()))
test_data['n_diff_words'] = test_data['question_text'].apply(lambda row: len(set(row.split(" "))))
test_data['avg_word_len'] = test_data['question_text'].apply(lambda row: sum(len(i)for i in row.split(" ")) /len(row.split(" ")))
 test_data['n_numerical_words'] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if word.isnumeric()))
test_data["nb_simley"] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if re.findall(r"^(:\(|:\)) +$", word)))
 test_data["nb_symbols"] = test_data['question_text'].apply(lambda row: sum(1 for word in row.split(" ")if re.findall(r"[@_!
test_data["nb_punct"] = test_data['question_text'].apply(lambda row: sum(1 for c in row if(c=="'" or c==';' or c=="/" or c=='.')))
 print(test_data.shape)
<define_variables>
genes_cols = [col for col in train_features.columns if col.startswith('g-')]
cells_cols = [col for col in train_features.columns if col.startswith('c-')]
puncts = [
',', '.', '"', ':', ')', '(', '-', '!', '?','|', ';', "'", '$', '&',
'/', '[', ']', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '→', '°', '€', '™', '›',
'♥', '←', '×', '§', '″', '′', 'Â', '█', 'à', '…', '“', '★', '”',
'–', '●', 'â', '►', '−', '¢', '¬', '░', '¶', '↑', '±', '▾',
'═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '⊕', '▼',
'▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',
'，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',
'¹', '≤', '‡', '₹', '´'
]<define_variables>
train_and_test_genes_features = pd.concat([train_features[genes_cols],test_features[genes_cols]])
 train_and_test_cell_features = pd.concat([train_features[cells_cols],test_features[cells_cols]])
 n_comp_genes = 600
pca_genes = PCA(n_components=n_comp_genes)
 pca_names_genes = ['pca_genes_' + str(i)for i in range(300)]
pca_genes.fit(train_and_test_genes_features)
 print(pca_genes.explained_variance_ratio_[:300].sum())
 n_comp_cells = 60
pca_cells = PCA(n_components=n_comp_cells)
 pca_names_cells = ['pca_cells_' + str(i)for i in range(n_comp_cells)]
pca_cells.fit(train_and_test_cell_features)
 print(pca_cells.explained_variance_ratio_.sum())
 train_pca_features_genes = pca_genes.transform(train_features[genes_cols])[:, :300]
test_pca_features_genes = pca_genes.transform(test_features[genes_cols])[:,:300]
train_features[pca_names_genes] = train_pca_features_genes
test_features[pca_names_genes] = test_pca_features_genes
train_pca_features_cells= pca_cells.transform(train_features[cells_cols])
 test_pca_features_cells = pca_cells.transform(test_features[cells_cols])
 train_features[pca_names_cells] = train_pca_features_cells
test_features[pca_names_cells] = test_pca_features_cells
abbreviations = {
"ain't": "is not",
"aren't": "are not",
"can't": "cannot",
"'cause": "because",
"could've": "could have",
"couldn't": "could not",
"didn't": "did not",
"doesn't": "does not",
"don't": "do not",
"hadn't": "had not",
"hasn't": "has not",
"haven't": "have not",
"he'd": "he would",
"he'll": "he will",
"he's": "he is",
"how'd": "how did",
"how'd'y": "how do you",
"how'll": "how will",
"how's": "how is",
"I'd": "I would",
"I'd've": "I would have",
"I'll": "I will",
"I'll've": "I will have",
"I'm": "I am",
"I've": "I have",
"i'd": "i would",
"i'd've": "i would have",
"i'll": "i will",
"i'll've": "i will have",
"i'm": "i am",
"i've": "i have",
"isn't": "is not",
"it'd": "it would",
"it'd've": "it would have",
"it'll": "it will",
"it'll've": "it will have",
"it's": "it is",
"let's": "let us",
"ma'am": "madam",
"mayn't": "may not",
"might've": "might have",
"mightn't": "might not",
"mightn't've": "might not have",
"must've": "must have",
"mustn't": "must not",
"mustn't've": "must not have",
"needn't": "need not",
"needn't've": "need not have",
"o'clock": "of the clock",
"oughtn't": "ought not",
"oughtn't've": "ought not have",
"shan't": "shall not",
"sha'n't": "shall not",
"shan't've": "shall not have",
"she'd": "she would",
"she'd've": "she would have",
"she'll": "she will",
"she'll've": "she will have",
"she's": "she is",
"should've": "should have",
"shouldn't": "should not",
"shouldn't've": "should not have",
"so've": "so have",
"so's": "so as",
"this's": "this is",
"that'd": "that would",
"that'd've": "that would have",
"that's": "that is",
"there'd": "there would",
"there'd've": "there would have",
"there's": "there is",
"here's": "here is",
"they'd": "they would",
"they'd've": "they would have",
"they'll": "they will",
"they'll've": "they will have",
"they're": "they are",
"they've": "they have",
"to've": "to have",
"wasn't": "was not",
"we'd": "we would",
"we'd've": "we would have",
"we'll": "we will",
"we'll've": "we will have",
"we're": "we are",
"we've": "we have",
"weren't": "were not",
"what'll": "what will",
"what'll've": "what will have",
"what're": "what are",
"what's": "what is",
"what've": "what have",
"when's": "when is",
"when've": "when have",
"where'd": "where did",
"where's": "where is",
"where've": "where have",
"who'll": "who will",
"who'll've": "who will have",
"who's": "who is",
"who've": "who have",
"why's": "why is",
"why've": "why have",
"will've": "will have",
"won't": "will not",
"won't've": "will not have",
"would've": "would have",
"wouldn't": "would not",
"wouldn't've": "would not have",
"y'all": "you all",
"y'all'd": "you all would",
"y'all'd've": "you all would have",
"y'all're": "you all are",
"y'all've": "you all have",
"you'd": "you would",
"you'd've": "you would have",
"you'll": "you will",
"you'll've": "you will have",
"you're": "you are",
"you've": "you have",
"who'd": "who would",
"who're": "who are",
"'re": " are",
"tryin'": "trying",
"doesn'": "does not",
'howdo': 'how do',
'whatare': 'what are',
'howcan': 'how can',
'howmuch': 'how much',
'howmany': 'how many',
'whydo': 'why do',
'doI': 'do I',
'theBest': 'the best',
'howdoes': 'how does',
}<define_variables>
target_scored_cols = train_targets_scored.columns.tolist() [1:]
target_nonscored_cols = train_targets_nonscored.columns.tolist() [1:]
train_df = train_features.merge(train_targets_scored,on='sig_id',how='left' ).merge(
 train_targets_nonscored,on='sig_id',how='left' ).merge(
 drug_ids,on='sig_id',how='left')
 df = train_df.sample(frac=1.,random_state=2020)
 df['kfold_scored'] = -1
kf = MultilabelStratifiedKFold(n_splits=5)
 y = df[target_scored_cols + ['drug_id']].values
for fold,(t_,v_)in enumerate(kf.split(X=df,y=y)) :
df.loc[v_,'kfold_scored'] = fold
df['kfold_nonscored'] = -1
kf = MultilabelStratifiedKFold(n_splits=5)
 y = df[target_nonscored_cols + target_scored_cols + ['drug_id']].values
for fold,(t_,v_)in enumerate(kf.split(X=df,y=y)) :
df.loc[v_,'kfold_nonscored'] = fold
spells = {
'colour': 'color',
'centre': 'center',
'favourite': 'favorite',
'travelling': 'traveling',
'counselling': 'counseling',
'theatre': 'theater',
'cancelled': 'canceled',
'labour': 'labor',
'organisation': 'organization',
'wwii': 'world war 2',
'citicise': 'criticize',
'youtu.be': 'youtube',
'youtu ': 'youtube ',
'qoura': 'quora',
'sallary': 'salary',
'Whta': 'what',
'whta': 'what',
'narcisist': 'narcissist',
'mastrubation': 'masturbation',
'mastrubate': 'masturbate',
"mastrubating": 'masturbating',
'pennis': 'penis',
'Etherium': 'ethereum',
'etherium': 'ethereum',
'narcissit': 'narcissist',
'bigdata': 'big data',
'2k17': '2017',
'2k18': '2018',
'2k19': '2020',
'qouta': 'quota',
'exboyfriend': 'ex boyfriend',
'exgirlfriend': 'ex girlfriend',
'airhostess': 'air hostess',
'whst': 'what',
'watsapp': 'whatsapp',
'demonitisation': 'demonetization',
'demonitization': 'demonetization',
'demonetisation': 'demonetization',
'quorans': 'quora user',
'quoran': 'quora user',
'pokémon': 'pokemon',
'bacteries': 'batteries',
'yr old': 'years old',
}
codes = ['\x7f', '\u200b', '\xa0', '\ufeff', '\u200e', '\u202a', '\u202c', '\u2060', '\uf0d8', '\ue019', '\uf02d', '\u200f', '\u2061', '\ue01b']
langs1 = r'[\p{Katakana}\p{Hiragana}\p{Han}]'
langs2 = r'[ஆய்தஎழுத்துஆயுதஎழுத்துशुषछछशुषدوउसशुष북한내제តើបងប្អូនមានមធ្យបាយអ្វីខ្លះដើម្បីរកឃើញឯកសារអំពីប្រវត្តិស្ត្រនៃប្រាសាទអង្គរវट्टरौरआदસંઘરાજ્યपीतऊनअहএকটিবাড়িএকটিখামারএরঅধীনেপদেরবাছাইপরীক্ষাএরপ্রশ্নওউত্তরসহকোথায়পেতেপারিص、。Емелядуракلكلمقاممقال수능ί서로가를행복하게기乡국고등학교는몇시간업니《》싱관없어나이रचा키کپڤ」मिलगईकलेजेकोठंडकऋॠऌॡर]'
compiled_langs1 = regex.compile(langs1)
 compiled_langs2 = re.compile(langs2)
<drop_column>
test_sig_ids = test_features[test_features['cp_type'] == 'ctl_vehicle']['sig_id'].values
cat_names = ['cp_type', 'cp_time', 'cp_dose']
cont_names = [c for c in train_features.columns if c not in cat_names
and c != 'sig_id'
and c != 'drug_id']
def _clean_math(x, compiled_re):
return compiled_re.sub(' <math> ', x )<categorify>
class LabelSmoothingCrossEntropyEgm(nn.Module):
y1 = None
lam = None
def __init__(self, smoothing:float=0.005, reduction='mean'):
super().__init__()
self.smoothing,self.reduction = smoothing,reduction
self.f = BCELoss(reduction = 'none')
 def forward(self, output, target):
c = output.size() [-1]
target=target.float()
with torch.no_grad() :
target = target *(1.0 - self.smoothing)+ 0.08 * self.smoothing
bce_loss = self.f(output, target)
 if(self.y1 == None):
loss = bce_loss.mean()
return loss
else:
target1=self.y1.float()
with torch.no_grad() :
target1 = target1 *(1.0 - self.smoothing)+ 0.08 * self.smoothing
bce_loss1 = self.f(output, target1)
 all_loss = torch.lerp(bce_loss, bce_loss1, self.lam)
 return all_loss.mean()
class ChangeLoss(Callback):
_order = 90
valid_loss = BCELossFlat()
train_loss = LabelSmoothingCrossEntropyEgm()
def before_batch(self, **kwargs):
val_condition =(self.learn.dls[1] == self.learn.dl)
 if(val_condition):
self.learn.loss_func = self.valid_loss
else:
self.learn.loss_func = self.train_loss
class NormalizeCallback(Callback):
def before_batch(self, **kwargs):
a_cat, a_cont = self.learn.xb
x_cont = a_cont.cpu()
x_cont =(a_cont.cpu() - mean)/std
x_cont = x_cont.float()
x_cont = x_cont.to(a_cont.device)
 self.learn.xb =(a_cat, x_cont)
 a, b = None,None
class CatMixUp(Callback):
run_before = [Normalize]
def __init__(self, alpha=0.4): self.distrib = Beta(tensor(alpha), tensor(alpha))
def before_batch(self, **kwargs):
ret_condition =(self.learn.dls[0] != self.learn.dl)
 if(ret_condition):
return
a=self.learn.xb
b, =self.learn.yb
lam = self.distrib.sample(( self.y.size(0),)).unsqueeze(-1)
 a_cat, a_cont = a
x_cat = a_cat.detach().clone()
x_cont = a_cont.detach().clone()
y=b.detach().clone()
codes = x_cat[:,0]*16 + x_cat[:,1]*4 +x_cat[:,2]
uniq_codes = torch.unique(codes)
 for code in uniq_codes:
indexes =(codes==code ).nonzero().view(-1)
 ind_perm = torch.randperm(len(indexes))
x_cont[indexes] = x_cont[indexes[ind_perm]]
y[indexes] = y[indexes[ind_perm]]
x_cont = x_cont.to(a_cont.device)
 y=y.to(a_cont.device)
 lam=lam.to(a_cont.device)
 out_cont = torch.lerp(a_cont, x_cont, lam)
 out_cont[:,772+100:772+100+300]=torch.tensor(pca_genes.transform(out_cont[:,:772].tolist())[:,:300])
 out_cont[:,-60:]=torch.tensor(pca_cells.transform(out_cont[:, 772:772+100].tolist()))
out_y = torch.lerp(b.float() , y.float() , lam)
 out_cont = out_cont.to(a_cont.device)
 out_y = out_y.to(b.device)
 self.learn.loss_func.lam = lam
self.learn.loss_func.y1 = y
self.learn.xb =(a_cat, out_cont)
 self.learn.yb =(out_y,)
 seeds = [42, 7, 9, 13, 37, 11, 5, 29, 31, 37, 41, 53]
BCE_LOSS = BCELoss(reduction='mean' )
def preprocess(x):
x = str(x ).lower()
return x
def _clean_unicode(x):
for u in codes:
if u in x:
x = x.replace(u, '')
 return x
def clean_math(df):
math_puncts = 'θπα÷⁴≠β²¾∫≥⇒¬∠＝∑Φ√½¼'
math_puncts_long = [r'\\frac', r'\[math\]', r'\[/math\]', r'\\lim']
compiled_math = re.compile('(%s)' % '|'.join(math_puncts))
compiled_math_long = re.compile('(%s)' % '|'.join(math_puncts_long))
df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math_long))
df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math))
return df
def clean_abbreviation(df, abbreviations):
compiled_abbreviation = re.compile('(%s)' % '|'.join(abbreviations.keys()))
def replace(match):
return abbreviations[match.group(0)]
df['question_text'] = df["question_text"].apply(
 lambda x: _clean_abreviation(x, compiled_abbreviation, replace)
)
 return df
def _clean_abreviation(x, compiled_re, replace):
return compiled_re.sub(replace, x)
 def clean_spells(df, spells):
compiled_spells = re.compile('(%s)' % '|'.join(spells.keys()))
def replace(match):
return spells[match.group(0)]
df['question_text'] = df["question_text"].apply(
 lambda x: _clean_spells(x, compiled_spells, replace)
)
 return df
def _clean_spells(x, compiled_re, replace):
return compiled_re.sub(replace, x)
 def _clean_language(x, compiled_re):
return compiled_re.sub(' <lang> ', x)
 def _clean_puncts(x, puncts):
x = str(x)
 for punct in puncts:
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x
def _clean_space(x, compiled_re):
return compiled_re.sub(" ", x)
<feature_engineering>
def get_data(fold, target_names,
procs = [Categorify, FillMissing],
cat_names = cat_names,
cont_names = cont_names):
val_idx = df[df.kfold_nonscored==fold].index
dls = TabularDataLoaders.from_df(df, path=PATH,
y_names=target_names,
cat_names = cat_names,
cont_names = cont_names,
procs = procs,
valid_idx=val_idx,
bs=64)
 return dls
def get_cbs(do_mixup):
ncb = NormalizeCallback()
ch_loss_cb = ChangeLoss()
if do_mixup:
return [ch_loss_cb,CatMixUp() ,ncb]
return [ncb, ch_loss_cb]
test_scores = []
results = []
def do_train_and_inf(num_iters=1, do_train=True, do_inference=False, cbs=get_cbs(False),
lr=9e-3, epochs=5, target_names=target_scored_cols,
use_pretr=False, pretr_model=False, file_name = 'something'):
global test_scores
global results
model = None
for ind in tqdm(range(num_iters)) :
seed_everything(seeds[ind])
 i = ind % 5
dls = get_data(i, target_names = target_names)
 model_dir = '/kaggle/working/' if TRAIN else '/kaggle/input/fastai-egm'
config = tabular_config(ps=0.2)
 learn = tabular_learner(dls , y_range=(0,1),
layers = [1024, 512, 512, 256],
loss_func = LabelSmoothingCrossEntropyEgm() ,
config=config,
model_dir=model_dir,
cbs=cbs 
)
 model = learn.model
if(use_pretr):
print("Will change model")
 remember = learn.model.layers[-2]
learn.model = copy.deepcopy(pretr_model)
 learn.model.layers[-2] = remember
name = file_name + str(ind)
 cb = SaveModelCallback(monitor='valid_loss',fname=name ,mode='min')
 if(do_train):
learn.fit_one_cycle(epochs, lr=slice(lr/(2.6**4),lr), cbs=cb)
 results = results + [learn.recorder.loss.value.item() ]
if(do_inference):
learn.load(name)
 test_dl = learn.dls.test_dl(test_features)
 sub = learn.get_preds(dl=test_dl)
 test_scores.append(sub[0].numpy())
 return model
data["question_text"] = data["question_text"].fillna("" ).apply(preprocess)
 data["question_text"] = data["question_text"].fillna("" ).apply(_clean_unicode)
 data = clean_math(data)
 data = clean_abbreviation(data, abbreviations)
 data = clean_spells(data, spells)
 data['question_text'] = data['question_text'].apply(lambda x: _clean_language(x, compiled_langs1))
data['question_text'] = data['question_text'].apply(lambda x: _clean_language(x, compiled_langs2))
data['question_text'] = data['question_text'].apply(lambda x: _clean_puncts(x, puncts))
compiled_re = re.compile(r"\s+")
 data['question_text'] = data["question_text"].apply(lambda x: _clean_space(x, compiled_re)) 
<feature_engineering>
model = do_train_and_inf(num_iters=1, do_train = True, do_inference=False,
cbs = get_cbs(False), lr = 9e-3, epochs = 5,
target_names = target_nonscored_cols + target_scored_cols,
use_pretr = False, pretr_model = None,
file_name = 'pretrain_')
test_data["question_text"] = test_data["question_text"].fillna("" ).apply(preprocess)
 test_data["question_text"] = test_data["question_text"].fillna("" ).apply(_clean_unicode)
 test_data = clean_math(test_data)
 test_data = clean_abbreviation(test_data, abbreviations)
 test_data = clean_spells(test_data, spells)
 test_data['question_text'] = test_data['question_text'].apply(lambda x: _clean_language(x, compiled_langs1))
test_data['question_text'] = test_data['question_text'].apply(lambda x: _clean_language(x, compiled_langs2))
test_data['question_text'] = test_data['question_text'].apply(lambda x: _clean_puncts(x, puncts))
compiled_re = re.compile(r"\s+")
 test_data['question_text'] = test_data["question_text"].apply(lambda x: _clean_space(x, compiled_re))
del puncts, spells, codes, langs1, langs2, compiled_langs1, compiled_langs2; gc.collect()<normalization>
do_train_and_inf(num_iters=10, do_train = True, do_inference=True,
cbs = get_cbs(True), lr = 9e-3, epochs = 10,
target_names = target_scored_cols,
use_pretr = True, pretr_model = model,
file_name = 'pretrain_')
 test_sc = np.array(test_scores )
col_names = [ 'q_len', 'q_words', 'u_chars', 'l_chars',
'n_stopwords', 'n_cap_words', 'n_diff_words', 'avg_word_len',
'n_numerical_words', 'nb_simley', 'nb_symbols', 'nb_punct']
data_features = data[col_names]
test_data_features = test_data[col_names]
scaler = StandardScaler().fit(data_features.values)
 train_features = scaler.transform(data_features.values)
 test_features = scaler.transform(test_data_features.values)
 data[col_names] = train_features
test_data[col_names] = test_features
del data_features, test_data_features, train_features, col_names, test_features, scaler; gc.collect()<string_transform>
for r in results:
print(str(r ).replace('.',','))
<concatenate><EOS> 
avg_prds = test_sc.mean(axis=0)
 submission = sample_submission.copy()
submission[target_scored_cols] = avg_prds
submission.loc[submission['sig_id'].isin(test_features.loc[test_features['cp_type'] =='ctl_vehicle', 'sig_id']), train_targets_scored.columns[1:]] = 0
submission.to_csv('submission.csv', index=False )