Kaggle competition: Mechanisms of Action (MoA) Prediction
%%time
pred_test_y = train_pred(model_lstm_atten_mix(embedding_matrix_mix), epochs = 3)
 outputs.append([pred_test_y, '2-LSTM-attention-mix'] )<predict_on_test>
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import GroupKFold, StratifiedKFold, KFold, train_test_split, cross_val_score
from sklearn.metrics import log_loss, make_scorer
from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer
import torch
import matplotlib.pyplot as plt
import random
%%time
pred_test_y = train_pred(model_lstm_atten_glove(embedding_matrix_glove), epochs = 3)
 outputs.append([pred_test_y, '2-LSTM-attention-glov'] )<split>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42)
%%time
pred_test_y_list = train_pred_snapshot(model_my_lstm(embedding_matrix_concat), epochs=5, start_epoch=3)
 outputs.extend([[pred, 'my_lstm'] for pred in pred_test_y_list] )<categorify>
test_features = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 train_drug = pd.read_csv("/kaggle/input/lish-moa/train_drug.csv")
 train_features = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_targets_scored = pd.read_csv("/kaggle/input/lish-moa/train_targets_scored.csv")
 train_targets_nonscored = pd.read_csv("/kaggle/input/lish-moa/train_targets_nonscored.csv" )
%%time
w_list = [0.15573309, 0.15155236, 0.20362984, 0.1206986, 0.14503201, 0.2233541]
w_normed = np.array(w_list)/ np.sum(w_list)
 pred_test_y = np.sum([outputs[i][0]*w_normed[i] for i in range(len(w_normed)) ], axis = 0)
 pred_test_y =(pred_test_y > 0.3434 ).astype(int )<save_to_csv>
common_names = [name for name in train_targets_nonscored.columns
if name in train_targets_scored.columns]
common_names
%%time
out_df = pd.DataFrame({"qid":test_df.values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<import_modules>
cp_indexes_train = train_features[train_features["cp_type"] == "ctl_vehicle"].index
cp_indexes_test = test_features[test_features["cp_type"] == "ctl_vehicle"].index
print(len(cp_indexes_train), len(cp_indexes_test))
tqdm.pandas(desc='Progress')
<define_variables>
embed_size = 300
max_features = 120000
maxlen = 70
batch_size = 512
n_epochs = 5
n_splits = 5
SEED = 1029<set_options>
GENES = [colname for colname in train_features.columns
if colname.startswith("g-")]
CELLS = [colname for colname in train_features.columns
if colname.startswith("c-")]
def seed_everything(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<features_selection>
def map_dataset(df):
df['cp_type'] = df['cp_type'].map({'ctl_vehicle': 0, 'trt_cp':1})
 df['cp_dose'] = df['cp_dose'].map({"D1":0, "D2":1})
 return df
def load_glove(word_index):
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')[:300]
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.005838499,0.48782197
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix
def load_para(word_index):
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = -0.0053247833,0.49346462
embed_size = all_embs.shape[1]
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
return embedding_matrix<load_from_csv>
train_features = map_dataset(train_features)
 test_features = map_dataset(test_features )
df_train = pd.read_csv(".. /input/train.csv")
 df_test = pd.read_csv(".. /input/test.csv")
 df = pd.concat([df_train ,df_test],sort=True )<feature_engineering>
def add_PCA(train, test):
n_gs = 200
n_cs = 20
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 united = pd.concat([train[GENES + CELLS], test[GENES+CELLS]], axis=0)
 united_pca_gs = pca_gs.fit_transform(united[GENES])
 united_pca_cs = pca_cs.fit_transform(united[CELLS])
 united_c_mean = np.expand_dims(united[CELLS].mean(axis=1), axis=1)
 united_g_mean = np.expand_dims(united[GENES].mean(axis=1), axis=1)
 generated_features = pd.DataFrame(np.concatenate([united_pca_gs, united_pca_cs,
united_g_mean, united_c_mean], axis=1))
train_generated = generated_features.iloc[:train.shape[0]].reset_index(drop=True)
 test_generated = generated_features.iloc[train.shape[0]:].reset_index(drop=True)
 train = pd.concat([train, train_generated], axis=1)
 print("before:", test.shape)
 print(test_generated.shape, test.shape)
 test = pd.concat([test, test_generated], axis=1)
 print("After:", test.shape)
 return train, test
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
vocab = build_vocab(df['question_text'] )<define_variables>
def QuantileTransform(train, test, transform_type):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution=transform_type)
 united = pd.concat([train[GENES+CELLS], test[GENES+CELLS]], axis=0)
 united = transformer.fit_transform(united)
 train[GENES+CELLS] = united[:train.shape[0]]
test[GENES+CELLS] = united[train.shape[0]:]
return train, test
sin = len(df_train[df_train["target"]==0])
 insin = len(df_train[df_train["target"]==1])
 persin =(sin/(sin+insin)) *100
perinsin =(insin/(sin+insin)) *100
print("
print("<feature_engineering>
train_features, test_features = QuantileTransform(train_features, test_features, 'normal' )
def build_vocab(texts):
sentences = texts.apply(lambda x: x.split() ).values
vocab = {}
for sentence in sentences:
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding" )<define_variables>
train_features = train_features.merge(train_drug, on='sig_id')
 train_features['drug_id'] = LabelEncoder().fit_transform(train_features['drug_id'] )
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<feature_engineering>
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam
from torch import optim
from torch.utils.data import Dataset, DataLoader
def add_features(df):
df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))
df['total_length'] = df['question_text'].progress_apply(len)
 df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),
axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
return df
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_numbers(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_numbers(x))
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_typical_misspell(x))
train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
train = add_features(train_df)
 test = add_features(test_df)
 features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)
 test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features, test_features)))
 features = ss.transform(features)
 test_features = ss.transform(test_features)
 tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
np.random.seed(SEED)
 trn_idx = np.random.permutation(len(train_X))
train_X = train_X[trn_idx]
train_y = train_y[trn_idx]
features = features[trn_idx]
return train_X, test_X, train_y, features, test_features, tokenizer.word_index 
<train_model>
from pytorch_tabnet.tab_model import TabNetRegressor
x_train, x_test, y_train, features, test_features, word_index = load_and_prec() 
<save_model>
class MoADataset(Dataset):
def __init__(self, X, y=None):
if y is not None:
assert X.shape[0] == y.shape[0]
self.X = torch.tensor(X, dtype=torch.float)
 self.y = None
if y is not None:
self.y = torch.tensor(y, dtype=torch.float)
 def __getitem__(self, idx):
tensor_X = self.X[idx]
if self.y is not None:
tensor_y = self.y[idx]
return tensor_X, tensor_y
else:
return tensor_X
def __len__(self):
return len(self.X )
np.save("x_train",x_train)
 np.save("x_test",x_test)
 np.save("y_train",y_train)
 np.save("features",features)
 np.save("test_features",test_features)
 np.save("word_index.npy",word_index )<load_pretrained>
class ModelMLP(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(ModelMLP, self ).__init__()
self.hidden_size = hidden_size
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.3)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, 2* hidden_size))
self.batch_norm2 = nn.BatchNorm1d(2*hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(2*hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.1)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm4 = nn.BatchNorm1d(hidden_size)
 self.dropout4 = nn.Dropout(0.1)
 self.dense4 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def get_freeze(self, target_size, freeze_all=True):
if freeze_all:
for param in self.parameters() :
param.requires_grad = False
self.dense3 = nn.Linear(self.hidden_size, self.hidden_size)
 self.dense4 = nn.Linear(self.hidden_size, target_size)
 def forward(self, x):
x = F.leaky_relu(self.dense1(x), 1e-2)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x_1 = F.relu(self.dense2(x))
x=self.batch_norm3(x_1)
 x = self.dropout3(x)
 x = self.dense4(x + x_1)
 return x
x_train = np.load("x_train.npy")
 x_test = np.load("x_test.npy")
 y_train = np.load("y_train.npy")
 features = np.load("features.npy")
 test_features = np.load("test_features.npy")
 word_index = np.load("word_index.npy" ).item()<normalization>
class ResidualNet(nn.Module):
def __init__(self, num_features, num_targets, n_hidden):
super(ResidualNet, self ).__init__()
self.block1 = nn.Sequential(
 nn.Linear(num_features, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.3),
nn.Linear(n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.3), 
)
 self.block2 = nn.Sequential(
 nn.Linear(num_features + n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.2),
nn.Linear(n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.2),
nn.Linear(n_hidden, n_hidden),
nn.ELU() ,
nn.Dropout(0.2),
nn.BatchNorm1d(n_hidden), 
)
 self.block3 = nn.Sequential(
 nn.Linear(n_hidden + n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.1),
nn.Linear(n_hidden, n_hidden),
nn.ELU() ,
nn.BatchNorm1d(n_hidden),
nn.Dropout(0.1),
nn.Linear(n_hidden, num_targets)
)
 def get_freeze(self, target_size, freeze_all=True):
if freeze_all:
for param in self.parameters() :
param.requires_grad = False
self.block3 = nn.Sequential(
 nn.Linear(512 + 256, 512),
nn.BatchNorm1d(512),
nn.Dropout(0.1),
nn.ELU() ,
nn.Linear(512, 256),
nn.BatchNorm1d(256),
nn.Dropout(0.1),
nn.ELU() ,
nn.Linear(256, target_size)
)
 def forward(self, x):
x_1 = self.block1(x)
 x_1_cat = torch.cat([x, x_1], axis=-1)
 x_2 = self.block2(x_1_cat)
 x_2_cat = torch.cat([x_1, x_2], axis=-1)
 output = self.block3(x_2_cat)
 return output
seed_everything()
glove_embeddings = load_glove(word_index)
 paragram_embeddings = load_para(word_index)
 fasttext_embeddings = load_fasttext(word_index)
 embedding_matrix = np.mean([glove_embeddings, paragram_embeddings, fasttext_embeddings], axis=0)
 del glove_embeddings, paragram_embeddings, fasttext_embeddings
gc.collect()
np.shape(embedding_matrix )<split>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size + num_features, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, inputs):
x = self.batch_norm1(inputs)
 x_1 = F.leaky_relu(self.dense1(x), 1e-2)
 x = self.batch_norm2(x_1)
 x_1 = self.dropout2(x)
 x = torch.cat([x_1, inputs], axis=-1)
 x_2 = F.leaky_relu(self.dense2(x), 1e-2)
 x = self.batch_norm3(x_2)
 x_2 = self.dropout3(x)
 x = self.dense3(x_2)
 return x
splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED ).split(x_train, y_train))
splits[:3]<choose_model_class>
class SimpleNet(nn.Module):
def __init__(self, num_features, num_targets, n_hidden):
super(SimpleNet, self ).__init__()
self.layer1 = nn.Linear(num_features, n_hidden)
 self.layer2 = nn.Linear(n_hidden, num_targets)
 def forward(self, inputs):
x = self.layer1(inputs)
 x = F.relu(x)
 x = self.layer2(x)
 return x
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 return lrs 
<choose_model_class>
PARAMS = {
"BATCH_SIZE": 1024,
"EPOCHS": 100,
"DEVICE": torch.device('cuda' if torch.cuda.is_available() else 'cpu'),
"PATIENCE": 10,
"NUM_FOLDS": 10,
"TARGETS_OUTPUT": 206,
"LEARNING_RATE": 1e-2,
"WEIGHT_DECAY" : 1e-5
}
embedding_dim = 300
embedding_path = '.. /save/embedding_matrix.npy'
use_pretrained_embedding = True
hidden_size = 60
gru_len = hidden_size
Routings = 4
Num_capsule = 5
Dim_capsule = 5
dropout_p = 0.25
rate_drop_dense = 0.28
LR = 0.001
T_epsilon = 1e-7
num_classes = 30
class Embed_Layer(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):
super(Embed_Layer, self ).__init__()
self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)
 if use_pretrained_embedding:
self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))
def forward(self, x, dropout_p=0.25):
return nn.Dropout(p=dropout_p )(self.encoder(x))
class GRU_Layer(nn.Module):
def __init__(self):
super(GRU_Layer, self ).__init__()
self.gru = nn.GRU(input_size=300,
hidden_size=gru_len,
bidirectional=True)

def init_weights(self):
ih =(param.data for name, param in self.named_parameters() if 'weight_ih' in name)
 hh =(param.data for name, param in self.named_parameters() if 'weight_hh' in name)
 b =(param.data for name, param in self.named_parameters() if 'bias' in name)
 for k in ih:
nn.init.xavier_uniform_(k)
 for k in hh:
nn.init.orthogonal_(k)
 for k in b:
nn.init.constant_(k, 0)
 def forward(self, x):
return self.gru(x)
 class Caps_Layer(nn.Module):
def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \
routings=Routings, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Caps_Layer, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = self.squash
else:
self.activation = nn.ReLU(inplace=True)
 if self.share_weights:
self.W = nn.Parameter(
 nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))
 else:
self.W = nn.Parameter(
 t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))
def forward(self, x):
if self.share_weights:
u_hat_vecs = t.matmul(x, self.W)
 else:
print('add later')
 batch_size = x.size(0)
 input_num_capsule = x.size(1)
 u_hat_vecs = u_hat_vecs.view(( batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)
 b = t.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = b.permute(0, 2, 1)
 c = F.softmax(b, dim=2)
 c = c.permute(0, 2, 1)
 b = b.permute(0, 2, 1)
 outputs = self.activation(t.einsum('bij,bijk->bik',(c, u_hat_vecs)))
 if i < self.routings - 1:
b = t.einsum('bik,bijk->bij',(outputs, u_hat_vecs))
return outputs
def squash(self, x, axis=-1):
s_squared_norm =(x ** 2 ).sum(axis, keepdim=True)
 scale = t.sqrt(s_squared_norm + T_epsilon)
 return x / scale
class Capsule_Main(nn.Module):
def __init__(self, embedding_matrix=None, vocab_size=None):
super(Capsule_Main, self ).__init__()
self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)
 self.gru_layer = GRU_Layer()
self.gru_layer.init_weights()
self.caps_layer = Caps_Layer()
self.dense_layer = Dense_Layer()
def forward(self, content):
content1 = self.embed_layer(content)
 content2, _ = self.gru_layer(
 content1)
 content3 = self.caps_layer(content2)
 output = self.dense_layer(content3)
 return output 
<normalization>
MAX_EPOCH=200
tabnet_params = dict(n_d=24, n_a=24, n_steps=1, gamma=1.3,
lambda_sparse=0, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=3e-2, weight_decay=PARAMS["WEIGHT_DECAY"]),
mask_type='entmax',
scheduler_params=dict(mode="min",
patience=5,
min_lr=1e-5,
factor=0.8,),
scheduler_fn=torch.optim.lr_scheduler.ReduceLROnPlateau,
verbose=5,
)
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1)
 class NeuralNet(nn.Module):
def __init__(self):
super(NeuralNet, self ).__init__()
fc_layer = 16
fc_layer1 = 16
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.bn = nn.BatchNorm1d(16, momentum=0.5)
 self.linear = nn.Linear(hidden_size*8+3, fc_layer1)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.fc = nn.Linear(fc_layer**2,fc_layer)
 self.out = nn.Linear(fc_layer, 1)
 self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)
 self.caps_layer = Caps_Layer()
def forward(self, x):
h_embedding = self.embedding(x[0])
 h_embedding = torch.squeeze(
 self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 content3 = self.caps_layer(h_gru)
 content3 = self.dropout(content3)
 batch_size = content3.size(0)
 content3 = content3.view(batch_size, -1)
 content3 = self.relu(self.lincaps(content3))
h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 f = torch.tensor(x[1], dtype=torch.float ).cuda()
conc = torch.cat(( h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool,f), 1)
 conc = self.relu(self.linear(conc))
conc = self.bn(conc)
 conc = self.dropout(conc)
 out = self.out(conc)
 return out<categorify>
from pytorch_tabnet.metrics import Metric
class MyDataset(Dataset):
def __init__(self,dataset):
self.dataset = dataset
def __getitem__(self, index):
data, target = self.dataset[index]
return data, target, index
def __len__(self):
return len(self.dataset )<define_variables>
class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1-y_true)*np.log(1-logits+1e-15)+ y_true*np.log(logits+1e-10)
 return np.mean(-aux )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
train_preds = np.zeros(( len(x_train)))
 test_preds = np.zeros(( len(df_test)))
 seed_everything()
x_test_cuda = torch.tensor(x_test, dtype=torch.long ).cuda()
test = torch.utils.data.TensorDataset(x_test_cuda)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 avg_losses_f = []
avg_val_losses_f = []<data_type_conversions>
class SmoothBCEwLogits(nn.Module):
def __init__(self, weight=None, reduction='mean', smoothing=0.0005):
super(SmoothBCEwLogits, self ).__init__()
self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0005):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def validation_fn(model, val_dataloader, criterion=F.binary_cross_entropy_with_logits):
net = model['model']
total_val_loss = 0
outputs = []
with torch.no_grad() :
for idx,(X, y)in enumerate(val_dataloader):
X, y = X.to(PARAMS["DEVICE"]), y.to(PARAMS["DEVICE"])
 preds = net(X)
 total_val_loss += criterion(preds[:, :206], y[:, :206] ).item()
outputs.append(preds.sigmoid().detach().cpu().numpy())
 outputs = np.concatenate(outputs)
 return outputs, total_val_loss / len(val_dataloader)
 def predict_fn(model, test_X, model_name=None):
model.eval()
test_dataset = MoADataset(test_X)
 test_loader = DataLoader(test_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=False)
 outputs = []
if model_name != 'TabNet':
with torch.no_grad() :
for idx, X in enumerate(test_loader):
X = X.to(PARAMS["DEVICE"])
 preds = model(X)
 outputs.append(preds.sigmoid().detach().cpu().numpy())
 return np.concatenate(outputs)
 else:
preds = model['model'].predict(test_X)
 preds = 1/(1 + np.exp(-model.predict(X_val)))
 return preds
def train_fn(model, optimizer, loss, train_loader, val_loader, scheduler, fold):
net, name = model['model'].to(PARAMS["DEVICE"]), model['name']
no_improvement_steps = 0
min_loss_val = 1
for epoch in range(PARAMS["EPOCHS"]):
if no_improvement_steps == PARAMS["PATIENCE"]:
print("Early stopping!")
 break
total_loss_train = 0
for idx,(X, y)in enumerate(train_loader):
optimizer.zero_grad()
X, y = X.to(PARAMS["DEVICE"]), y.to(PARAMS["DEVICE"])
 preds = net(X)
 loss_val = loss(preds, y)
 loss_val.backward()
optimizer.step()
scheduler.step()
total_loss_train += loss_val.item()
total_loss_train /= len(train_loader)
 outputs, total_loss_val = validation_fn(model, val_loader)
 if total_loss_val < min_loss_val:
min_loss_val = total_loss_val
torch.save(net.state_dict() , "{}_{}.pth".format(name, fold))
no_improvement_steps = 0
else:
no_improvement_steps += 1
print("Fold: %d Epoch: %d, train loss:%.6f | validation loss: %.6f"%(fold, epoch, total_loss_train, total_loss_val))
model['model'].load_state_dict(torch.load("{}_{}.pth".format(name, fold)))
 return model['model'], outputs
def return_model(model_name, input_size, target_size, hidden_size):
print(model_name)
 if model_name == 'ResNet':
return {"name":model_name,
"model":ResidualNet(input_size, target_size, 512)}
elif model_name == 'ModelMLP':
return {"name": model_name,
"model": ModelMLP(input_size, target_size, hidden_size)
 }
elif model_name == 'Model':
return {'name': model_name,
'model': Model(input_size, target_size, hidden_size)}
elif model_name == 'simple':
return {"name":'simple',
'model': SimpleNet(input_size, target_size, hidden_size)}
else:
raise RuntimeError("Such network doesn't exists")
 def train(model_names, train_X, train_y, groups, n_seeds, transfer_learning=None):
print("Train shape:", train_X.shape, " train_y shape:", train_y.shape)
 models_predictions = []
trained_models = []
for model_name in model_names:
val = np.random.randint(8, 30)
 n_hidden = np.random.randint(512, 1524)
 group_kfold = MultilabelStratifiedKFold(n_splits=PARAMS["NUM_FOLDS"], shuffle=True)
 preds = np.zeros(( train_y.shape[0], train_y.shape[1]), dtype=float)
 loss = SmoothBCEwLogits()
folds_models = []
for idx,(train_index, test_index)in enumerate(group_kfold.split(train_X, train_y)) :
X_train, y_train = train_X[train_index], train_y[train_index]
X_val, y_val = train_X[test_index], train_y[test_index]
if model_name != "TabNet":
if transfer_learning is not None:
model = return_model(model_name, transfer_learning['in_size'], transfer_learning['out_size'])
 model["model"].load_state_dict(torch.load("{}_{}.pth".format(model["name"], idx)))
 model["model"].get_freeze(train_y.shape[-1], True)
 else:
model = return_model(model_name, X_train.shape[-1], y_train.shape[-1], n_hidden)
 print("Train - Test:", len(train_index), len(test_index))
train_dataset = MoADataset(X_train, y_train)
 val_dataset = MoADataset(X_val, y_val)
 train_loader = DataLoader(train_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=True)
 val_loader = DataLoader(val_dataset, batch_size=PARAMS["BATCH_SIZE"], shuffle=False)
 optimizer = Adam(model["model"].parameters() , lr=PARAMS["LEARNING_RATE"], weight_decay=PARAMS["WEIGHT_DECAY"])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=PARAMS["EPOCHS"],
steps_per_epoch=len(train_loader))
model, outputs = train_fn(model, optimizer, loss, train_loader, val_loader, scheduler, idx)
 preds[test_index] = outputs
else:
tabnet_params["n_a"] = val
tabnet_params['n_d'] = val
model = TabNetRegressor(**tabnet_params)
 model.fit(X_train=X_train,
y_train=y_train[:, :206],
eval_set=[(X_val, y_val[:, :206])],
eval_name = ["validation loss"],
eval_metric = ["logits_ll"],
max_epochs=MAX_EPOCH,
patience=PARAMS['PATIENCE'], batch_size=PARAMS["BATCH_SIZE"],
virtual_batch_size=128,
num_workers=1, drop_last=False,
loss_fn=SmoothBCEwLogits())
 preds[test_index] = 1/(1 + np.exp(-model.predict(X_val)))
 folds_models.append(model)
 trained_models.append({'name': model_name, 'fold_models':folds_models})
 models_predictions.append(preds)
 return trained_models, models_predictions
for i,(train_idx, valid_idx)in enumerate(splits):
x_train = np.array(x_train)
 y_train = np.array(y_train)
 features = np.array(features)
 x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
kfold_X_features = features[train_idx.astype(int)]
kfold_X_valid_features = features[valid_idx.astype(int)]
x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32 ).cuda()
model = NeuralNet()
model.cuda()
loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')
 step_size = 300
base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,
step_size=step_size, mode='exp_range',
gamma=0.99994)
 train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)
 valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)
 train = MyDataset(train)
 valid = MyDataset(valid)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)
 print(f'Fold {i + 1}')
 for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(train_loader):
f = kfold_X_features[index]
y_pred = model([x_batch,f])
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
valid_preds_fold = np.zeros(( x_val_fold.size(0)))
 test_preds_fold = np.zeros(( len(df_test)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch, index)in enumerate(valid_loader):
f = kfold_X_valid_features[index]
y_pred = model([x_batch,f] ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(valid_loader)
 valid_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))
avg_losses_f.append(avg_loss)
 avg_val_losses_f.append(avg_val_loss)
 for i,(x_batch,)in enumerate(test_loader):
f = test_features[i * batch_size:(i+1)* batch_size]
y_pred = model([x_batch,f] ).detach()
test_preds_fold[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
train_preds[valid_idx] = valid_preds_fold
test_preds += test_preds_fold / len(splits)
 print('All \t loss={:.4f} \t val_loss={:.4f} \t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))
<compute_test_metric>
print("preds dimensions:", train_targets_scored.shape,train_targets_nonscored.shape)
 aux_train_y = train_targets_nonscored.iloc[:, 1:].to_numpy()
merged_train = train_targets_scored.merge(train_targets_nonscored, on='sig_id' ).iloc[:, 1:].to_numpy()
train_X = train_features.iloc[:, 1:-1].to_numpy()
train_y = train_targets_scored.iloc[:, 1:].to_numpy()
test_X = test_features.iloc[:, 1:].to_numpy()
groups = train_features['drug_id']
def bestThresshold(y_train,train_preds):
tmp = [0,0,0]
delta = 0
for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)) :
tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))
return delta
delta = bestThresshold(y_train,train_preds )<save_to_csv>
input_size, target_size = train_X.shape[-1], train_y.shape[-1]
MLP = ModelMLP(input_size, target_size, 1024)
 ResNet = ResidualNet(input_size, target_size, 512)
 model_names = [ 'simple','TabNet', "Model", "TabNet", 'simple', 'Model', 'TabNet', 'simple', "Model"]
submission = df_test[['qid']].copy()
submission['prediction'] =(test_preds > delta ).astype(int)
 submission.to_csv('submission.csv', index=False )<import_modules>
transfer_learning = {'in_size':train_X.shape[-1],
'out_size': aux_train_y.shape[-1]}
models, predictions = train(model_names, train_X, train_y, groups, 1 )
import gc
import os
import pickle
import random
import time
from collections import Counter, defaultdict
from functools import partial
from pathlib import Path
from psutil import cpu_count
import librosa
import numpy as np
import pandas as pd
from PIL import Image
from sklearn.model_selection import train_test_split
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam
from torch.optim.lr_scheduler import CosineAnnealingLR
from fastprogress import master_bar, progress_bar
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import transforms<set_options>
def make_folds_predictions(models, test_X):
all_predictions = []
for model in models:
name = model['name']
folds_models = model['fold_models']
predictions = []
for fold_model in folds_models:
predictions.append(predict_fn(fold_model, test_X, name))
all_predictions.append(sum(predictions)/ len(predictions))
return all_predictions
def stack_predict(meta_models, weak_models, test_X, meta_with_source=False):
weak_preds = []
weak_preds = make_folds_predictions(weak_models, test_X)
 if meta_with_source:
stacked_input = np.concatenate(test_X + weak_preds, axis=-1)
 else:
stacked_input = np.concatenate(weak_preds, axis=-1)
 meta_predictions = make_folds_predictions(meta_models, stacked_input)
 final_predictions = sum(meta_predictions)/ len(meta_predictions)
 return final_predictions
def blend_stack_models(all_models, test_X):
predictions = []
for models in all_models:
predictions.append(stack_predict(models['meta_models'],
models['weak_models'],test_X))
return sum(predictions)/ len(predictions)
 def train_stack_models(weak_learners, meta_learners, train_X, train_y, meta_params, weak_params):
control_fold = KFold(n_splits=3, shuffle=True, random_state=0)
 predictions = np.zeros(train_y.shape)
 all_models = []
for fold_num,(train_index_ctl, test_index_ctl)in enumerate(control_fold.split(train_X, train_y)) :
weak_folds = MultilabelStratifiedKFold(n_splits=3, shuffle=True, random_state=1)
 train_X_ctl, train_y_ctl = train_X[train_index_ctl], train_y[train_index_ctl]
test_X_ctl, test_y_ctl = train_X[test_index_ctl], train_y[test_index_ctl]
weak_meta_models = []
for train_index, test_index in weak_folds.split(train_X_ctl, train_y_ctl):
weak_train_X, weak_train_y = train_X_ctl[train_index], train_y_ctl[train_index]
meta_train_X, meta_train_y = train_X_ctl[test_index], train_y_ctl[test_index]
PARAMS['NUM_FOLDS'] = 3
PARAMS['EPOCHS'] = 30
print("-------------TRAIN WEAK MODELS-------------")
 weak_models, _ = train(weak_learners, weak_train_X, weak_train_y, None, 1)
 weak_predictions = make_folds_predictions(weak_models, meta_train_X)
 weak_predictions = np.concatenate(weak_predictions, axis=-1)
 print("------------TRAIN META MODELS--------------")
 PARAMS['EPOCHS'] = 100
meta_models, _ = train(meta_learners, weak_predictions, meta_train_y, None, 1)
 weak_meta_models.append({ 'weak_models': weak_models,
'meta_models': meta_models})
 all_models.append(weak_meta_models)
 preds = blend_stack_models(weak_meta_models, test_X_ctl)
 print("Preds shape", preds.shape, predictions[test_index_ctl].shape)
 predictions[test_index_ctl] = preds
return [predictions], all_models
torch.cuda.is_available()<set_options>
def compute_score(y_pred, y_true):
assert y_pred.shape == y_true.shape
score = 0
scores = []
for i in range(PARAMS["TARGETS_OUTPUT"]):
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_
scores.append(score_)
 return score / PARAMS["TARGETS_OUTPUT"], scores
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
SEED = 2019
seed_everything(SEED )<feature_engineering>
def find_best(scores):
num_targets = len(scores[0])
 target_model = {}
for i in range(num_targets):
neurals_target_scores = list([scores[j][i] for j in range(len(scores)) ])
 min_score = min(neurals_target_scores)
 index = neurals_target_scores.index(min_score)
 target_model[i] = index
return target_model
def create_final_prediction(models_predictions, target_model):
final_prediction = np.zeros(models_predictions[0].shape)
 for i in range(final_prediction.shape[-1]):
final_prediction[:, i] = models_predictions[target_model[i]][:, i]
return final_prediction
N_JOBS = cpu_count()
os.environ['MKL_NUM_THREADS'] = str(N_JOBS)
 os.environ['OMP_NUM_THREADS'] = str(N_JOBS)
 DataLoader = partial(DataLoader, num_workers=N_JOBS )<define_variables>
ensemble_prediction = sum([pred[:, :206] for pred in predictions])/ len(predictions)
 total_predictions = predictions + [ensemble_prediction]
print(len(total_predictions), len(predictions))
def _one_sample_positive_class_precisions(scores, truth):

 num_classes = scores.shape[0]
pos_class_indices = np.flatnonzero(truth > 0)
 if not len(pos_class_indices):
return pos_class_indices, np.zeros(0)
 retrieved_classes = np.argsort(scores)[::-1]
class_rankings = np.zeros(num_classes, dtype=np.int)
 class_rankings[retrieved_classes] = range(num_classes)
 retrieved_class_true = np.zeros(num_classes, dtype=np.bool)
 retrieved_class_true[class_rankings[pos_class_indices]] = True
retrieved_cumulative_hits = np.cumsum(retrieved_class_true)
 precision_at_hits =(
 retrieved_cumulative_hits[class_rankings[pos_class_indices]] / 
(1 + class_rankings[pos_class_indices].astype(np.float)))
 return pos_class_indices, precision_at_hits
def calculate_per_class_lwlrap(truth, scores):

 assert truth.shape == scores.shape
num_samples, num_classes = scores.shape
precisions_for_samples_by_classes = np.zeros(( num_samples, num_classes))
for sample_num in range(num_samples):
pos_class_indices, precision_at_hits =(
 _one_sample_positive_class_precisions(scores[sample_num, :],
truth[sample_num, :]))
precisions_for_samples_by_classes[sample_num, pos_class_indices] =(
 precision_at_hits)
 labels_per_class = np.sum(truth > 0, axis=0)
 weight_per_class = labels_per_class / float(np.sum(labels_per_class))
per_class_lwlrap =(np.sum(precisions_for_samples_by_classes, axis=0)/
np.maximum(1, labels_per_class))
return per_class_lwlrap, weight_per_class<define_variables>
score = [compute_score(preds, train_y)[0] for preds in total_predictions]
scores = [compute_score(preds, train_y)[-1] for preds in total_predictions]
print(len(scores))
dataset_dir = Path('.. /input/freesound-audio-tagging-2019')
 preprocessed_dir = Path('.. /input/fat2019_prep_mels1' )<define_variables>
target_model = find_best(scores)
 final_prediction = create_final_prediction(total_predictions, target_model )
csvs = {
'train_curated': dataset_dir / 'train_curated.csv',
'train_noisy': preprocessed_dir / 'trn_noisy_best50s.csv',
'sample_submission': dataset_dir / 'sample_submission.csv',
}
dataset = {
'train_curated': dataset_dir / 'train_curated',
'train_noisy': dataset_dir / 'train_noisy',
'test': dataset_dir / 'test',
}
mels = {
'train_curated': preprocessed_dir / 'mels_train_curated.pkl',
'train_noisy': preprocessed_dir / 'mels_trn_noisy_best50s.pkl',
'test': preprocessed_dir / 'mels_test.pkl',
}<load_from_csv>
compute_score(final_prediction, train_y)[0]
train_curated = pd.read_csv(csvs['train_curated'])
 train_noisy = pd.read_csv(csvs['train_noisy'])
 train_df = pd.concat([train_curated, train_noisy], sort=True, ignore_index=True)
 train_df.head()<load_from_csv>
def final_preds(all_models, test_X, weights=None):
if weights is None:
weights = [1/len(model_names)
 for i in range(len(model_names)) ]
models_preds = []
for models in all_models:
model_name = models['name']
fold_preds = []
for fold_model in models['fold_models']:
if model_name == 'TabNet':
preds = fold_model.predict(test_X)
 preds = 1/(1 + np.exp(-preds))
else:
preds = predict_fn(fold_model, test_X)
 fold_preds.append(preds[:, :206])
 print(len(fold_preds))
models_preds.append(sum(fold_preds)/len(fold_preds))
return models_preds
test_df = pd.read_csv(csvs['sample_submission'])
 test_df.head()<data_type_conversions>
print(test_X.shape, test_features.shape)
 print(train_X.shape)
 test_preds = final_preds(models, test_X)
 ensemble_preds = sum(test_preds)/ len(test_preds)
 total_test_preds = test_preds + [ensemble_preds]
labels = test_df.columns[1:].tolist()
labels<define_variables>
test_preds = create_final_prediction(total_test_preds, target_model )
num_classes = len(labels)
 num_classes<prepare_x_and_y>
print(test_preds.shape)
 predictions = pd.concat([test_features['sig_id'],
pd.DataFrame(test_preds, columns=train_targets_scored.columns[1:])],
axis=1)
 print(predictions.shape )
y_train = np.zeros(( len(train_df), num_classes)).astype(int)
 for i, row in enumerate(train_df['labels'].str.split(',')) :
for label in row:
idx = labels.index(label)
 y_train[i, idx] = 1
y_train.shape<load_pretrained>
predictions.to_csv("submission.csv", index=False )
with open(mels['train_curated'], 'rb')as curated, open(mels['train_noisy'], 'rb')as noisy:
x_train = pickle.load(curated)
 x_train.extend(pickle.load(noisy))
with open(mels['test'], 'rb')as test:
x_test = pickle.load(test)
 len(x_train), len(x_test )<categorify>
import xgboost as xgb
import lightgbm as lgb
from catboost import CatBoostRegressor
from hyperopt import Trials, STATUS_OK, tpe, hp, fmin
class FATTestDataset(Dataset):
def __init__(self, fnames, mels, transforms, tta=5):
super().__init__()
self.fnames = fnames
self.mels = mels
self.transforms = transforms
self.tta = tta
def __len__(self):
return len(self.fnames)* self.tta
def __getitem__(self, idx):
new_idx = idx % len(self.fnames)
 image = Image.fromarray(self.mels[new_idx], mode='RGB')
 time_dim, base_dim = image.size
crop = random.randint(0, time_dim - base_dim)
 image = image.crop([crop, 0, crop + base_dim, base_dim])
 image = self.transforms(image ).div_(255)
 fname = self.fnames[new_idx]
return image, fname<categorify>
class ObjectiveFunction() :
def __init__(self, clf, X, y, cv=5, scoring=None):
self.X = X
self.y = y
self.cv = cv
self.scoring = scoring
self.clf = clf
def change_model(self, clf):
self.clf = clf
return self
def __call__(self, space):
clf = self.clf(**space)
 scores = cross_val_score(clf, self.X, self.y, cv=self.cv, scoring=self.scoring)
 return sum(scores)/ len(scores)
 def train_classifiers(clf, X_train, y_train, space, clf_params):
predictions = []
predictors = []
X_train,X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.3, shuffle=True)
 num_targets = y_train.shape[-1]
for target in range(num_targets):
print("target :{}".format(target))
fit_labels = y_train[:, target]
predictor = clf(**clf_params)
 predictor.fit(X_train, y=fit_labels, eval_set =(X_val, y_val[:, target]))
predictors.append(predictor)
 preds = predictor.predict(X_val)
 loss = LoglossMetric().evaluate([preds], y_val[:, target])[0] / len(preds)
 print("loss:", loss)
 predictions.append(loss)
 return predictors, predictions
transforms_dict = {
'train': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
'test': transforms.Compose([
transforms.RandomHorizontalFlip(0.5),
transforms.ToTensor() ,
]),
}<choose_model_class>
space_gbm = {
'max_depth' : hp.choice('max_depth', range(5, 30, 1)) ,
'learning_rate' : hp.uniform('learning_rate', 0.01, 1),
'n_estimators' : hp.choice('n_estimators', range(20, 200, 1)) ,
'gamma' : hp.uniform('gamma', 0, 1),
'min_child_weight' : hp.uniform('min_child_weight', 1, 10),
'subsample' : hp.uniform('subsample', 0.1, 1),
'colsample_bytree' : hp.quniform('colsample_bytree', 0.1, 1.0, 0.01),
'reg_alpha' : hp.uniform('aplha', 0, 1),
'reg_lambda' : hp.uniform('lambda', 0, 1)}
space_xgb = {
'max_depth' : hp.choice('max_depth', range(2, 30, 1)) ,
'learning_rate' : hp.uniform('learning_rate', 0.01, 1),
'n_estimators' : hp.choice('n_estimators', range(20, 200, 1)) ,
'gamma' : hp.uniform('gamma', 0, 1),
'min_child_weight' : hp.uniform('min_child_weight', 0, 10),
'subsample' : hp.uniform('subsample', 0.01, 1),
'lambda': hp.uniform('lambda', 0.00, 10)}
class ConvBlock(nn.Module):
def __init__(self, in_channels, out_channels, cut_channels=0, skip=True):
super().__init__()
self.skip = skip
self.out_channels = out_channels
self.cut_channels = cut_channels
self.conv1 = nn.Sequential(
 nn.Conv2d(in_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self.conv2 = nn.Sequential(
 nn.Conv2d(out_channels, out_channels, 3, 1, 1),
nn.BatchNorm2d(out_channels),
nn.ReLU() , 
)
 self._init_weights()
def _init_weights(self):
for m in self.modules() :
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
 if m.bias is not None:
nn.init.zeros_(m.bias)
 elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
 nn.init.zeros_(m.bias)
 def forward(self, ipt):
if self.cut_channels:
ipt = ipt[:, :-self.cut_channels]
x = self.conv1(ipt)
 x = self.conv2(x)
 if self.skip:
x = torch.cat([x, ipt], 1)
 x = F.avg_pool2d(x, 2)
 return x<choose_model_class>
class LoglossObjective(object):
def calc_ders_range(self, approxes, targets, weights=None):
assert len(approxes)== len(targets)
 if weights is not None:
assert len(weights)== len(approxes)
 result = []
for index in range(len(targets)) :
e = np.exp(approxes[index])
 p = e /(1 + e)
 der1 = targets[index] - p
der2 = -p *(1 - p)
 if weights is not None:
der1 *= weights[index]
der2 *= weights[index]
result.append(( der1, der2))
return result
class LoglossMetric(object):
def get_final_error(self, error, weight):
return error /(weight + 1e-38)
 def is_max_optimal(self):
return False
def evaluate(self, approxes, target, weight=None):
assert len(approxes)== 1
assert len(target)== len(approxes[0])
 approx = approxes[0]
error_sum = 0.0
weight_sum = 0.0
for i in range(len(approx)) :
e = np.exp(approx[i])
 p = e /(1 + e)
 w = 1.0 if weight is None else weight[i]
weight_sum += w
error_sum += -w *(target[i] * np.log(p)+(1 - target[i])* np.log(1 - p))
return error_sum, weight_sum
class Classifier(nn.Module):
def __init__(self, num_classes):
super().__init__()
self.conv = nn.Sequential(
 ConvBlock(in_channels=3, out_channels=64),
ConvBlock(in_channels=64 + 3, out_channels=128),
ConvBlock(in_channels=128 + 64, out_channels=256, cut_channels=3),
ConvBlock(in_channels=256 + 128, out_channels=512, cut_channels=64, skip=False), 
)
 self.fc = nn.Sequential(
 nn.Dropout(0.2),
nn.Linear(512, 128),
nn.LeakyReLU(0.2),
nn.BatchNorm1d(128),
nn.Dropout(0.1),
nn.Linear(128, num_classes), 
)
 def forward(self, x):
x = self.conv(x)
 x = torch.mean(x, dim=3)
 x, _ = torch.max(x, dim=2)
 x = self.fc(x)
 return x<choose_model_class>
def predict_predictors(predictors, test_X):
for target_predictor in predictor:
preds = target_predictor.predict(test_X)
 predictions.append(preds)
 return np.asarray(predictions)
Classifier(num_classes=num_classes )<split>
def compute_score(y_pred, y_true):
assert y_pred.shape == y_true.shape
score = 0
scores = []
for i in range(PARAMS["TARGETS_OUTPUT"]):
score_ = log_loss(y_true[:, i], y_pred[:, i])
 print("target {}:{}".format(i, score_))
score += score_
scores.append(score_)
 return score / PARAMS["TARGETS_OUTPUT"], scores
<train_model><EOS> 
def choose_model(neural_nets_scores, catboost_predictors_scores):
target_model = {}
for target in range(206):
neural_scores = [neural_nets_scores[i][target] for i in range(len(neural_nets_scores)) ]
neural_min = min(neural_scores)
 argmin_neural = neural_scors.index(neural_min)
 if neural_min < cat_boost_predictors[target]:
target_model[target] = argmin_neural
else:
target_model[target] = -1
return target_model