Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<drop_column>
!pip install.. /input/pytorch16gpu/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl --quiet
def unknown_punct(embed, punct):
unknown = ''
for p in punct:
if p not in embed:
unknown += p
unknown += ' '
return unknown<categorify>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
def clean_special_chars(text, punct, mapping):
for p in mapping:
text = text.replace(p, mapping[p])
 for p in punct:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
text = text.replace(s, specials[s])
 return text<categorify>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 cols = [c for c in ss.columns.values if c != 'sig_id']
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
def get_mappings() :

 mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
return mispell_dict, contraction_mapping, punct_mapping, punct<categorify>
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
y_pred_clip = np.clip(y_pred, 1e-15, 1 - 1e-15)
 return - np.mean(y_true * np.log(y_pred_clip)+(1 - y_true)* np.log(1 - y_pred_clip))
train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']
del train_targets_nonscored['sig_id']
qt = load('.. /input/moa-preprocess/qt')
 train[GENES+CELLS] = qt.transform(train[GENES+CELLS])
 test[GENES+CELLS] = qt.transform(test[GENES+CELLS])
 n_comp_genes = 600
data = pd.concat([pd.DataFrame(train[GENES]), pd.DataFrame(test[GENES])])
 pca_genes = load('.. /input/moa-preprocess/pca_genes')
 data2 = pca_genes.transform(data[GENES])
 train2 = data2[:train.shape[0]]; test2 = data2[-test.shape[0]:]
train2 = pd.DataFrame(train2, columns = [f'pca_G-{i}' for i in range(n_comp_genes)])
 test2 = pd.DataFrame(test2, columns = [f'pca_G-{i}' for i in range(n_comp_genes)])
 train = pd.concat(( train, train2), axis = 1)
 test = pd.concat(( test, test2), axis = 1)
 n_comp_cells = 50
data = pd.concat([pd.DataFrame(train[CELLS]), pd.DataFrame(test[CELLS])])
 pca_cells = load('.. /input/moa-preprocess/pca_cells')
 data2 = pca_cells.transform(data[CELLS])
 train2 = data2[:train.shape[0]]; test2 = data2[-test.shape[0]:]
train2 = pd.DataFrame(train2, columns = [f'pca_C-{i}' for i in range(n_comp_cells)])
 test2 = pd.DataFrame(test2, columns = [f'pca_C-{i}' for i in range(n_comp_cells)])
 train = pd.concat(( train, train2), axis = 1)
 test = pd.concat(( test, test2), axis = 1)
 data = train.append(test)
 var_thresh = load('.. /input/moa-preprocess/var_thresh')
 data_transformed = var_thresh.transform(data.iloc[:, 3:])
 train_transformed = data_transformed[ : train.shape[0]]
test_transformed = data_transformed[-test.shape[0] : ]
train = pd.DataFrame(train[['cp_type','cp_time','cp_dose']].values.reshape(-1, 3),\
columns=['cp_type','cp_time','cp_dose'])
 train = pd.concat([train, pd.DataFrame(train_transformed)], axis=1)
 test = pd.DataFrame(test[['cp_type','cp_time','cp_dose']].values.reshape(-1, 3),\
columns=['cp_type','cp_time','cp_dose'])
 test = pd.concat([test, pd.DataFrame(test_transformed)], axis=1)
 print(train.shape)
 print(test.shape)
 train_targets = train_targets.loc[train['cp_type'] == 0].reset_index(drop = True)
 train_targets_nonscored = train_targets_nonscored.loc[train['cp_type'] == 0].reset_index(drop = True)
 train = train.loc[train['cp_type'] == 0].reset_index(drop = True)
 print(train.shape)
 top_feats = np.arange(1, train.shape[1])
 print(top_feats)
 cat_tr, cat_test, numerical_tr, numerical_test = train.loc[:, train.columns[1:3]], test.loc[:, test.columns[1:3]], train.loc[:, train.columns[3:]].values, test.loc[:, test.columns[3:]].values
cat_tr.loc[:, 'cp_time'] = cat_tr.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 cat_test.loc[:, 'cp_time'] = cat_test.loc[:, 'cp_time'].map({24: 0, 48: 1, 72: 2})
 cat_tr = cat_tr.values
cat_test = cat_test.values
targets_tr = train_targets[cols].values.astype(np.float32 )
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x<categorify>
def evals(model, X, y, verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.clamp(y_preds, 0.0,1.0 ).detach().numpy()
score = log_loss_multi(y, y_preds)
 return y_preds, score
def inference_fn(model, X ,verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.sigmoid(torch.as_tensor(y_preds)).numpy()
return y_preds
def log_loss_score(actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
def log_loss_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
def check_targets(targets):
for i in range(targets.shape[1]):
if len(np.unique(targets[:,i])) != 2:
return False
return True
def auc_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
try:
results[i] = roc_auc_score(y_true[:,i], y_pred[:,i])
 except:
pass
return results.mean()
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets, self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
sbcewlogits = SmoothBCEwLogits(smoothing = 0.0008 )
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x<feature_engineering>
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False, pretrain=False, optimizer_params=None):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 if not pretrain:
self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 else:
self.optimizer = self.optimizer_fn(self.network.parameters() ,
**optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except:
with open(os.path.join(filepath, "model_params.json")) as f:
loaded_params = json.load(f)
 saved_state_dict = torch.load(os.path.join(filepath, "network.pt"), map_location="cpu")
 except KeyError:
raise KeyError("Your zip file is missing at least one component")
 if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
loaded_params["device_name"] = device_name
self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': total_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res
def add_features(df, textCol, outputFileName):
df_features = pd.DataFrame()
df_features['total_length'] = df[textCol].apply(len)
 df_features['capitals'] = df[textCol].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df_features['caps_vs_length'] = df_features["capitals"] / df_features['total_length']
df_features['num_words'] = df[textCol].str.count('\S+')
 df_features['num_unique_words'] = df[textCol].apply(lambda comment: len(set(w for w in comment.split())))
 df_features['words_vs_unique'] = df_features['num_unique_words'] / df_features['num_words']
df_features = df_features.fillna(0)
 ss = StandardScaler()
df_features = ss.fit_transform(df_features)
 print("
FEATURES DF SAVED TO:")
 print("{0}_features.npy".format(outputFileName))
np.save("{0}_features".format(outputFileName), df_features )<feature_engineering>
class Config(object):
def __init__(self):
self.num_class = targets_tr.shape[1]
self.verbose=False
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = N_SPLITS
self.EPOCHS = 200
self.num_ensembling = N_STARTS
self.seed = 0
self.cat_emb_dim=[1] * cat_tr.shape[1]
self.cats_idx = list(range(cat_tr.shape[1]))
self.cat_dims = [len(np.unique(cat_tr[:, i])) for i in range(cat_tr.shape[1])]
self.num_numericals = numerical_tr.shape[1]
self.save_name = ".. /input/multilabel-pbestpre-tabnet/tabnet_raw_step1"
self.strategy = "KFOLD"
cfg = Config()
def run_text_preprocessing(df, textCol, outputFileName):
mispell_map, contraction_map, punct_map, punct = get_mappings()
Working on: " + outputFileName)
 ****** WORD COVERAGE BEFORE PREPROCESSING ******")
 df['lowered_question'] = df[textCol].apply(lambda x: x.lower())
 df['treated_question'] = df['lowered_question'].apply(lambda x: clean_contractions(x, contraction_map))
df['treated_question'] = df['treated_question'].apply(lambda x: clean_special_chars(x, punct, punct_map))
df['treated_question'] = df['treated_question'].apply(lambda x: clean_numbers(x))
****** FINAL WORD COVERAGE AFTER PREPROCESSING ******")
 print("
DATAFRAME SAVED TO:")
 print("{0}_processed_text.csv".format(outputFileName))
df.to_csv("{0}_processed_text.csv".format(outputFileName))
del(mispell_map)
 del(contraction_map)
 del(punct_map)
 del(punct)
 gc.collect()
return df<load_from_csv>
X_test = np.concatenate([cat_test, numerical_test], axis=1)
 if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
res = np.zeros(targets_tr.shape)
 for nums, seed in enumerate(range(cfg.num_ensembling)) :
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[ 50,100,150], gamma=0.9), scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 if CALCULATE_OOF:
preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 res[val_idx] += preds / cfg.num_ensembling
oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 p = np.stack(p)
 preds_test.append(p)
 if CALCULATE_OOF:
oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<load_from_csv>
if CALCULATE_OOF:
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 print(f'TabNet Overall OOF CV Score:', overall_oof_score )
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )<feature_engineering>
submission_tabnet = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_tabnet[cols] = preds_test.mean(1 ).mean(0)
 submission_tabnet.loc[test['cp_type'] == 1, cols] = 0
add_features(train, "question_text", "train")
 add_features(test, "question_text", "test" )<categorify>
def create_mlp_elu(num_columns, hidden_units, dropout_rates, lr):
inp = tf.keras.layers.Input(shape =(num_columns,), name = 'inp')
 x = tf.keras.layers.BatchNormalization(name = 'bn0' )(inp)
 x = tf.keras.layers.Dropout(0.3, name = 'dp0' )(x)
 for i, units in enumerate(hidden_units):
x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(units, activation = 'elu', name = f'd{i}'), name = f'w{i}' )(x)
 x = tf.keras.layers.Dropout(dropout_rates, name = f'dp{i + 1}' )(x)
 x = tf.keras.layers.BatchNormalization(name = f'bn{i + 1}' )(x)
 out = tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = 'sigmoid',
bias_initializer=tf.keras.initializers.Constant(6.3),
name = 'out206'), name = 'wn_out206' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 metrics = [tf.keras.losses.BinaryCrossentropy(name = 'mean_loss')]
model.compile(optimizer = tfa.optimizers.AdamW(weight_decay = 1e-5, learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0008),
metrics = metrics, 
)
 return model
train = run_text_preprocessing(train, "question_text", "train")
 test = run_text_preprocessing(test, "question_text", "test" )<save_to_csv>
def create_resnet(num_columns, hidden_units, dropout_rates, lr):
inp = layers.Input(shape =(num_columns,), name = 'inp')
 head_1 = tf.keras.models.Sequential([
layers.BatchNormalization() ,
layers.Dropout(dropout_rates[0]),
layers.Dense(hidden_units[0], activation = "elu"),
layers.BatchNormalization() ,
layers.Dense(hidden_units[1], activation = "elu")
 ], name = 'Head1')
 inp1 = head_1(inp)
 head_2 = tf.keras.models.Sequential([
layers.BatchNormalization() ,
layers.Dropout(dropout_rates[0]),
layers.Dense(hidden_units[2], "relu"),
layers.BatchNormalization() ,
layers.Dense(hidden_units[2], "elu"),
layers.BatchNormalization() ,
layers.Dense(hidden_units[1], "relu"),
layers.BatchNormalization() ,
layers.Dense(hidden_units[1], "elu")
 ], name = 'Head2')
 inp2 = head_2(inp1)
 inp2_avg = layers.Average(name = 'average' )([inp1, inp2])
 head_3 = tf.keras.models.Sequential([
layers.BatchNormalization() ,
layers.Dense(hidden_units[3], kernel_initializer = 'lecun_normal', activation = 'selu'),
layers.BatchNormalization() ,
layers.Dense(206, kernel_initializer = 'lecun_normal', activation = 'selu'),
layers.BatchNormalization() ,
], name = 'Head3')
 out0 = head_3(inp2_avg)
 out = layers.Dense(206, activation = "sigmoid", name = 'out_206' )(out0)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.AdamW(weight_decay = 1e-5, learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0008),
metrics = tf.keras.losses.BinaryCrossentropy(name = 'mean_loss'), 
)
 return model
df_texts = pd.concat([train.drop('target', axis=1),test])
 df_texts.to_csv("texts_processed.csv" )<drop_column>
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, lr):
inp = tf.keras.layers.Input(shape =(num_columns,), name = 'inp')
 x = tf.keras.layers.BatchNormalization(name = 'bn0' )(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0], name = 'dp0' )(x)
 for i in range(len(hidden_units)) :
x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units[i], name = f'd{i}'), name = f'wn{i}' )(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish, name = f'a{i}' )(x)
 x = tf.keras.layers.BatchNormalization(name = f'bn{i+1}' )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1], name = f'dp{i+1}' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(num_labels,
bias_initializer = tf.keras.initializers.Constant(6.3),
name = f'output_d{num_labels}'),
name = f'output_wn{num_labels}' )(x)
 out = tf.keras.layers.Activation('sigmoid', name = f'output_a{num_labels}' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.AdamW(weight_decay = 1e-5, learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0008),
metrics = tf.keras.losses.BinaryCrossentropy(name = 'mean_loss'), 
)
 return model
del(train)
 del(test)
 gc.collect()<load_pretrained>
def create_rtn(num_columns, num_labels, rethink_iter, num_layers, hidden_units, dropout_rates, lr):
inp = tf.keras.layers.Input(shape =(num_columns,), name = 'inp')
 x = tf.keras.layers.BatchNormalization(name = 'bn0' )(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0], name = 'dp0' )(x)
 x = tf.keras.layers.RepeatVector(rethink_iter, name = 'rv' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(hidden_units[0], name = 'd0'), name = 'wn0' )(x)
 x = tf.keras.layers.Activation('swish', name = 'a0' )(x)
 for i in range(num_layers):
if i != num_layers - 1:
x = tf.keras.layers.LSTM(hidden_units[1], return_sequences = True, dropout = dropout_rates[1], name = f'lstm{i}' )(x)
 else:
x = tf.keras.layers.LSTM(hidden_units[1], return_sequences = False, dropout = dropout_rates[1], name = f'lstm{i}' )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(num_labels, name = f'd{num_labels}'), name = f'wn{num_labels}' )(x)
 out = tf.keras.layers.Activation('sigmoid', name = f'out{num_labels}' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tfa.optimizers.AdamW(weight_decay = 1e-5, learning_rate = lr),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0008),
metrics = tf.keras.losses.BinaryCrossentropy(name = 'mean_loss'), 
)
 return model
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
paragram = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
wiki_news = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
NUM_WORDS = None
word_idx = build_vocab(df_texts["treated_question"], NUM_WORDS)
 NUM_WORDS = len(word_idx)+ 1
print(NUM_WORDS)
 print("
****** LOAD EMBEDDINGS ******")
 print("
---Extracting GloVe embedding---")
 embed_glove = load_embed(glove)
 index_embs(embed_glove, word_idx, NUM_WORDS, fileName="glove")
 del(embed_glove)
 gc.collect()
print("
---Extracting Paragram embedding---")
 embed_paragram = load_embed(paragram)
 index_embs(embed_paragram, word_idx, NUM_WORDS, fileName="paragram")
 del(embed_paragram)
 gc.collect()
---Extracting FastText embedding---")
<set_options>
def inference_model(X_train, Y_train_2, Y_nonscored, features, model_name, model_name_2, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name_2:
model = create_rtn(len(features), 206, **model_params)
 elif 'RESNET' in model_name_2:
model = create_resnet(len(features), **model_params)
 elif 'ELU' in model_name_2:
model = create_mlp_elu(len(features), **model_params)
 elif 'MLP' in model_name_2:
model = create_mlp(len(features), 206, **model_params)
 for nums, seed in enumerate(range(num_seeds)) :
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
if CALCULATE_OOF:
x_tr, x_val = X_train.values[tr][:, features], X_train.values[te][:, features]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if 'ELU' in model_name_2:
ckp_path = save_path + f'{model_name}_{seed}_{n}.hdf5'
else:
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 if X_test is not None:
x_tt = X_test.values[:, features]
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = log_loss_metric(y_val, val_predict)
 mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
if CALCULATE_OOF:
pass
if X_test is not None:
sub.loc[X_test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF:
oof.loc[X_train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub
gc.collect()
%reset -f<import_modules>
def inference_ensemble(X_train, Y_train_2, model_name, model_name_2, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name_2:
model = create_rtn(X_train.shape[1], 206, **model_params)
 elif 'RESNET' in model_name_2:
model = create_resnet(X_train.shape[1], **model_params)
 elif 'ELU' in model_name_2:
model = create_mlp_elu(X_train.shape[1], **model_params)
 elif 'MLP' in model_name_2:
model = create_mlp(X_train.shape[1], 206, **model_params)
 for seed in range(num_seeds):
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
if CALCULATE_OOF:
x_tr, x_val = X_train[tr], X_train[te]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if 'ELU' in model_name_2:
ckp_path = save_path + f'{model_name}_{seed}_{n}.hdf5'
else:
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 if X_test is not None:
x_tt = X_test
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = log_loss_metric(y_val, val_predict)
 mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
if CALCULATE_OOF:
pass
if X_test is not None:
sub.loc[test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF:
oof.loc[train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub
startTimePt2 = time.time()<init_hyperparams>
model_names = ['ELU_0', 'ELU_1', 'ELU_2', 'RESNET_0', 'RESNET_1', 'RESNET_2', 'MLP_0', 'MLP_1', 'MLP_2', 'RTN']
model_params = [{'hidden_units': [512, 512, 4096],
'dropout_rates': 0.45,
'lr': 1e-4,
},
{'hidden_units': [512, 1024],
'dropout_rates': 0.463,
'lr': 1e-4,
},
{'hidden_units': [1024, 1024],
'dropout_rates': 0.5,
'lr': 1e-4,
},
{'hidden_units': [128, 896, 256, 1024],
'dropout_rates': [0.5615750059111406, 0.381766362379825],
'lr': 1e-4,
} ,
{'hidden_units': [256, 1024, 512, 896],
'dropout_rates': [0.5493076960151594, 0.5121645764863383],
'lr': 1e-4,
},
{'hidden_units': [384, 1024, 256, 896],
'dropout_rates': [0.5923827297670073, 0.37098621422772815],
'lr': 1e-4,
},
{'hidden_units': [128, 1024],
'dropout_rates': [0.41458519175008285, 0.38992411412605404, 0.10265155152086326],
'lr': 1e-4,
},
{'hidden_units': [896, 128, 1664],
'dropout_rates': [0.532412647744322, 0.2498368055693044, 0.36619131749273925, 0.1386672227832089],
'lr': 1e-4,
},
{'hidden_units': [128, 896, 1024, 1792],
'dropout_rates': [0.4131287697962003, 0.33921841003415876, 0.13058255266781393, 0.20075775903486198, 0.3354496944535896],
'lr': 1e-4,
},
{'rethink_iter': 3,
'num_layers': 1,
'hidden_units': [128, 512],
'dropout_rates': [0.3, 0.4],
'lr': 1e-4,
},]
NUM_WORDS = 192111
EMBEDDING_DIM = 300
NUM_FILTERS = 100
MAX_LEN = 72
BATCH_SIZE = 512
RANDOM_STATE = 201942
NUM_EPOCH = 6
LR = 0.003
LR_MAX = LR * 6
WD = 0.011 *(BATCH_SIZE / 979591 / NUM_EPOCH)**0.5
STEP_SIZE_CLR = 2 *(int(979591 * 2)/ BATCH_SIZE)
<define_variables>
VERBOSE = 0
sample_sub_path = '.. /input/lish-moa/sample_submission.csv'
overall_oof_scores = []
oof_elu = []
oof_resnet = []
oof_mlp = []
submission_elu = []
submission_resnet = []
submission_mlp = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 if 'ELU' in model_names[m]:
model_idx = model_names[m].split('_')[1]
save_path = '.. /input/multilabel-v2/'
model_name = f'Model_{model_idx}'
elif 'RESNET' in model_names[m]:
model_idx = model_names[m].split('_')[1]
save_path = '.. /input/multilabel-v4/'
model_name = f'Model{model_idx}'
elif 'MLP' in model_names[m]:
model_idx = model_names[m].split('_')[1]
save_path = '.. /input/miltilabel-mlpstack/'
model_name = f'Model{model_idx}'
else:
save_path = '.. /input/multilabel-rtn-single/'
model_name = f'Model0'
oof_score, res, ss = inference_model(train, train_targets, train_targets_nonscored, top_feats, model_name, model_names[m],
save_path, N_STARTS, N_SPLITS, model_params[m], test, sample_sub_path, VERBOSE)
 if 'ELU' in model_names[m]:
oof_elu.append(res)
 submission_elu.append(ss)
 elif 'RESNET' in model_names[m]:
oof_resnet.append(res)
 submission_resnet.append(ss)
 elif 'MLP' in model_names[m]:
oof_mlp.append(res)
 submission_mlp.append(ss)
 else:
oof_rtn = res
submission_rtn = ss
if CALCULATE_OOF:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_scores.append(overall_oof_score)
 print(f'{model_names[m]} Overall OOF CV Score:', overall_oof_score )
np.random.seed(RANDOM_STATE)
 set_random_seed(RANDOM_STATE )<load_from_csv>
if CALCULATE_OOF:
for n, name in enumerate(model_names):
print(f'{name} OOF:\t', overall_oof_scores[n] )
df_train = pd.read_csv("train_processed_text.csv")
 df_test = pd.read_csv("test_processed_text.csv")
 print("Train shape : ", df_train.shape)
 print("Test shape : ", df_test.shape )<prepare_x_and_y>
if CALCULATE_OOF:
train_new_elu = np.concatenate([oof0[cols].values for oof0 in oof_elu], axis = 1)
 train_new_resnet = np.concatenate([oof0[cols].values for oof0 in oof_resnet], axis = 1)
 train_new_mlp = np.concatenate([oof0[cols].values for oof0 in oof_mlp], axis = 1)
 else:
train_new_elu = np.zeros(( train.shape[0], len(cols)* 3))
train_new_resnet = np.zeros(( train.shape[0], len(cols)* 3))
train_new_mlp = np.zeros(( train.shape[0], len(cols)* 3))
test_new_elu = np.concatenate([sub[cols].values for sub in submission_elu], axis = 1)
 test_new_resnet = np.concatenate([sub[cols].values for sub in submission_resnet], axis = 1)
 test_new_mlp = np.concatenate([sub[cols].values for sub in submission_mlp], axis = 1 )
X_train = df_train["treated_question"].fillna("_na_" ).values
X_test = df_test["treated_question"].fillna("_na_" ).values
y_train = df_train['target'].values<prepare_x_and_y>
model_names = ['ELU_stack', 'RESNET_stack', 'MLP_stack']
model_params = [{'hidden_units': [1024, 1024],
'dropout_rates': 0.336,
'lr': 1e-4,
},
{'hidden_units': [1024, 896, 896, 256],
'dropout_rates': [0.5398428318872255, 0.5734093398641228],
'lr': 1e-4,
},
{'hidden_units': [1920, 768],
'dropout_rates': [0.36130273975713795, 0.38130486900003896, 0.44485672673556004],
'lr': 1e-4,
},]
X_features_train = np.load("train_features.npy")
 X_features_test = np.load("test_features.npy" )<load_pretrained>
overall_oof_stack_scores = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 if 'ELU' in model_names[m]:
save_path = '.. /input/multilabel-v2/'
model_name = f'EModel'
oof_score, res, ss = inference_ensemble(train_new_elu, train_targets, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test_new_elu, sample_sub_path, VERBOSE)
 oof_elu_stack = res.copy()
submission_elu_stack = ss.copy()
elif 'RESNET' in model_names[m]:
save_path = '.. /input/multilabel-v4/'
model_name = f'EModel_Stack'
oof_score, res, ss = inference_ensemble(train_new_resnet, train_targets, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test_new_resnet, sample_sub_path, VERBOSE)
 oof_resnet_stack = res.copy()
submission_resnet_stack = ss.copy()
elif 'MLP' in model_names[m]:
save_path = '.. /input/miltilabel-mlpstack/'
model_name = f'EModel_Stack'
oof_score, res, ss = inference_ensemble(train_new_mlp, train_targets, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test_new_mlp, sample_sub_path, VERBOSE)
 oof_mlp_stack = res.copy()
submission_mlp_stack = ss.copy()
if CALCULATE_OOF:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_stack_scores.append(overall_oof_score)
 print(f'{model_names[m]} Overall OOF CV Score:', overall_oof_score )
with open('tokenizer.pickle', 'rb')as handle:
tokenizer = pickle.load(handle )<define_variables>
if CALCULATE_OOF:
for n, name in enumerate(model_names):
print(f'{name} OOF:\t', overall_oof_stack_scores[n] )
NUM_WORDS = len(tokenizer.word_index)+ 1
NUM_WORDS<string_transform>
@njit
def post_process(pred, low, high):
pred_copy = pred.copy()
num_idx = 0
for i in range(pred_copy.shape[0]):
flag = np.zeros(pred_copy.shape[1])
 array = pred_copy[i].copy()
for j in range(pred_copy.shape[1]):
if(pred_copy[i, j] <= low)or(pred_copy[i, j] >= high):
flag[j] = 1
array[j] = round(array[j])
 if flag.all() and pred_copy[i].any() :
pred_copy[i] = array
num_idx += 1
return pred_copy, num_idx
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 print("hello word" )<string_transform>
ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss[cols] = 0.0877289129911273 * submission_elu_stack[cols].values + \
0.026620985745766827 * submission_resnet_stack[cols].values + \
0.04467573105638814 * submission_mlp_stack[cols].values + \
0.3382592931952609 * submission_rtn[cols].values + \
0.5027150770114568 * submission_tabnet[cols].values
ss.head(10 )
X_train = pad_sequences(X_train, maxlen=MAX_LEN)
 X_test = pad_sequences(X_test, maxlen=MAX_LEN)
 print("hello word" )<drop_column>
if POST_PROCESS:
low = 0.012
high = 0.98
ss[cols], num_idx = post_process(ss[cols].values, low, high)
 print(num_idx )
del(df_train)
 del(df_test )<load_pretrained>
del data, data2, train2, test2, data_transformed, train_transformed, test_transformed
if CALCULATE_OOF:
del tr_targets, overall_oof_score, overall_oof_scores, oof_preds, oof_preds_all, oof_targets, oof_targets_all, oof_score
del oof_elu_stack, oof_resnet_stack, oof_mlp_stack, oof_elu, oof_resnet, oof_mlp, oof_rtn
rubbish = gc.collect()
embedding_matrix_1 = np.load("glove.npy")
 embedding_matrix_3 = np.load("paragram.npy")
 print("Done..." )<compute_test_metric>
if FINETUNE:
pseudo_targets = ss.loc[test['cp_type'] == 0, cols].values
pseudo_train = test.loc[test['cp_type'] == 0, test.columns].values
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )<drop_column>
if FINETUNE:
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
res = np.zeros(targets_tr.shape)
 for nums, seed in enumerate(range(cfg.num_ensembling)) :
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 x_tr, y_tr = np.concatenate([cat_tr[train_idx], numerical_tr[train_idx]], axis=1), targets_tr[train_idx]
x_tr, y_tr = np.concatenate([x_tr, X_test[test['cp_type'] == 0, :]]), np.concatenate([y_tr, pseudo_targets])
 X_train, y_train = torch.as_tensor(x_tr), torch.as_tensor(y_tr)
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx]], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 model.fit(X_train=X_train, y_train=y_train, X_valid=X_val, y_valid=y_val, max_epochs=200, patience=5, batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=sbcewlogits, pretrain=True, optimizer_params=dict(lr=1e-4, weight_decay=1e-5))
model.load_best_model()
save_name = f"PL_tabnet_raw_step1_fold{j}_{seed}"
model.save_model(save_name)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 if CALCULATE_OOF_PL:
preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 res[val_idx] += preds / cfg.num_ensembling
oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 if CALCULATE_OOF_PL:
oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )
del(embedding_matrix_1)
 del(embedding_matrix_3 )<set_options>
if FINETUNE:
if CALCULATE_OOF_PL:
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 print(f'TabNet Overall OOF CV Score:', overall_oof_score)
 oof_tabnet2 = res_all
gc.collect()
print("done" )<choose_model_class>
if FINETUNE:
submission_tabnet2 = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 submission_tabnet2[cols] = preds_test.mean(1 ).mean(0)
 submission_tabnet2.loc[test['cp_type'] == 1, cols] = 0
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
K.set_value(self.model.optimizer.lr, self.clr() )<choose_model_class>
def train_model(X_train, Y_train_2, Y_nonscored, features, model_name, model_name_2, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, pseudo_labeling = True, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name_2:
model = create_rtn(len(features), 206, **model_params)
 elif 'RESNET' in model_name_2:
model = create_resnet(len(features), **model_params)
 elif 'ELU' in model_name_2:
model = create_mlp_elu(len(features), **model_params)
 elif 'MLP' in model_name_2:
model = create_mlp(len(features), 206, **model_params)
 for nums, seed in enumerate(range(num_seeds)) :
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
x_tr, x_val = X_train.values[tr][:, features], X_train.values[te][:, features]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if pseudo_labeling:
x_tr = np.concatenate([x_tr, pseudo_train[:, features]])
 y_tr = np.concatenate([y_tr, pseudo_targets])
 if X_test is not None:
x_tt = X_test.values[:, features]
if 'ELU' in model_name_2:
ckp_path = save_path + f'{model_name}_{seed}_{n}.hdf5'
else:
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 rlr = ReduceLROnPlateau(monitor = 'val_mean_loss', factor = 0.1, patience = 3,
verbose = verbose, min_delta = 1e-4, mode = 'min')
 ckp = ModelCheckpoint(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5', monitor = 'val_mean_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 es = EarlyStopping(monitor = 'val_mean_loss', min_delta = 1e-4, patience = 5, mode = 'min',
baseline = None, restore_best_weights = True, verbose = verbose)
 history = model.fit(x_tr, y_tr, validation_data =(x_val, y_val), epochs = 1000,
batch_size = 128, callbacks = [rlr, ckp, es], verbose = verbose)
 hist = pd.DataFrame(history.history)
 model.load_weights(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5')
 if X_test is not None:
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF_PL:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = hist['val_mean_loss'].min()
mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
print(f'[{str(datetime.timedelta(seconds = time() - start_time_fold)) [0:7]}] {model_name} Seed {seed}, Fold {n}:', fold_score)
 if CALCULATE_OOF_PL:
print(f'[{str(datetime.timedelta(seconds = time() - start_time_seed)) [0:7]}] {model_name} Seed {seed} Mean Score:', mean_score)
 if X_test is not None:
sub.loc[X_test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF_PL:
oof.loc[X_train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub
class AdamW(Optimizer):

 def __init__(self, lr=0.001, beta_1=0.9, beta_2=0.999, weight_decay=1e-4,
epsilon=1e-8, decay=0., **kwargs):
super(AdamW, self ).__init__(**kwargs)
 with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
 self.lr = K.variable(lr, name='lr')
 self.init_lr = lr
self.beta_1 = K.variable(beta_1, name='beta_1')
 self.beta_2 = K.variable(beta_2, name='beta_2')
 self.decay = K.variable(decay, name='decay')
 self.wd = K.variable(weight_decay, name='weight_decay')
 self.epsilon = epsilon
self.initial_decay = decay
@interfaces.legacy_get_updates_support
def get_updates(self, loss, params):
grads = self.get_gradients(loss, params)
 self.updates = [K.update_add(self.iterations, 1)]
wd = self.wd
lr = self.lr
if self.initial_decay > 0:
lr *=(1./(1.+ self.decay * K.cast(self.iterations,
K.dtype(self.decay))))
eta_t = lr / self.init_lr
t = K.cast(self.iterations, K.floatx())+ 1
lr_t = lr *(K.sqrt(1.- K.pow(self.beta_2, t)) / 
(1.- K.pow(self.beta_1, t)))
 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
self.weights = [self.iterations] + ms + vs
for p, g, m, v in zip(params, grads, ms, vs):
m_t =(self.beta_1 * m)+(1.- self.beta_1)* g
v_t =(self.beta_2 * v)+(1.- self.beta_2)* K.square(g)
 p_t = p - lr_t * m_t /(K.sqrt(v_t)+ self.epsilon)- eta_t * wd * p
self.updates.append(K.update(m, m_t))
self.updates.append(K.update(v, v_t))
new_p = p_t
if getattr(p, 'constraint', None)is not None:
new_p = p.constraint(new_p)
 self.updates.append(K.update(p, new_p))
return self.updates
def get_config(self):
config = {'lr': float(K.get_value(self.lr)) ,
'beta_1': float(K.get_value(self.beta_1)) ,
'beta_2': float(K.get_value(self.beta_2)) ,
'decay': float(K.get_value(self.decay)) ,
'weight_decay': float(K.get_value(self.wd)) ,
'epsilon': self.epsilon}
base_config = super(AdamW, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<define_search_model>
model_names = ['ELU_0', 'ELU_1', 'ELU_2', 'RTN']
model_params = [{'hidden_units': [512, 512, 4096],
'dropout_rates': 0.45,
'lr': 1e-4,
},
{'hidden_units': [512, 1024],
'dropout_rates': 0.463,
'lr': 1e-4,
},
{'hidden_units': [1024, 1024],
'dropout_rates': 0.5,
'lr': 1e-4,
},
{'rethink_iter': 3,
'num_layers': 1,
'hidden_units': [128, 512],
'dropout_rates': [0.3, 0.4],
'lr': 1e-4,
},]
glorotInit = initializers.glorot_uniform(seed=RANDOM_STATE)
 orthoInit = initializers.Orthogonal(seed=RANDOM_STATE )<statistical_test>
if FINETUNE:
Pseudo_Labeling = True
VERBOSE = 0
sample_sub_path = '.. /input/lish-moa/sample_submission.csv'
overall_oof_scores = []
oof_elu2 = []
submission_elu2 = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 if 'ELU' in model_names[m]:
model_idx = model_names[m].split('_')[1]
save_path = '.. /input/multilabel-v2/'
model_name = f'Model_{model_idx}'
elif 'RTN' in model_names[m]:
save_path = '.. /input/multilabel-rtn-single/'
model_name = f'Model0'
oof_score, res, ss = train_model(train, train_targets, train_targets_nonscored, top_feats, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test, sample_sub_path, Pseudo_Labeling, VERBOSE)
 if 'ELU' in model_names[m]:
oof_elu2.append(res)
 submission_elu2.append(ss)
 elif 'RTN' in model_names[m]:
oof_rtn2 = res
submission_rtn2 = ss
if CALCULATE_OOF_PL:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_scores.append(overall_oof_score)
 print(f'{model_name} Overall OOF CV Score:', overall_oof_score )
def squash(x, axis=-1):
s_squared_norm = K.sum(K.square(x), axis, keepdims=True)
 scale = K.sqrt(s_squared_norm + K.epsilon())
 return x / scale
class Capsule(Layer):
def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Capsule, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = squash
else:
self.activation = Activation(activation)
 def build(self, input_shape):
super(Capsule, self ).build(input_shape)
 input_dim_capsule = input_shape[-1]
if self.share_weights:
self.W = self.add_weight(name='capsule_kernel',
shape=(1, input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer=glorotInit,
trainable=True)
 else:
input_num_capsule = input_shape[-2]
self.W = self.add_weight(name='capsule_kernel',
shape=(input_num_capsule,
input_dim_capsule,
self.num_capsule * self.dim_capsule),
initializer=glorotInit,
trainable=True)
 def call(self, u_vecs):
if self.share_weights:
u_hat_vecs = K.conv1d(u_vecs, self.W)
 else:
u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])
 batch_size = K.shape(u_vecs)[0]
input_num_capsule = K.shape(u_vecs)[1]
u_hat_vecs = K.reshape(u_hat_vecs,(batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = K.permute_dimensions(u_hat_vecs,(0, 2, 1, 3))
b = K.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = K.permute_dimensions(b,(0, 2, 1))
c = K.softmax(b)
 c = K.permute_dimensions(c,(0, 2, 1))
b = K.permute_dimensions(b,(0, 2, 1))
outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))
if i < self.routings - 1:
b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])
 return outputs
def compute_output_shape(self, input_shape):
return(None, self.num_capsule, self.dim_capsule)
<choose_model_class>
if FINETUNE:
if CALCULATE_OOF_PL:
print(f'{model_names} OOF:\t', overall_oof_scores )
def capsule_model() :
K.clear_session()
inp_features = Input(shape=(6,))
inp = Input(shape=(MAX_LEN,))
emb = Embedding(NUM_WORDS, EMBEDDING_DIM, weights=[embedding_matrix], trainable=False )(inp)
 emb_drop = SpatialDropout1D(rate=0.2 )(emb)
 rnn = Bidirectional(CuDNNLSTM(75, return_sequences=True,
kernel_initializer=glorotInit, recurrent_initializer=orthoInit))(emb_drop)
 caps = Capsule(num_capsule=4, dim_capsule=10, routings=8, share_weights=True )(rnn)
 flat = Flatten()(caps)
 dense_1 = Dense(100, activation="elu", kernel_initializer=glorotInit )(flat)
 concat = concatenate([dense_1, inp_features])
 out_drop = Dropout(0.12 )(concat)
 out_bn = BatchNormalization()(out_drop)
 out = Dense(1, activation="sigmoid" )(out_bn)
 model = Model(inputs=[inp_features, inp], outputs=out)
 model.compile(loss='binary_crossentropy', optimizer=AdamW(lr=LR, weight_decay=WD), metrics=["acc", f1])
 return model<compute_test_metric>
def train_ensemble(X_train, Y_train_2, model_name, model_name_2, save_path, num_seeds, num_splits,
model_params, X_test = None, sample_sub_path = None, verbose = 0):
start_time_all = time()
oof = Y_train_2.copy()
oof.loc[:, Y_train_2.columns] = 0
overall_score = []
if X_test is not None:
sub = pd.read_csv(sample_sub_path)
 sub.loc[:, Y_train_2.columns] = 0
else:
sub = None
if 'RTN' in model_name_2:
model = create_rtn(X_train.shape[1], 206, **model_params)
 elif 'RESNET' in model_name_2:
model = create_resnet(X_train.shape[1], **model_params)
 elif 'ELU' in model_name_2:
model = create_mlp_elu(X_train.shape[1], **model_params)
 elif 'MLP' in model_name_2:
model = create_mlp(X_train.shape[1], 206, **model_params)
 for seed in range(num_seeds):
start_time_seed = time()
tf.random.set_seed(seed)
 np.random.seed(seed)
 random.seed(seed)
 mean_score = 0
skf = MultilabelStratifiedKFold(n_splits = num_splits, random_state = seed, shuffle = True)
 for n,(tr, te)in enumerate(skf.split(Y_train_2, Y_train_2)) :
print(f'Model:{model_name}, Seed:{seed}, Fold:{n}', end = '\r')
 start_time_fold = time()
x_tr, x_val = X_train[tr], X_train[te]
y_tr, y_val = Y_train_2.values[tr], Y_train_2.values[te]
if 'ELU' in model_name_2:
ckp_path = save_path + f'{model_name}_{seed}_{n}.hdf5'
else:
ckp_path = save_path + f'{model_name}_Seed_{seed}_Fold_{n}.hdf5'
model.load_weights(ckp_path)
 rlr = ReduceLROnPlateau(monitor = 'val_mean_loss', factor = 0.1, patience = 3,
verbose = verbose, min_delta = 1e-4, mode = 'min')
 ckp = ModelCheckpoint(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5', monitor = 'val_mean_loss', verbose = 0,
save_best_only = True, save_weights_only = True, mode = 'min')
 es = EarlyStopping(monitor = 'val_mean_loss', min_delta = 1e-4, patience = 5, mode = 'min',
baseline = None, restore_best_weights = True, verbose = verbose)
 history = model.fit(x_tr, y_tr, validation_data =(x_val, y_val), epochs = 1000,
batch_size = 128, callbacks = [rlr, ckp, es], verbose = verbose)
 hist = pd.DataFrame(history.history)
 model.load_weights(f'{model_name}_Seed_{seed}_Fold_{n}.hdf5')
 if X_test is not None:
x_tt = X_test
test_predict = model.predict(x_tt, batch_size = 1024)
 sub.loc[:, Y_train_2.columns] += test_predict /(num_splits * num_seeds)
 if CALCULATE_OOF_PL:
val_predict = model.predict(x_val, batch_size = 1024)
 fold_score = hist['val_mean_loss'].min()
mean_score += fold_score / num_splits
oof.loc[te, Y_train_2.columns] += val_predict / num_seeds
print(f'[{str(datetime.timedelta(seconds = time() - start_time_fold)) [0:7]}] {model_name} Seed {seed}, Fold {n}:', fold_score)
 if CALCULATE_OOF_PL:
print(f'[{str(datetime.timedelta(seconds = time() - start_time_seed)) [0:7]}] {model_name} Seed {seed} Mean Score:', mean_score)
 if X_test is not None:
sub.loc[test['cp_type'] == 1, Y_train_2.columns] = 0
if CALCULATE_OOF_PL:
oof.loc[train['cp_type'] == 1, Y_train_2.columns] = 0
overall_score = log_loss_metric(Y_train_2.values, oof[Y_train_2.columns].values)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time_all)) [0:7]}] {model_name} OOF Score:', overall_score)
 return overall_score, oof, sub
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result<normalization>
if FINETUNE:
if CALCULATE_OOF_PL:
train_new_elu2 = np.concatenate([oof0[cols].values for oof0 in oof_elu2], axis = 1)
 else:
train_new_elu2 = np.zeros(( train.shape[0], len(cols)* 3))
test_new_elu2 = np.concatenate([sub[cols].values for sub in submission_elu2], axis = 1 )
def step_decay(epoch):
initial_lrate = LR
drop = 0.6
epochs_drop = 5.0
lrate = initial_lrate * math.pow(drop, math.floor(( 1+epoch)/epochs_drop))
return lrate<choose_model_class>
model_names = ['ELU_stack']
model_params = [{'hidden_units': [1024, 1024],
'dropout_rates': 0.336,
'lr': 1e-4,
},]
clr = CyclicLR(base_lr=0.001, max_lr=0.003,
step_size=300., mode='exp_range',
gamma=0.99994 )<find_best_model_class>
if FINETUNE:
overall_oof_stack_scores = []
for m in range(len(model_params)) :
print(model_names[m], model_params[m])
 if 'ELU' in model_names[m]:
save_path = '.. /input/multilabel-v2/'
model_name = f'EModel'
oof_score, res, ss = train_ensemble(train_new_elu2, train_targets, model_name, model_names[m], save_path,
N_STARTS, N_SPLITS, model_params[m], test_new_elu2, sample_sub_path, VERBOSE)
 oof_elu_stack2 = res.copy()
submission_elu_stack2 = ss.copy()
if CALCULATE_OOF_PL:
tr_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv' ).drop('sig_id', axis = 1)
 res_all = np.zeros(tr_targets[cols].shape)
 res_all[train_features['cp_type'] == 0] = res[cols].values
overall_oof_score = log_loss_metric(tr_targets[cols].values, res_all)
 overall_oof_stack_scores.append(overall_oof_score)
 print(f'{model_names[m]} Overall OOF CV Score:', overall_oof_score )
def train_pred(model, train_X,train_features_X, train_y, val_X, val_features_X, val_y, test_data, callbacks=None):
st = time.time()
model.fit([train_features_X, np.array(train_X)], train_y, batch_size=BATCH_SIZE, epochs=NUM_EPOCH,
validation_data=([val_features_X, np.array(val_X)], val_y), callbacks = callbacks, verbose=2)
 pred_val_y = model.predict([val_features_X, val_X], batch_size=1024, verbose=0)
 best_thresh = threshold_search(val_y, pred_val_y)
 print("\tVal F1 Score: {:.4f}\tThresh: {:.2f}".format(best_thresh["f1"], best_thresh["threshold"]))
best_score = best_thresh["f1"]
pred_test_y = model.predict(test_data, batch_size=1024, verbose=0)
 print("Training time was: " + str(time.time() - st))
print('=' * 60)
 return pred_val_y, pred_test_y, best_score<split>
if CALCULATE_OOF_PL:
for n, name in enumerate(model_names):
print(f'{name} OOF:\t', overall_oof_stack_scores[n] )
train_meta = np.zeros(y_train.shape)
 test_meta = np.zeros(len(X_test))
splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE ).split(X_train, y_train))
for idx,(train_idx, valid_idx)in enumerate(splits):
X_train_tmp = X_train[train_idx]
X_features_train_tmp = X_features_train[train_idx]
y_train_tmp = y_train[train_idx]
X_val_tmp = X_train[valid_idx]
X_features_val_tmp = X_features_train[valid_idx]
y_val_tmp = y_train[valid_idx]
model = capsule_model()
pred_val_y, pred_test_y, best_score = train_pred(model, X_train_tmp, X_features_train_tmp, y_train_tmp, X_val_tmp,
X_features_val_tmp, y_val_tmp, [X_features_test ,X_test],
callbacks=[clr])
 train_meta[valid_idx] = pred_val_y.reshape(-1)
 test_meta += pred_test_y.reshape(-1)/ len(splits)
 del(X_train_tmp)
 del(y_train_tmp)
 del(X_val_tmp)
 del(y_val_tmp)
 gc.collect()<save_to_csv>
if FINETUNE:
ss = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss[cols] = 0.0877289129911273 * submission_elu_stack2[cols].values + \
0.026620985745766827 * submission_resnet_stack[cols].values + \
0.04467573105638814 * submission_mlp_stack[cols].values + \
0.3382592931952609 * submission_rtn2[cols].values + \
0.5027150770114568 * submission_tabnet2[cols].values
<feature_engineering><EOS> 
ss.to_csv('submission.csv', index = False)
 ss.head(10 )