Kaggle competition: Mechanisms of Action (MoA) Prediction
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
 history = model.fit(X_train, y_train,
epochs=3,
batch_size=256,
validation_data=(X_test, y_test))<predict_on_test>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
plt.style.use("fivethirtyeight")
 c_ = Fore.CYAN
m_ = Fore.MAGENTA
r_ = Fore.RED
b_ = Fore.BLUE
y_ = Fore.YELLOW
g_ = Fore.GREEN
pretrained_model_pred = model.predict([X_test], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(y_test,(pretrained_model_pred>thresh ).astype(int))))<save_to_csv>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
test_predictions = model.predict([output_test_data], batch_size=256, verbose=1)
 test_predictions =(test_predictions>0.44 ).astype(int)
 output = pd.DataFrame({"qid":test["qid"].values})
 output['prediction'] = test_predictions
output.to_csv("submission.csv", index=False )<import_modules>
def preprocessor_TabNet() :
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 train_drug = pd.read_csv(data_path + "train_drug.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv")
 if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp = train[cols_numeric].loc[:, mask]
train = pd.concat([train[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
test = pd.concat([test[["sig_id", "cp_type", "cp_time", "cp_dose"]], test.loc[:,cols_numeric]], axis = 1)
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
if scale == "rankgauss":
print(b_, "Rank Gauss")
 for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train[col].values)
 vec_len_test = len(test[col].values)
 raw_vec = train[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
else:
pass
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train = pd.concat([train, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test[GENES])
 pca_cells_test = pca_cells.transform(test[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test = pd.concat([test, pca_genes_test, pca_cells_test], axis = 1)
 else:
pass
if encoding == "dummy":
print(b_, "One-Hot")
 train = pd.get_dummies(train, columns = ["cp_time", "cp_dose"])
 test = pd.get_dummies(test, columns = ["cp_time", "cp_dose"])
 else:
pass
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
train["g_" + stats] = getattr(train[GENES], stats )(axis = 1)
 train["c_" + stats] = getattr(train[CELLS], stats )(axis = 1)
 train["gc_" + stats] = getattr(train[GENES + CELLS], stats )(axis = 1)
 test["g_" + stats] = getattr(test[GENES], stats )(axis = 1)
 test["c_" + stats] = getattr(test[CELLS], stats )(axis = 1)
 test["gc_" + stats] = getattr(test[GENES + CELLS], stats )(axis = 1)
 gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577',
'g-153','g-389','g-60','g-370','g-248','g-167',
'g-203','g-177','g-301','g-332','g-517','g-6',
'g-744','g-224','g-162','g-3','g-736','g-486',
'g-283','g-22','g-359','g-361','g-440','g-335',
'g-106','g-307','g-745','g-146','g-416','g-298',
'g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
for df in [train, test]:
df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in gsquarecols:
if feature in GENES:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in CELLS:
df[f'{feature}_squared'] = df[feature] ** 2
train = train.merge(targets, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 target_cols = [x for x in targets.columns if x != 'sig_id']
train_df = train
train_df.reset_index(drop=True, inplace=True)
 test.reset_index(drop = True, inplace = True)
 features_to_drop = ["sig_id", "cp_type"]
test.drop(features_to_drop, axis = 1, inplace = True)
 feature_cols = [c for c in train_df.columns if c not in targets.columns]
feature_cols = [c for c in feature_cols if c not in [ 'sig_id', 'drug_id', 'cp_type']]
return train, targets, test, submission, target_cols, feature_cols
tqdm.pandas()
warnings.simplefilter(action='ignore', category=FutureWarning )<set_options>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = [x for x in range(42,49)]
NFOLDS = 10
DRUG_THRESH = 18
train_df = make_cv_folds(train_df, SEEDS, NFOLDS, DRUG_THRESH )
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
tf.set_random_seed(seed )<define_variables>
X_test = test_TabNet.values
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(
 mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10 
)
 class LogitsLogLoss(Metric):

 def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):

 logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 1e-15)+ y_true * np.log(logits + 1e-15)
 return np.mean(-aux)
 test_cv_preds = []
for s in SEEDS:
tabnet_params['seed'] = s
for fold_id in range(NFOLDS):
print(b_,"FOLDS: ", r_, fold_id + 1, y_, 'seed:', tabnet_params['seed'])
 print(g_, '*' * 60, c_)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/TabNet_seed_'\
+str(s)+'_fold_'+str(fold_id+1)
 saved_file = '/kaggle/working/model'
shutil.make_archive(saved_file,'zip',saved_path_name)
 print(f'Successfully saved model at {saved_file}.zip')
 device =('cuda' if torch.cuda.is_available() else 'cpu')
 with zipfile.ZipFile('model.zip')as z:
with z.open('model_params.json')as f:
loaded_params = json.load(f)
 with z.open('network.pt')as f:
try:
saved_state_dict = torch.load(f, map_location=device)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.bytesIO(f.read()),map_location = device)
 loaded_model = TabNetRegressor(**loaded_params)
 loaded_model._set_network()
loaded_model.network.load_state_dict(saved_state_dict)
 loaded_model.network.eval()
preds_test = loaded_model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 test_preds_all = np.stack(test_cv_preds)
 saved_path_name = '.. /input/tabnetfe7seeds424810folds/'
oof_TabNet_all = np.load(saved_path_name + 'oof_TabNet_all.npy' )
seed = 5583
seed_everything(seed)
 begin = time.time()
sns.set_style('whitegrid' )<define_variables>
data_path = ".. /input/lish-moa/"
all_feat = [col for col in submission_TabNet.columns if col not in ["sig_id"]]
test_TabNet = pd.read_csv(data_path + "test_features.csv")
 sig_id = test_TabNet[test_TabNet["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
submission_TabNet = pd.merge(test_TabNet[["sig_id"]], tmp, on = "sig_id", how = "left")
 submission_TabNet.fillna(0, inplace = True)
 submission_TabNet.to_csv("submission_TabNet.csv", index = None)
 submission_TabNet.head()
GLOVE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
PARAGRAM = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
FASTTEXT = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'<load_from_csv>
del targets_TabNet, test_TabNet, tmp, train_df
gc.collect()
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<define_variables>
seed = 42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(seed)
 def preprocessor_nn_transfer() :
variance_threshould = 0.8
ncompo_genes = 600
ncompo_cells = 50
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 train_features = train_features[train_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test_features = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 train_features = train_features.drop('cp_type', axis=1)
 test_features = test_features.drop('cp_type', axis=1)
 GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pca_genes = PCA(n_components = ncompo_genes, random_state = seed)
 pca_genes_train = pca_genes.fit_transform(train_features[GENES])
 pca_cells = PCA(n_components = ncompo_cells, random_state = seed)
 pca_cells_train = pca_cells.fit_transform(train_features[CELLS])
 pca_genes_train = pd.DataFrame(pca_genes_train, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_train = pd.DataFrame(pca_cells_train, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 train_features = pd.concat([train_features, pca_genes_train, pca_cells_train], axis = 1)
 pca_genes_test = pca_genes.transform(test_features[GENES])
 pca_cells_test = pca_cells.transform(test_features[CELLS])
 pca_genes_test = pd.DataFrame(pca_genes_test, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells_test = pd.DataFrame(pca_cells_test, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 test_features = pd.concat([test_features, pca_genes_test, pca_cells_test], axis = 1)
 PCA')
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
mask =(train_features[cols_numeric].var() >= variance_threshould ).values
tmp = train_features[cols_numeric].loc[:, mask]
train_features = pd.concat([train_features[["sig_id", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(train_features.columns)if feat not in ["sig_id", "cp_time", "cp_dose"]]
test_features = pd.concat([test_features[["sig_id", "cp_time", "cp_dose"]], test_features.loc[:,cols_numeric]], axis = 1)
 train_features = pd.get_dummies(train_features, columns = ['cp_time', 'cp_dose'])
 test_features = pd.get_dummies(test_features, columns = ['cp_time', 'cp_dose'])
 Feature selection')
 train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 test = test_features
Join')
 target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 num_targets: {}'.format(num_targets))
')
 return train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols
augment_insincere_count = 0<string_transform>
train, test, num_targets, num_aux_targets, num_all_targets, target_cols, aux_target_cols, all_target_cols = preprocessor_nn_transfer()
class MarkovifyTextAugmentor(object):
def __init__(self, docs, gen_max_len, filter_min_len=10):

 self.mk_text_model = mk.Text(docs)
 self.gen_max_len = gen_max_len
self.filter_min_len = filter_min_len
def genenerate(self, count):
texts = []
for _ in range(count):
text = self.mk_text_model.make_short_sentence(self.gen_max_len)
 if text and len(text.split(' ')) > self.filter_min_len:
texts.append(text)
 return texts<string_transform>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
insincere_df = train_df[train_df['target'] == 1]
insincere_texts = insincere_df['question_text'].apply(" ".join)
 median_length = int(insincere_texts.str.len().median())
 mk_augmentor = MarkovifyTextAugmentor(insincere_texts, median_length, filter_min_len=6 )<define_variables>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
feature_cols = [c for c in train.columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 model = Model(num_features, num_all_targets)
 def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 display(train.head())
 def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = train
test_ = test
kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/nn-transfer-learning-oof/SCORED_ONLY_SEED_{seed_id}_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 print(f"Pretrained_Model_Loaded_SCORED_ONLYSEED: {seed_id}, FOLD: {fold_id} ")
 return 0.02, predictions
%%time
def gen_text_process(count):
return mk_augmentor.genenerate(count)
 num_cores = psutil.cpu_count()
pool = multiprocessing.Pool(processes=num_cores)
 apply_results = []
process_counts = augment_insincere_count // num_cores
for _ in range(num_cores):
apply_results.append(pool.apply_async(gen_text_process,(process_counts,)))
 pool.close()
pool.join()
generated_texts = []
for res in apply_results:
generated_texts.extend(res.get())
 gen_df = pd.DataFrame({'question_text': generated_texts, 'target': [1] * len(generated_texts)} )<define_variables>
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
str(timedelta(seconds=time_diff))
data_dir = '.. /input/lish-moa/'
train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_scored.head()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1])
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 sub_nn_transfer = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub_nn_transfer.to_csv('submission_nn_transfer.csv', index=False )
print('Generated {} insincere questions'.format(len(generated_texts)) )<concatenate>
saved_path_name = '.. /input/nn-transfer-learning-oof/'
oof_nn_transfer = np.load(saved_path_name + 'oof_nn_transfer_all.npy')
 oof_nn_transfer.shape
if augment_insincere_count > 0:
train_df = pd.concat(( train_df, gen_df), axis=0, sort=True)
 train_df = shuffle(train_df, random_state=seed )<define_variables>
del sample_submission, test, train, train_targets_scored, valid_results
gc.collect()
toxic_words = ['4r5e', '5h1t', '5hit', 'a55', 'anal', 'anus', 'ar5e', 'arrse', 'arse', 'ass', 'ass-fucker', 'asses', 'assfucker', 'assfukka', 'asshole', 'assholes', 'asswhole', 'a_s_s', 'b!tch', 'b00bs', 'b17ch', 'b1tch', 'ballbag', 'balls', 'ballsack', 'bastard', 'beastial', 'beastiality', 'bellend', 'bestial', 'bestiality', 'bi+ch', 'biatch', 'bitch', 'bitcher', 'bitchers', 'bitches', 'bitchin', 'bitching', 'bloody', 'blow', 'job', 'blowjob', 'blowjobs', 'boiolas', 'bollock', 'bollok', 'boner', 'boob', 'boobs', 'booobs', 'boooobs', 'booooobs', 'booooooobs', 'breasts', 'buceta', 'bugger', 'bum', 'bunny', 'fucker', 'butt', 'butthole', 'buttmuch', 'buttplug', 'c0ck', 'c0cksucker', 'carpet', 'muncher', 'cawk', 'chink', 'cipa', 'cl1t', 'clit', 'clitoris', 'clits', 'cnut', 'cock', 'cock-sucker', 'cockface', 'cockhead', 'cockmunch', 'cockmuncher', 'cocks', 'cocksuck', 'cocksucked', 'cocksucker', 'cocksucking', 'cocksucks', 'cocksuka', 'cocksukka', 'cok', 'cokmuncher', 'coksucka', 'coon', 'cox', 'crap', 'cum', 'cummer', 'cumming', 'cums', 'cumshot', 'cunilingus', 'cunillingus', 'cunnilingus', 'cunt', 'cuntlick', 'cuntlicker', 'cuntlicking', 'cunts', 'cyalis', 'cyberfuc', 'cyberfuck', 'cyberfucked', 'cyberfucker', 'cyberfuckers', 'cyberfucking', 'd1ck', 'damn', 'dick', 'dickhead', 'dildo', 'dildos', 'dink', 'dinks', 'dirsa', 'dlck', 'dog-fucker', 'doggin', 'dogging', 'donkeyribber', 'doosh', 'duche', 'dyke', 'ejaculate', 'ejaculated', 'ejaculates', 'ejaculating', 'ejaculatings', 'ejaculation', 'ejakulate', 'f', 'u', 'c', 'k', 'f', 'u', 'c', 'k', 'e', 'r', 'f4nny', 'fag', 'fagging', 'faggitt', 'faggot', 'faggs', 'fagot', 'fagots', 'fags', 'fanny', 'fannyflaps', 'fannyfucker', 'fanyy', 'fatass', 'fcuk', 'fcuker', 'fcuking', 'feck', 'fecker', 'felching', 'fellate', 'fellatio', 'fingerfuck', 'fingerfucked', 'fingerfucker', 'fingerfuckers', 'fingerfucking', 'fingerfucks', 'fistfuck', 'fistfucked', 'fistfucker', 'fistfuckers', 'fistfucking', 'fistfuckings', 'fistfucks', 'flange', 'fook', 'fooker', 'fuck', 'fucka', 'fucked', 'fucker', 'fuckers', 'fuckhead', 'fuckheads', 'fuckin', 'fucking', 'fuckings', 'fuckingshitmotherfucker', 'fuckme', 'fucks', 'fuckwhit', 'fuckwit', 'fudge', 'packer', 'fudgepacker', 'fuk', 'fuker', 'fukker', 'fukkin', 'fuks', 'fukwhit', 'fukwit', 'fux', 'fux0r', 'f_u_c_k', 'gangbang', 'gangbanged', 'gangbangs', 'gaylord', 'gaysex', 'goatse', 'God', 'god-dam', 'god-damned', 'goddamn', 'goddamned', 'hardcoresex', 'hell', 'heshe', 'hoar', 'hoare', 'hoer', 'homo', 'hore', 'horniest', 'horny', 'hotsex', 'jack-off', 'jackoff', 'jap', 'jerk-off', 'jism', 'jiz', 'jizm', 'jizz', 'kawk', 'knob', 'knobead', 'knobed', 'knobend', 'knobhead', 'knobjocky', 'knobjokey', 'kock', 'kondum', 'kondums', 'kum', 'kummer', 'kumming', 'kums', 'kunilingus', 'l3i+ch', 'l3itch', 'labia', 'lmfao', 'lust', 'lusting', 'm0f0', 'm0fo', 'm45terbate', 'ma5terb8', 'ma5terbate', 'masochist', 'master-bate', 'masterb8', 'masterbat*', 'masterbat3', 'masterbate', 'masterbation', 'masterbations', 'masturbate', 'mo-fo', 'mof0', 'mofo', 'mothafuck', 'mothafucka', 'mothafuckas', 'mothafuckaz', 'mothafucked', 'mothafucker', 'mothafuckers', 'mothafuckin', 'mothafucking', 'mothafuckings', 'mothafucks', 'mother', 'fucker', 'motherfuck', 'motherfucked', 'motherfucker', 'motherfuckers', 'motherfuckin', 'motherfucking', 'motherfuckings', 'motherfuckka', 'motherfucks', 'muff', 'mutha', 'muthafecker', 'muthafuckker', 'muther', 'mutherfucker', 'n1gga', 'n1gger', 'nazi', 'nigg3r', 'nigg4h', 'nigga', 'niggah', 'niggas', 'niggaz', 'nigger', 'niggers', 'nob', 'nob', 'jokey', 'nobhead', 'nobjocky', 'nobjokey', 'numbnuts', 'nutsack', 'orgasim', 'orgasims', 'orgasm', 'orgasms', 'p0rn', 'pawn', 'pecker', 'penis', 'penisfucker', 'phonesex', 'phuck', 'phuk', 'phuked', 'phuking', 'phukked', 'phukking', 'phuks', 'phuq', 'pigfucker', 'pimpis', 'piss', 'pissed', 'pisser', 'pissers', 'pisses', 'pissflaps', 'pissin', 'pissing', 'pissoff', 'poop', 'porn', 'porno', 'pornography', 'pornos', 'prick', 'pricks', 'pron', 'pube', 'pusse', 'pussi', 'pussies', 'pussy', 'pussys', 'rectum', 'retard', 'rimjaw', 'rimming', 's', 'hit', 's.o.b.', 'sadist', 'schlong', 'screwing', 'scroat', 'scrote', 'scrotum', 'semen', 'sex', 'sh!+', 'sh!t', 'sh1t', 'shag', 'shagger', 'shaggin', 'shagging', 'shemale', 'shi+', 'shit', 'shitdick', 'shite', 'shited', 'shitey', 'shitfuck', 'shitfull', 'shithead', 'shiting', 'shitings', 'shits', 'shitted', 'shitter', 'shitters', 'shitting', 'shittings', 'shitty', 'skank', 'slut', 'sluts', 'smegma', 'smut', 'snatch', 'son-of-a-bitch', 'spac', 'spunk', 's_h_i_t', 't1tt1e5', 't1tties', 'teets', 'teez', 'testical', 'testicle', 'tit', 'titfuck', 'tits', 'titt', 'tittie5', 'tittiefucker', 'titties', 'tittyfuck', 'tittywank', 'titwank', 'tosser', 'turd', 'tw4t', 'twat', 'twathead', 'twatty', 'twunt', 'twunter', 'v14gra', 'v1gra', 'vagina', 'viagra', 'vulva', 'w00se', 'wang', 'wank', 'wanker', 'wanky', 'whoar', 'whore', 'willies', 'willy', 'xrated', 'xxx']<string_transform>
def readin_data_resnet() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets_drug = train_targets_scored.merge(train_drug, on='sig_id')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
targets_drug = targets_drug.iloc[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/ttestpcarfelogisticregression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return train_features,labels_train,test_features,targets_drug,predictors,cs,gs
def toxic_words_ratio(text, toxic_words=toxic_words):
count = 0
text = word_tokenize(text)
 for word in text:
count += int(word.lower() in toxic_words)
 return count / len(text )<feature_engineering>
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
def resnet_preprocessor(train,test,cs,gs):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test
def build_vocab(texts):
vocab = {}
for sentence in texts:
for word in sentence.split(' '):
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab<define_variables>
def resnet_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.25),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model
word_count = build_vocab(list(train_df['question_text'])+ list(test_df['question_text']))<count_values>
def resnet_make_cv_folds(targets_drug,target_cols, SEEDS, NFOLDS, DRUG_THRESH):
vc = targets_drug.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in SEEDS:
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = targets_drug.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = targets_drug.loc[targets_drug.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 targets_drug[kfold_col] = targets_drug.drug_id.map(dct1)
 targets_drug.loc[targets_drug[kfold_col].isna() , kfold_col] = targets_drug.loc[targets_drug[kfold_col].isna() , 'sig_id'].map(dct2)
 targets_drug[kfold_col] = targets_drug[kfold_col].astype('int8')
 return targets_drug
def freq_count(text):
text = text.split(" ")
 all_count = 0
for word in text:
all_count += word_count[word]
return len(text)/ all_count<feature_engineering>
def predict_resnet(train_features,test_features,labels_train, targets_drug1,SEEDS, NFOLDS,cs,gs):
n_labels = labels_train.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
n_seeds = len(SEEDS)
 n_folds = NFOLDS
y_pred = np.zeros(( n_test,n_labels))
oof = np.zeros(( n_train,n_labels))
train_all,data_test = resnet_preprocessor(train_features.values,
test_features.drop('cp_type',axis=1 ).values,cs,gs)
 for seed in SEEDS:
kfold_col = 'kfold_{}'.format(seed)
 kf = targets_drug1[kfold_col]
for fold in range(NFOLDS):
train = targets_drug1[kfold_col] != fold
test = targets_drug1[kfold_col] == fold
X_train, X_test = train_all[train],train_all[test]
X_train_2 = train_features[train][predictors].values
X_test_2 = train_features[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = keras.models.load_model('/kaggle/input/resnet-001843/TwoHeads_seed_'+str(seed)+'_fold_'+str(fold), custom_objects={'logloss':logloss})
 y_val = model.predict([X_test,X_test_2])
 oof[test] += y_val/ n_seeds
y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 return oof,y_pred
def make_features(df):
df['total_length'] = df['question_text'].apply(len)
 df['capitals'] = df['question_text'].apply(lambda comment: sum(1 for c in comment if c.isupper()))
df['caps_vs_length'] = df.apply(lambda row: float(row['capitals'])/float(row['total_length']), axis=1)
 df['num_words'] = df.question_text.str.count('\S+')
 df['num_unique_words'] = df['question_text'].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique'] = df['num_unique_words'] / df['num_words']
df['toxic_ratio'] = df['question_text'].apply(toxic_words_ratio)
 df['word_freq'] = df['question_text'].apply(freq_count )<define_variables>
train_features,labels_train,test_features,targets_drug,predictors,cs,gs = readin_data_resnet()
p_min = 0.0005
p_max = 0.9995
DRUG_THRESH = 18
n_seeds = 7
np.random.seed(42)
 SEEDS = np.random.randint(0,100,size=n_seeds)
 NFOLDS = 10
target_cols = targets_drug.drop(['sig_id','drug_id'], axis=1 ).columns.values.tolist()
targets_drug_copy = targets_drug.copy()
targets_drug1 = resnet_make_cv_folds(targets_drug_copy,target_cols, SEEDS, NFOLDS, DRUG_THRESH)
 oof_rs, test_pred_rs = predict_resnet(train_features,test_features,labels_train, targets_drug1,SEEDS, NFOLDS,cs,gs )
features = ['caps_vs_length', 'words_vs_unique', 'toxic_ratio', 'total_length', 'word_freq']
nb_features = len(features)
 print(f'Generated {nb_features} features :', ', '.join(features))<feature_engineering>
sub_rs = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_rs.iloc[:,1:] = np.clip(test_pred_rs,p_min,p_max)
 sub_rs.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_rs.to_csv('submission_rs.csv', index=False )
%%time
make_features(train_df)
 make_features(test_df )<normalization>
%who DataFrame
features_train = train_df[features].fillna(0)
 features_test = test_df[features].fillna(0)
 ss = StandardScaler()
ss.fit(np.vstack(( features_train, features_test)))
 features_train = ss.transform(features_train)
 features_test = ss.transform(features_test )<define_variables>
del targets_drug, targets_drug1, targets_drug_copy, train_features
gc.collect()
useful_punct = ['_', '☹', '＞', '½', '△', '¿', '¼', '∆', '≥', '⇒', '¬', '∨', '＾', 'µ', '?', '!', '.', ',', '"', '
useless_punct = ['च', '不', 'ঢ়', '平', 'ᠠ', '錯', '判', '∙', '言', 'ς', 'ل', '្', 'ジ', 'あ', '得', '水', 'ь', '◦', '创', '康', '華', 'ḵ', '☺', '支', '就', '„', '」', '어', '谈', '陈', '团', '腻', '权', '年', '业', 'マ', 'य', 'ا', '売', '甲', '拼', '˂', 'ὤ', '贯', '亚', 'ि', '放', 'ʻ', 'ទ', 'ʖ', '點', '્', '発', '青', '能', '木', 'д', '微', '藤', '̃', '僕', '妒', '͜', 'ន', 'ध', '이', '希', '特', 'ड', '¢', '滢', 'ส', '나', '女', 'క', '没', '什', 'з', '天', '南', 'ʿ', 'ค', 'も', '凰', '步', '籍', '西', 'ำ', '−', 'л', 'ڤ', 'ៃ', '號', 'ص', 'स', '®', 'ʋ', '批', 'រ', '치', '谢', '生', '道', '═', '下', '俄', 'ɖ', '觀', 'வ', '—', 'ی', '您', '♥', '一', 'や', '⊆', 'ʌ', '語', 'ี', '兴', '惶', '瀛', '狐', '⁴', 'प', '臣', 'ద', '―', 'ì', 'ऌ', 'ీ', '自', '信', '健', '受', 'ɨ', '시', 'י', 'ছ', '嬛', '湾', '吃', 'ち', 'ड़', '反', '红', '有', '配', 'ে', 'ឯ', '宮', 'つ', 'μ', '記', '口', '℅ι', 'ो', '狸', '奇', 'о', 'ट', '聖', '蘭', '読', 'ū', '標', '要', 'ត', '识', 'で', '汤', 'ま', 'ʀ', '局', 'リ', '्', 'ไ', '呢', '工', 'ल', '沒', 'τ', 'ិ', 'ö', 'せ', '你', 'ん', 'ュ', '枚', '部', '大', '罗', 'হ', 'て', '表', '报', '攻', 'ĺ', 'ฉ', '∩', '宝', '对', '字', '文', '这', '∑', '髪', 'り', '่', '능', '罢', '내', '阻', '为', '菲', 'ي', 'न', 'ί', 'ɦ', '開', '†', '茹', '做', '東', 'ত', 'に', 'ت', '晓', '키', '悲', 'સ', '好', '›', '上', '存', '없', '하', '知', 'ធ', '斯', ' ', '授', 'ł', '傳', '兰', '封', 'ோ', 'و', 'х', 'だ', '人', '太', '品', '毒', 'ᡳ', '血', '席', '剔', 'п', '蛋', '王', '那', '梦', 'ី', '彩', '甄', 'и', '柏', 'ਨ', '和', '坊', '⌚', '广', '依', '∫', 'į', '故', 'ś', 'ऊ', '几', '日', 'ک', '音', '×', '”', '▾', 'ʊ', 'ज', 'ด', 'ठ', 'उ', 'る', '清', 'ग', 'ط', 'δ', 'ʏ', '官', '∛', '়', '้', '男', '骂', '复', '∂', 'ー', '过', 'য', '以', '短', '翻', 'র', '教', '儀', 'ɛ', '‹', 'へ', '¾', '合', '学', 'ٌ', '학', '挑', 'ष', '比', '体', 'م', 'س', 'អ', 'ת', '訓', '∀', '迎', 'វ', 'ɔ', '٨', '▒', '化', 'చ', '‛', 'প', 'º', 'น', '업', '说', 'ご', '¸', '₹', '儿', '︠', '게', '骨', 'ท', 'ऋ', 'ホ', '茶', '는', 'જ', 'ุ', '羡', '節', 'ਮ', 'উ', '番', 'ড়', '讲', 'ㅜ', '등', '伟', 'จ', '我', 'ล', 'す', 'い', 'ញ', '看', 'ċ', '∧', 'भ', 'ઘ', 'ั', 'ម', '街', 'ય', '还', '鰹', 'ខ', 'ు', '訊', 'म', 'ю', '復', '杨', 'ق', 'त', '金', '味', 'ব', '风', '意', '몇', '佬', '爾', '精', '¶', 'ం', '乱', 'χ', '교', 'ה', '始', 'ᠰ', '了', '个', '克', '্', 'ห', '已', 'ʃ', 'わ', '新', '译', '︡', '本', 'ง', 'б', 'け', 'ి', '明', '¯', '過', 'ك', 'ῥ', 'ف', 'ß', '서', '进', 'ដ', '样', '乐', '寧', '€', 'ณ', 'ル', '乡', '子', 'ﬁ', 'ج', '慕', '–', 'ᡵ', 'Ø', '͡', '제', 'Ω', 'ប', '絕', '눈', 'फ', 'ম', 'గ', '他', 'α', 'ξ', '§', 'ஜ', '黎', 'ね', '복', 'π', 'ú', '鸡', '话', '会', 'ক', '八', '之', '북', 'ن', '¦', '가', 'ו', '恋', '地', 'ῆ', '許', '产', 'ॡ', 'ش', '़', '野', 'ή', 'ɒ', '啧', 'យ', '᠌', 'ᠨ', 'ب', '皎', '老', '公', '☆', 'व', 'ি', 'ល', 'ر', 'គ', '행', 'ង', 'ο', '让', 'ំ', 'λ', 'خ', 'ἰ', '家', 'ট', 'ब', '理', '是', 'め', 'र', '√', '기', 'ν', '玉', '한', '入', 'ד', '别', 'د', 'ะ', '电', 'ા', '♫', 'ع', 'ં', '堵', '嫉', '伊', 'う', '千', '관', '篇', 'क', '非', '荣', '粵', '瑜', '英', '를', '美', '条', '`', '宋', '←', '수', '後', '•', '³', 'ी', '고', '肉', '℃', 'し', '漢', '싱', 'ϵ', '送', 'ه', '落', 'న', 'ក', 'க', 'ℇ', 'た', 'ះ', '中', '射', '♪', '符', 'ឃ', '谷', '分', '酱', 'び', 'থ', 'ة', 'г', 'σ', 'と', '楚', '胡', '饭', 'み', '禮', '主', '直', '÷', '夢', 'ɾ', 'চ', '⃗', '統', '高', '顺', '据', 'ら', '頭', 'よ', '最', 'ా', 'ੁ', '亲', 'ស', '花', '≡', '眼', '病', '…', 'の', '發', 'ா', '汝', '★', '氏', 'ร', '景', 'ᡠ', '读', '件', '仲', 'শ', 'お', 'っ', 'پ', 'ᡤ', 'ч', '♭', '悠', 'ं', '六', '也', 'ռ', 'য়', '恐', 'ह', '可', '啊', '莫', '书', '总', 'ষ', 'ք', '̂', '간', 'な', '此', '愛', 'ర', 'ใ', '陳', 'Ἀ', 'ण', '望', 'द', '请', '油', '露', '니', 'ş', '宗', 'ʍ', '鳳', 'अ', '邋', '的', 'ព', '火', 'ा', 'ก', '約', 'ட', '章', '長', '商', '台', '勢', 'さ', '국', 'Î', '簡', 'ई', '∈', 'ṭ', '經', '族', 'ु', '孫', '身', '坑', 'স', '么', 'ε', '失', '殺', 'ž', 'ર', 'が', '手', 'ា', '心', 'ਾ', '로', '朝', '们', '黒', '欢', '早', '️', 'া', 'आ', 'ɸ', '常', '快', '民', 'ﷺ', 'ូ', '遢', 'η', '国', '无', '江', 'ॠ', '「', 'ন', '™', 'ើ', 'ζ', '紫', 'ె', 'я', '“', '♨', '國', 'े', 'อ', '∞']
letter_mapping = {'\u200b':' ', 'ũ': "u", 'ẽ': 'e', 'é': "e", 'á': "a", 'ķ': 'k', 'ï': 'i', 'Ź': 'Z', 'Ż': 'Z', 'Š': 'S', 'Π': ' pi ', 'Ö': 'O', 'É': 'E', 'Ñ': 'N', 'Ž': 'Z', 'ệ': 'e', '²': '2', 'Å': 'A', 'Ā': 'A', 'ế': 'e', 'ễ': 'e', 'ộ': 'o', '⧼': '<', '⧽': '>', 'Ü': 'U', 'Δ': 'delta', 'ợ': 'o', 'İ': 'I', 'Я': 'R', 'О': 'O', 'Č': 'C', 'П': 'pi', 'В': 'B', 'Φ': 'phi', 'ỵ': 'y', 'օ': 'o', 'Ľ': 'L', 'ả': 'a', 'Γ': 'theta', 'Ó': 'O', 'Í': 'I', 'ấ': 'a', 'ụ': 'u', 'Ō': 'O', 'Ο': 'O', 'Σ': 'sigma', 'Â': 'A', 'Ã': 'A', 'ᗯ': 'w', 'ᕼ': "h", "ᗩ": "a", "ᖇ": "r", "ᗯ": "w", "O": "o", "ᗰ": "m", "ᑎ": "n", "ᐯ": "v", "н": "h", "м": "m", "o": "o", "т": "t", "в": "b", "υ": "u", "ι": "i","н": "h", "č": "c", "š": "s", "ḥ": "h", "ā": "a", "ī": "i", "à": "a", "ý": "y", "ò": "o", "è": "e", "ù": "u", "â": "a", "ğ": "g", "ó": "o", "ê": "e", "ạ": "a", "ü": "u", "ä": "a", "í": "i", "ō": "o", "ñ": "n", "ç": "c", "ã": "a", "ć": "c", "ô": "o", "с": "c", "ě": "e", "æ": "ae", "î": "i", "ő": "o", "å": "a", "Ä": "A", }<define_variables>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
mispell_dict = {"trimp": "trump", "wanket": "wanker",'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'qoura': 'quora', 'sallary': 'salary', 'whta': 'what', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'etherium': 'ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization',}<categorify>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
def remove_ctl(no_ctl,train,test,targets):
if no_ctl:
print('not_ctl')
 train = train[train['cp_type']!='ctl_vehicle']
test = test[test['cp_type']!='ctl_vehicle']
targets = targets.iloc[train.index]
train.reset_index(drop=True, inplace=True)
 test.reset_index(drop=True, inplace=True)
 targets.reset_index(drop=True, inplace=True)
 return train,test,targets
def remove_small_variance(variance_threshould,train,test):
cols_numeric = [feat for feat in list(train.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train[cols_numeric].var() >= variance_threshould ).values
tmp_train = train[cols_numeric].loc[:, mask]
tmp_test = test[cols_numeric].loc[:, mask]
train = pd.concat([train[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_train], axis=1)
 test = pd.concat([test[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp_test], axis=1)
 return train,test
def pca_process(ncompo_genes,ncompo_cells,train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
pca_genes = PCA(n_components=ncompo_genes, random_state=base_seed ).fit(train[GENES])
 pca_cells = PCA(n_components=ncompo_cells, random_state=base_seed ).fit(train[CELLS])
 train_pca_gene = pca_genes.transform(train[GENES])
 test_pca_gene = pca_genes.transform(test[GENES])
 train_pca_cell = pca_cells.transform(train[CELLS])
 test_pca_cell = pca_cells.transform(test[CELLS])
 train_pca_gene = pd.DataFrame(train_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 test_pca_gene = pd.DataFrame(test_pca_gene, columns=[f'pca_g-{i}' for i in range(ncompo_genes)])
 train_pca_cell = pd.DataFrame(train_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 test_pca_cell = pd.DataFrame(test_pca_cell, columns=[f'pca_c-{i}' for i in range(ncompo_cells)])
 train_df = pd.concat([train, train_pca_gene, train_pca_cell], axis=1)
 test_df = pd.concat([test, test_pca_gene, test_pca_cell], axis=1)
 return train_df,test_df
def scale_data(scale, train_df,test_df):
if scale == 'minmax':
print('minmax')
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis=0)
 elif scale == 'rankgauss':
print('rankgauss')
 cols_numeric = [feat for feat in list(train_df.columns)if feat not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose'] and "pca_" not in feat]
scaler = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 train_df[cols_numeric] = scaler.fit_transform(train_df[cols_numeric])
 test_df[cols_numeric] = scaler.transform(test_df[cols_numeric])
 else:
pass
return train_df,test_df
def add_features(train,test):
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
for stats in tqdm.tqdm(['sum', 'mean', 'std', 'kurt', 'skew']):
train['g_'+stats] = getattr(train[GENES], stats )(axis=1)
 train['c_'+stats] = getattr(train[CELLS], stats )(axis=1)
 train['gc_'+stats] = getattr(train[GENES+CELLS], stats )(axis=1)
 test['g_'+stats] = getattr(test[GENES], stats )(axis=1)
 test['c_'+stats] = getattr(test[CELLS], stats )(axis=1)
 test['gc_'+stats] = getattr(test[GENES+CELLS], stats )(axis=1)
 return train,test
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
def data_processing(train,test, targets):
train,test,targets = remove_ctl(no_ctl,train,test,targets)
 print("remove control: ", train.shape,test.shape)
 train,test = remove_small_variance(variance_threshould,train,test)
 print("remove small_var: ", train.shape,test.shape)
 train_df,test_df = pca_process(ncompo_genes,ncompo_cells,train,test)
 print("do PCA: ", train_df.shape,test_df.shape)
 train_df,test_df = scale_data(scale, train_df,test_df)
 print("rankGauss: ", train_df.shape,test_df.shape)
 train_df,test_df = add_features(train_df,test_df)
 print("add stats feature:", train_df.shape,test_df.shape)
 return train_df,test_df,targets
def add_maj(dic):
d = dic.copy()
for w in dic.keys() :
try:
d[w[0].upper() + w[1:]] = dic[w][0].upper() + dic[w][1:]
except:
d[w[0].upper() ] = dic[w][0].upper()
return d<categorify>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
substitution_dic = {}
substitution_dic.update(mispell_dict)
 substitution_dic = add_maj(substitution_dic)
 substitution_dic.update(letter_mapping )<categorify>
no_ctl = True
scale = 'rankgauss'
variance_threshould = 0.8
decompo = 'PCA'
ncompo_genes = 50
ncompo_cells = 8
encoding = 'dummy'
base_seed = 42
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 print(DEVICE)
 EPOCHS = 20
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
hidden_size=1024
SEEDS = 7
SEED = [0, 1, 2, 3, 4, 5, 6]
DRUG_THRESH = 18
def _get_substitution(sub_dic):
sub_re = re.compile('(%s)' % '|'.join(sub_dic.keys()))
return sub_dic, sub_re
substitutions, substitutions_re = _get_substitution(substitution_dic)
 def replace_substitution(text):
def replace(match):
return substitutions[match.group(0)]
return substitutions_re.sub(replace, text )<drop_column>
seed_everything(base_seed)
 train, test, targets = data_processing(train_features,test_features, train_targets_scored)
 target_cols = train_targets_scored.drop('sig_id', axis=1 ).columns.values.tolist()
print(len(target_cols))
train_comb = train.merge(targets, on='sig_id')
 train_comb.drop('cp_type', axis=1, inplace=True)
 print("combine train and targets to prepare for kfolds ", train_comb.shape)
 feature_cols = [c for c in process_data(train_comb ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id', 'drug_id']]
print(len(feature_cols))
gen_cols = [c for c in feature_cols if 'g-' == c[:2]]
cell_cols = [c for c in feature_cols if 'c-' == c[:2]]
pca_gen_cols = [c for c in feature_cols if 'pca_g' == c[:5]]
pca_cell_cols = [c for c in feature_cols if 'pca_c' == c[:5]]
ohe_cols = [c for c in feature_cols if c not in gen_cols+cell_cols+pca_gen_cols+pca_cell_cols]
print(len(gen_cols), len(cell_cols), len(pca_gen_cols), len(pca_cell_cols),len(ohe_cols))
num_features=len(feature_cols)
 num_targets=len(target_cols)
 folds = train_comb.copy()
folds = folds.merge(train_drug, on='sig_id')
 folds = make_cv_folds(folds, SEEDS, NFOLDS, DRUG_THRESH )
def clean_apostrophes(x):
apostrophes = ["’", "‘", "´", "`"]
for s in apostrophes:
x = re.sub(s, "'", x)
 return x<drop_column>
class Model(nn.Module):
def __init__(self, num_gen_features, num_cell_features,num_targets, hidden_size):
super(Model, self ).__init__()
self.gen_batch_norm1 = nn.BatchNorm1d(num_gen_features)
 self.gen_dropout1 = nn.Dropout(0.2)
 self.gen_dense1 = nn.utils.weight_norm(nn.Linear(num_gen_features, hidden_size))
self.gen_batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.gen_dropout2 = nn.Dropout(0.3)
 self.gen_dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.cell_batch_norm1 = nn.BatchNorm1d(num_cell_features)
 self.cell_dropout1 = nn.Dropout(0.2)
 self.cell_dense1 = nn.utils.weight_norm(nn.Linear(num_cell_features, int(hidden_size/2)))
 self.cell_batch_norm2 = nn.BatchNorm1d(int(hidden_size/2))
self.cell_dropout2 = nn.Dropout(0.3)
 self.cell_dense2 = nn.utils.weight_norm(nn.Linear(int(hidden_size/2), int(hidden_size/2)))
 self.batch_norm3 = nn.BatchNorm1d(hidden_size + int(hidden_size/2))
self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size + int(hidden_size/2), num_targets))
def forward(self, gen_x, cell_x):
gen_x = self.gen_batch_norm1(gen_x)
 gen_x = self.gen_dropout1(gen_x)
 gen_x = F.relu(self.gen_dense1(gen_x))
gen_x = self.gen_batch_norm2(gen_x)
 gen_x = self.gen_dropout2(gen_x)
 gen_x = F.relu(self.gen_dense2(gen_x))
cell_x = self.cell_batch_norm1(cell_x)
 cell_x = self.cell_dropout1(cell_x)
 cell_x = F.relu(self.cell_dense1(cell_x))
cell_x = self.cell_batch_norm2(cell_x)
 cell_x = self.cell_dropout2(cell_x)
 cell_x = F.relu(self.cell_dense2(cell_x))
x = torch.cat(( gen_x,cell_x),dim=1)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
def remove_s(x):
if len(x)> 2:
return re.sub("('$ |'$|'s |'s)", ' ', x)
 else:
return x<string_transform>
class MoADataset:
def __init__(self, gen_features, cell_features, targets):
self.gen_features = gen_features
self.cell_features = cell_features
self.targets = targets
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, gen_features, cell_features):
self.gen_features = gen_features
self.cell_features = cell_features
def __len__(self):
return(self.gen_features.shape[0])
 def __getitem__(self, idx):
dct = {
'gen_x' : torch.tensor(self.gen_features[idx, :], dtype=torch.float),
'cell_x' : torch.tensor(self.cell_features[idx, :], dtype=torch.float)
 }
return dct
spaces = ['\u200b', '\u200e', '\u202a', '\u202c', '\ufeff', '\uf0d8', '\u2061', '\x10', '\x7f', '\x9d', '\xad', '\xa0']
def clean_spaces(text):
for space in spaces:
text = text.replace(space, ' ')
 text = text.strip()
text = re.sub('\s+', ' ', text)
 return text<categorify>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
gen_x, cell_x, targets = data['gen_x'].to(device), data['cell_x'].to(device), data['y'].to(device)
 outputs = model(gen_x, cell_x)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
gen_x, cell_x = data['gen_x'].to(device), data['cell_x'].to(device)
 with torch.no_grad() :
outputs = model(gen_x, cell_x)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
def decontract(text):
text = re.sub(r"(W|w)on('|\’)t ", "will not ", text)
 text = re.sub(r"(C|c)an('|\’)t ", "can not ", text)
 text = re.sub(r"(Y|y )('|\’)all ", "you all ", text)
 text = re.sub(r"(Y|y)a('|\’)ll ", "you all ", text)
 text = re.sub(r"(I|i )('|\’)m ", "i am ", text)
 text = re.sub(r"(A|a)isn('|\’)t ", "is not ", text)
 text = re.sub(r"n('|\’)t ", " not ", text)
 text = re.sub(r"('|\’)re ", " are ", text)
 text = re.sub(r"('|\’)s ", " is ", text)
 text = re.sub(r"('|\’)d ", " would ", text)
 text = re.sub(r"('|\’)ll ", " will ", text)
 text = re.sub(r"('|\’)t ", " not ", text)
 text = re.sub(r"('|\’)ve ", " have ", text)
 return text<categorify>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def clean_numbers(text):
text = re.sub(r'(\d+ )([a-zA-Z])', '\g<1> \g<2>', text)
 text = re.sub(r'(\d+ )(th|st|nd|rd)', '\g<1>\g<2> ', text)
 text = re.sub(r'(\d+),(\d+)', '\g<1>\g<2>', text)
 return text<categorify>
def get_oof_predictions(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 loss_fn = nn.BCEWithLogitsLoss()
model = Model(
 num_gen_features=len(gen_cols + pca_gen_cols + ohe_cols),
num_cell_features=len(cell_cols + pca_cell_cols + ohe_cols),
num_targets=len(target_cols),
hidden_size=hidden_size)
 model.to(DEVICE)
 model.load_state_dict(torch.load(f".. /input/jp-multi-heads/FOLD{fold}_SEED{seed}.pth"))
gen_x_test, cell_x_test = test_[gen_cols + pca_gen_cols + ohe_cols].values, test_[cell_cols + pca_cell_cols + ohe_cols].values
testdataset = TestDataset(gen_x_test, cell_x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 print('predict on test...', end='')
 predictions = np.zeros(( len(test_), targets.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 print(' done.
 ')
 return predictions
def clean_special_chars(text, punct=useful_punct):
for p in punct:
text = text.replace(p, f' {p} ')
 return text<string_transform>
def run_k_fold(NFOLDS, seed):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
pred_ = get_oof_predictions(fold, seed)
 predictions += pred_ / NFOLDS
return predictions
predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
predictions_ = run_k_fold(NFOLDS, seed)
 predictions += predictions_ / len(SEED)
def clean_latex_tag(text):
corr_t = []
for t in text.split(" "):
t = t.strip()
if t != '':
corr_t.append(t)
 text = ' '.join(corr_t)
 text = re.sub('(\[ math \] ).+(\[ / math \])', 'mathematical formula', text)
 return text<feature_engineering>
%who DataFrame
def build_vocab(texts):
vocab = {}
for sentence in texts:
for word in sentence.split(' '):
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab<categorify>
del folds, sample_submission, targets, test, train, train_comb, train_drug, train_features, train_targets_nonscored, train_targets_scored
gc.collect()
print("Number of available cpu cores: {}".format(cpu_count()))
def process_in_parallel(function, list_):
with Pool(cpu_count())as p:
tmp = p.map(function, list_)
 return tmp<categorify>
def treat_texts(texts):
texts = process_in_parallel(clean_apostrophes, texts)
 texts = process_in_parallel(replace_substitution, texts)
 texts = process_in_parallel(decontract, texts)
 texts = process_in_parallel(remove_s, texts)
 texts = process_in_parallel(clean_numbers, texts)
 texts = process_in_parallel(clean_special_chars, texts)
 texts = process_in_parallel(clean_spaces, texts)
 texts = process_in_parallel(clean_latex_tag, texts)
 return texts<feature_engineering>
predictions_clip = np.clip(predictions, 0.0005, 0.9995)
 predictions_clip = pd.DataFrame(predictions_clip, columns=target_cols )
%%time
train_df["question_text"] = treat_texts(train_df["question_text"])
 test_df["question_text"] = treat_texts(test_df["question_text"] )<define_variables>
oof_multi_heads_all = np.load('.. /input/jp-multi-heads/multi_heads_oof.npy' )
vocab = build_vocab(list(train_df["question_text"])+ list(test_df["question_text"]))<define_variables>
test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sig_id = test[test['cp_type']!='ctl_vehicle'].sig_id.reset_index(drop=True)
 predictions_clip['sig_id'] = sig_id
sub_mh = pd.merge(test[['sig_id']], predictions_clip, on='sig_id', how='left')
 sub_mh.fillna(0, inplace=True)
 sub_mh.to_csv('submission_mh.csv', index=False )
len_voc = None
max_len = 70<string_transform>
def log_loss_numpy(y_pred):
y_true_ravel = np.asarray(y_true ).ravel()
y_pred = np.asarray(y_pred ).ravel()
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 loss = np.where(y_true_ravel == 1, - np.log(y_pred), - np.log(1 - y_pred))
return loss.mean()
def func_numpy_metric(weights):
oof_blend = np.tensordot(weights, oof, axes =(( 0),(0)))
 return log_loss_numpy(oof_blend)
 def grad_func(weights):
oof_clip = np.clip(oof, 1e-15, 1 - 1e-15)
 gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
@njit
def grad_func_jit(weights):
oof_clip = np.minimum(1 - 1e-15, np.maximum(oof, 1e-15))
gradients = np.zeros(oof.shape[0])
 for i in range(oof.shape[0]):
a, b, c = y_true, oof_clip[i], np.zeros(( oof.shape[1], oof.shape[2]))
for j in range(oof.shape[0]):
if j != i:
c += weights[j] * oof_clip[j]
gradients[i] = -np.mean(( -a*b+(b**2)*weights[i]+b*c)/(( b**2)*(weights[i]**2)+2*b*c*weights[i]-b*weights[i]+(c**2)-c))
return gradients
def make_input_data(X_train, X_test):
t = Tokenizer(num_words=len_voc, filters='', lower=False)
 t.fit_on_texts(( np.concatenate(( X_train, X_test), axis=0)))
 X_train = t.texts_to_sequences(X_train)
 X_test = t.texts_to_sequences(X_test)
 X_train = pad_sequences(X_train, maxlen=max_len, padding='post', truncating='post')
 X_test = pad_sequences(X_test, maxlen=max_len, padding='post', truncating='post')
 return X_train, X_test, t.word_index<prepare_x_and_y>
y_true = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv', index_col = 'sig_id' ).values
oof_dict = {'Model 1': '.. /input/nn-transfer-learning-oof/oof_nn_transfer_all.npy',
'Model 2': '.. /input/tabnetfe7seeds424810folds/oof_TabNet_all.npy',
'Resnet': '.. /input/prediction/resnet_oof.npy',
'Multi_heads': '.. /input/jp-multi-heads/multi_heads_oof.npy'
}
oof = np.zeros(( len(oof_dict), y_true.shape[0], y_true.shape[1]))
for i in range(oof.shape[0]):
oof[i] = np.load(list(oof_dict.values())[i] )
%%time
X_train, X_test, word_index = make_input_data(train_df['question_text'], test_df['question_text'] )<define_variables>
%%time
log_loss_scores = {}
for n, key in enumerate(oof_dict.keys()):
score_oof = log_loss_numpy(oof[n])
 log_loss_scores[key] = score_oof
print(f'{key} CV:\t', score_oof)
 print('-' * 50 )
len_voc = len(word_index)+ 1<prepare_x_and_y>
test_weights = np.array([1 / oof.shape[0]] * oof.shape[0] )
y_train = train_df['target'].values<split>
tol = 1e-10
init_guess = [1 / oof.shape[0]] * oof.shape[0]
bnds = [(0, 1)for _ in range(oof.shape[0])]
cons = {'type': 'eq',
'fun': lambda x: np.sum(x)- 1,
'jac': lambda x: [1] * len(x)}
print('Inital Blend OOF:', func_numpy_metric(init_guess))
start_time = time()
res_scipy = minimize(fun = func_numpy_metric,
x0 = init_guess,
method = 'SLSQP',
jac = grad_func_jit,
bounds = bnds,
constraints = cons,
tol = tol)
 print(f'[{str(datetime.timedelta(seconds = time() - start_time)) [2:7]}] Optimised Blend OOF:', res_scipy.fun)
 print('Optimised Weights:', res_scipy.x )
def make_embed_index(file, word_index, vocab=[], exceptions={}):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 if file == FASTTEXT:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100 and o.split(" ")[0] in word_index)
 if file == PARAGRAM:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding="utf8", errors='ignore')if len(o)>100 and o.split(" ")[0] in word_index)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file)if o.split(" ")[0] in word_index)
 for word in vocab:
try:
_ = embeddings_index[word]
except:
try:
embeddings_index[word] = embeddings_index[word.lower() ]
except:
try:
embeddings_index[word] = embeddings_index[word.upper() ]
except:
try:
embeddings_index[word] = embeddings_index[word[0].upper() + word[1:].lower() ]
except:
pass
return embeddings_index<categorify>
print('Check the sum of all weights:', np.sum(res_scipy.x))
if np.sum(res_scipy.x)- 1 <= tol:
print('Great! The sum of all weights equals to 1!')
 else:
print('Manual adjustion is needed to modify the weights.' )
def make_embed_mat(embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
embedding_matrix = np.random.normal(emb_mean, emb_std,(len(word_index)+1, embed_size))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix<prepare_output>
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub.iloc[:,1:] = np.clip(res_scipy.x[0]*sub_nn_transfer.iloc[:, 1:].values + \
res_scipy.x[1]*submission_TabNet.iloc[:,1:].values + \
res_scipy.x[2]*sub_rs.iloc[:,1:].values + \
res_scipy.x[3]*sub_mh.iloc[:,1:].values, 0.00005,0.99995)
 sub.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub.to_csv('submission.csv', index=False )
<prepare_output><EOS> 
!rm submission_TabNet.csv