Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<choose_model_class>
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
model = Model(num_features, num_all_targets)
 model<define_search_model>
sample_submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv" )
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()<train_model>
def text_cleaning_hyperlink(text):
return re.sub(r"http\S+","",text )
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<categorify>
def text_cleaning_punctuation(text):
translator = str.maketrans('', '', string.punctuation)
 return text.translate(translator )
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
def text_cleaning_stopwords(text):
stop_words = set(stopwords.words('english'))
word_token = word_tokenize(text)
 filtered_sentence = [w for w in word_token if not w in stop_words]
return ' '.join(filtered_sentence)
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions 
<compute_test_metric>
def text_cleaning_lowercase(text):
return text.lower()
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )<save_to_csv>
def text_extract(text_lst):
txt = []
for i,x in enumerate(text_lst):
for j,p in enumerate(x):
txt.append(p)
 return txt
submission = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 submission.to_csv('submission.csv', index=False)
<categorify>
def remove_digits(txt):
no_digits = ''.join(i for i in txt if not i.isdigit())
 return no_digits
def auc_multi(y_true, y_pred, exclude_index=None):
M = y_true.shape[1]
results = torch.zeros(M)
 sample_weight = None
if exclude_index is not None:
sample_weight = torch.ones(y_true.shape[0] ).type_as(y_pred)
 sample_weight[exclude_index] = 0.0
for i in range(M):
try:
results[i] = FM.auroc(
 y_pred[:, i], y_true[:, i], sample_weight=sample_weight 
)
 except:
pass
return results.mean()
def get_activation(activation):
if activation == "relu":
return nn.ReLU()
elif activation == "leaky_relu":
return nn.LeakyReLU()
else:
raise ValueError
class InvaseModel(pl.LightningModule):
enable_print_log = True
@staticmethod
def add_model_specific_args(parent_parser):
parser = ArgumentParser(parents=[parent_parser], add_help=False)
 parser.add_argument("--learning_rate", type=float, default=0.0002)
 parser.add_argument("--max_learning_rate", type=float, default=None)
 parser.add_argument("--vehicle_weight", type=float, default=0.1)
 parser.add_argument("--num_layers", type=int, default=3)
 parser.add_argument("--actor_h_dim", type=int, default=100)
 parser.add_argument("--critic_h_dim", type=int, default=200)
 parser.add_argument("--weight_decay", type=float, default=1e-5)
 parser.add_argument("--lambda_", type=float, default=0.1)
 parser.add_argument("--minus", action="store_true")
 parser.add_argument("--scheduler_patience", type=int, default=10)
 parser.add_argument("--activation", type=str, default="relu")
 parser.add_argument("--dropout", type=float, default=0.2)
 return parser
def __init__(self,
learning_rate,
input_dim: int = 874,
output_dim: int = 206,
include_vehicle: bool = False,
vehicle_weight: float = 1.0,
minus: bool = False,
scheduler_patience: int = 10,
class_weight=None,
weight_decay: float = 1e-5,
actor_h_dim: int = 100,
critic_h_dim: int = 200,
num_layers: int = 3,
activation: str = "relu",
dropout: float = 0.2,
lambda_: float = 0.1,
**kwargs, 
):
super(InvaseModel, self ).__init__()
self.class_weight = class_weight
self.output_dim = output_dim
self.learning_rate = learning_rate
self.include_vehicle = include_vehicle
self.vehicle_weight = vehicle_weight
self.weight_decay = weight_decay
self.minus = minus
self.scheduler_patience = scheduler_patience
self.lambda_ = lambda_
self._last_train_loss = None
self.actor = self.build_layers(input_dim, actor_h_dim, input_dim,
num_layers - 2, activation, dropout)
 self.critic = self.build_layers(input_dim, critic_h_dim, output_dim,
num_layers - 2, activation, dropout)
 self._cache = None
if self.minus:
self.baseline = None
else:
self.baseline = self.build_layers(input_dim, critic_h_dim, output_dim,
num_layers - 2, activation, dropout)
 self.save_hyperparameters()
def build_layers(self, input_dim: int, hidden_dim: int, output_dim: int, n_layer: int,
activation: str,
dropout: float):
layers = [
nn.BatchNorm1d(input_dim),
nn.Dropout(dropout),
nn.utils.weight_norm(nn.Linear(input_dim, hidden_dim)) ,
get_activation(activation),
]
for i in range(n_layer):
layers += [
nn.BatchNorm1d(hidden_dim),
nn.Dropout(dropout),
nn.utils.weight_norm(nn.Linear(hidden_dim, hidden_dim)) ,
get_activation(activation),
]
layers += [
nn.BatchNorm1d(hidden_dim),
nn.Dropout(dropout),
nn.utils.weight_norm(nn.Linear(hidden_dim, output_dim))
]
return nn.Sequential(*layers)
 def actor_loss(self, selection, actor_out, baseline_loss, critic_loss, y):
if self.minus:
reward = - critic_loss
else:
reward = -(critic_loss - baseline_loss)
 y_pred = actor_out.sigmoid()
actor_loss = selection * torch.log(y_pred + 1e-8)+(1 - selection)* torch.log(1 - y_pred + 1e-8)
 actor_loss = reward * actor_loss.sum(1)- self.lambda_ * y_pred.mean(1)
 return(-actor_loss ).mean()
def _train_loss(self, out, x, y):
return F.binary_cross_entropy_with_logits(out, y)
 def training_step(self, batch, batch_idx, optimizer_idx):
x = batch["x"].float()
y = batch["y"].float()
selection, actor_out, baseline_out, critic_out = self(x)
 if optimizer_idx == 0:
critic_loss = self._train_loss(critic_out, x, y)
 self.log("train_critic_loss", critic_loss, logger=True, prog_bar=True)
 return critic_loss
elif self.minus:
if optimizer_idx == 1:
critic_loss = self._train_loss(critic_out, x, y)
 actor_loss = self.actor_loss(selection, actor_out,
None, critic_loss.detach() , y)
 self.log("train_actor_loss", actor_loss, logger=True, prog_bar=True)
 return actor_loss
else:
if optimizer_idx == 1:
baseline_loss = self._train_loss(baseline_out, x, y)
 self.log("train_baseline_loss", baseline_loss, logger=True, prog_bar=True)
 return baseline_loss
elif optimizer_idx == 2:
critic_loss = self._train_loss(critic_out, x, y)
 baseline_loss = self._train_loss(baseline_out, x, y)
 actor_loss = self.actor_loss(selection, actor_out,
baseline_loss.detach() , critic_loss.detach() , y)
 self.log("train_actor_loss", actor_loss, logger=True, prog_bar=True)
 return actor_loss
def validation_step(self, batch, batch_idx):
x = batch["x"].float()
y = batch["y"].float()
selection, actor_out, baseline_out, critic_out = self(x)
 loss = F.binary_cross_entropy_with_logits(critic_out, y)
 auc_score = auc_multi(y, critic_out.sigmoid())
 metrics = {"val_loss": loss, "val_auc": auc_score}
self.log_dict(metrics, logger=True, prog_bar=True)
 return loss
def validation_epoch_end(self, validation_step_outputs):
val_loss = np.mean([t.item() for t in validation_step_outputs])
 if self.enable_print_log:
print(
 f"epoch: {self.current_epoch} val_loss: {val_loss:.4f}" 
)
 def predict(self, x):
with torch.no_grad() :
_, _, _, out = self(x)
 pred = out.sigmoid().detach().cpu()
return pred
def forward(self, x):
actor_out = self.actor(x)
 prob = actor_out.detach().sigmoid()
prob = torch.where(torch.isnan(prob), torch.zeros_like(prob ).to(prob.device), prob)
 selection = torch.bernoulli(prob)
 critic_out = self.critic(x * selection)
 if self.minus:
baseline_out = None
else:
baseline_out = self.baseline(x)
 return selection, actor_out, baseline_out, critic_out
def configure_optimizers(self):
optimizers = []
lr_schedulers = []
optimizer0 = torch.optim.Adam(
 self.critic.parameters() , lr=self.learning_rate,
weight_decay=self.weight_decay 
)
 lr_scheduler0 = {
"scheduler": torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer0, mode="min", factor=0.1, patience=self.scheduler_patience,
eps=1e-4, verbose=False 
),
"monitor": "val_loss",
"interval": "epoch",
"frequency": 1,
}
optimizers.append(optimizer0)
 lr_schedulers.append(lr_scheduler0)
 if not self.minus:
optimizer1 = torch.optim.Adam(
 self.baseline.parameters() , lr=self.learning_rate,
weight_decay=self.weight_decay 
)
 lr_scheduler1 = {
"scheduler": torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer1, mode="min", factor=0.1, patience=self.scheduler_patience,
eps=1e-4, verbose=False 
),
"monitor": "val_loss",
"interval": "epoch",
"frequency": 1,
}
optimizers.append(optimizer1)
 lr_schedulers.append(lr_scheduler1)
 optimizer2 = torch.optim.Adam(
 self.actor.parameters() , lr=self.learning_rate,
weight_decay=self.weight_decay 
)
 lr_scheduler2 = {
"scheduler": torch.optim.lr_scheduler.ReduceLROnPlateau(
 optimizer2, mode="min", factor=0.1, patience=self.scheduler_patience,
eps=1e-4, verbose=False 
),
"monitor": "val_loss",
"interval": "epoch",
"frequency": 1,
}
optimizers.append(optimizer2)
 lr_schedulers.append(lr_scheduler2)
 return optimizers, lr_schedulers 
<categorify>
train.keyword = train.keyword.apply(lambda x: text_cleaning_stopwords(text_cleaning_punctuation(text_cleaning_hyperlink(remove_digits(x.lower())))) if type(x)== str else x )
def get_pca(all_features, col_prefix, n_comp):
cols = [col for col in all_features.columns if col.startswith(col_prefix)]
data2 = PCA(n_components=n_comp, random_state=42 ).fit_transform(all_features[cols])
 data2 = pd.DataFrame(data2, columns=[f"pca_{col_prefix}-{i}" for i in range(n_comp)])
 return data2
def handle_category(all_features, include_vehicle=False):
cols = list(all_features.columns)[1:]
mapping = {
"cp_type": {"trt_cp": 0, "ctl_vehicle": 1},
"cp_time": {48: 0, 72: 1, 24: 2},
"cp_dose": {"D1": 0, "D2": 1},
}
start_col = 0 if include_vehicle else 1
categorical_cols = cols[start_col:3]
numerical_cols = cols[3:]
all_features.loc[:, categorical_cols] = np.stack(
 [all_features[c].apply(lambda x: mapping[c][x] ).values for c in categorical_cols],
axis=1, 
)
 return all_features, categorical_cols, numerical_cols
def prepare_data(base_dir, add_pca=False, include_vehicle=False,
stat: bool = False,
return_dataframe: bool = False,
pca_g: int = 600,
pca_c: int = 81, 
):
base_train = pd.read_csv(str(Path(base_dir)/ "train_features.csv"))
train_targets_scored = pd.read_csv(str(Path(base_dir)/ "train_targets_scored.csv"))
test_features = pd.read_csv(str(Path(base_dir)/ "test_features.csv"))
train_features = base_train
if not include_vehicle:
train_rows = train_features.shape[0]
test_rows = test_features.shape[0]
train_features = train_features.loc[
base_train["cp_type"] == "trt_cp"
].reset_index(drop=True)
 train_targets_scored = train_targets_scored.loc[
base_train["cp_type"] == "trt_cp"
].reset_index(drop=True)
 test_features = test_features.loc[
test_features["cp_type"] == "trt_cp"
].reset_index(drop=True)
 print(f"exclude vehicle train rows {train_rows} -> {train_features.shape[0]}"
f" test rows {test_rows} -> {test_features.shape[0]}")
 all_features = pd.concat([train_features, test_features], ignore_index=True)
 all_features, categorical_cols, numerical_cols = handle_category(
 all_features,
include_vehicle=include_vehicle, 
)
 if add_pca:
scaler = QuantileTransformer(n_quantiles=100, random_state=0, output_distribution="normal")
 all_features.loc[:, numerical_cols] = scaler.fit_transform(all_features[numerical_cols])
 pca_start_col = -1
if add_pca:
old_cols = all_features.shape[1]
pca0 = get_pca(all_features, col_prefix="g-", n_comp=pca_g)
 pca1 = get_pca(all_features, col_prefix="c-", n_comp=pca_c)
 all_features = pd.concat([all_features, pca0, pca1], axis=1)
 new_cols = all_features.shape[1]
print(f"pca add {new_cols - old_cols} {old_cols} -> {new_cols}")
 pca_start_col = old_cols
if stat:
g_cols = [col for col in all_features.columns if col.startswith("g-")]
c_cols = [col for col in all_features.columns if col.startswith("c-")]
old_cols = all_features.shape[1]
for s in ["sum", "mean", "std", "kurt", "skew"]:
all_features[f"g_{s}"] = getattr(all_features[g_cols], s )(axis=1)
 all_features[f"c_{s}"] = getattr(all_features[c_cols], s )(axis=1)
 all_features[f"gc_{s}"] = getattr(all_features[g_cols + c_cols], s )(axis=1)
 new_cols = all_features.shape[1]
print(f"stat add {new_cols - old_cols} {old_cols} -> {new_cols}")
 assert all_features.shape[0] == train_features.shape[0] + test_features.shape[0]
train_features = all_features.iloc[:train_features.shape[0], :]
test_features = all_features.iloc[train_features.shape[0]:, :]
if return_dataframe:
return train_features, train_targets_scored, test_features
start_col = 1 if include_vehicle else 2
base_train_np = train_features.iloc[:, start_col:].values.astype(np.float)
 X_test = test_features.iloc[:, start_col:].values.astype(np.float)
 targets_tr = train_targets_scored.iloc[:, 1:].values.astype(
 np.float32 
)
 assert base_train_np.shape[0] == targets_tr.shape[0]
assert base_train_np.shape[1] == X_test.shape[1]
return base_train_np, targets_tr, X_test 
<train_model>
train.text = train.text.apply(lambda x: text_cleaning_stopwords(text_cleaning_punctuation(text_cleaning_hyperlink(remove_digits(x.lower())))))
 train.text = train.text.apply(lambda x: list(set(x.split(' '))))
base_train, targets_tr, test_X = prepare_data(".. /input/lish-moa/", add_pca=True, pca_g=375, pca_c=45)
 print(base_train.shape, targets_tr.shape, test_X.shape )<categorify>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return self.features.shape[0]
def __getitem__(self, idx):
dct = {
"x": torch.tensor(self.features[idx, :], dtype=torch.float),
"y": torch.tensor(self.targets[idx, :], dtype=torch.float),
}
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return self.features.shape[0]
def __getitem__(self, idx):
dct = {"x": torch.tensor(self.features[idx, :], dtype=torch.float)}
return dct 
<predict_on_test>
train['k_t'] = train.apply(lambda x : x['text'] + [x['keyword']] if type(x['keyword'])== str else x['text'], axis=1 )
def predict(test_features, model_path: Path, model=None):
test_dataset = TestDataset(test_features)
 dataloader = torch.utils.data.DataLoader(
 test_dataset, 128, shuffle=False, drop_last=False 
)
 if model is None:
model = InvaseModel.load_from_checkpoint(str(model_path))
preds = []
for data in dataloader:
pred = model.predict(data["x"].to(model.device)).cpu().numpy()
preds.append(pred)
 return np.concatenate(preds, axis=0)
 def train_(train_X, train_Y, valid_X, valid_Y, test_X, fold, seed, epochs=100):
train_dataset = MoADataset(train_X, train_Y)
 valid_dataset = MoADataset(valid_X, valid_Y)
 train_dataloader = torch.utils.data.DataLoader(train_dataset, 256,
shuffle=True, drop_last=False, num_workers=10,
pin_memory=True)
 valid_dataloader = torch.utils.data.DataLoader(valid_dataset, 256, shuffle=False, drop_last=False,
num_workers=10,
pin_memory=True)
 early_stop_callback = EarlyStopping(monitor="val_loss", patience=10)
 trainer_root = Path("/kaggle/log")
 if not trainer_root.exists() :
trainer_root.mkdir(parents=True)
 trainer = pl.Trainer(gpus=1, callbacks=[early_stop_callback],
progress_bar_refresh_rate=0,
default_root_dir=str(trainer_root),
max_epochs=epochs,
logger=False)
 model = InvaseModel(input_dim=train_X.shape[1], output_dim=train_Y.shape[1], learning_rate=1e-3,
actor_h_dim=1024, critic_h_dim=1500, num_layers=3, lambda_=0.01)
 trainer.fit(model, train_dataloader, valid_dataloader)
 print(f"val loss best: {trainer.checkpoint_callback.best_model_score}")
 save_path = Path("/kaggle/model")/ f"ckpt/{seed}"
if not save_path.exists() :
save_path.mkdir(parents=True)
 model_save_path = save_path / f"model_{fold}.ckpt"
shutil.copy(str(Path(trainer.checkpoint_callback.best_model_path)) ,
str(model_save_path))
pred = predict(test_X, model_save_path)
 return pred<load_from_csv>
ps_1 = PorterStemmer()
wnl_1 = WordNetLemmatizer()
text_reconstruct = []
for i,x in enumerate(train.k_t.values):
try:
a = wnl_1.lemmatize(ps_1.stem(x))
except AttributeError:
a = list(set([wnl_1.lemmatize(ps_1.stem(word)) for j,word in enumerate(x)]))
text_reconstruct.append(a )
def create_fold(base_dir, n_split, seed, include_vehicle):
target_df = pd.read_csv(str(Path(base_dir)/ "train_targets_scored.csv"))
columns = target_df.columns[1:]
drug_df = pd.read_csv(str(Path(base_dir)/ "train_drug.csv"))
if not include_vehicle:
train_df = pd.read_csv(str(Path(base_dir)/ "train_features.csv"))
target_df = target_df[train_df["cp_type"] == "trt_cp"].reset_index(drop=True)
 target_df = target_df.merge(drug_df, on='sig_id', how='left')
 vc = target_df.drug_id.value_counts()
vc1 = vc.loc[vc <= 18].index.sort_values()
vc2 = vc.loc[vc > 18].index.sort_values()
if n_split >= 2:
splits = MultilabelStratifiedKFold(n_splits=n_split, random_state=1 + seed, shuffle=True)
 else:
splits = MultilabelStratifiedKFold(n_splits=10, random_state=1 + seed, shuffle=True)
 drug_id2fold = {}
sig_id2fold = {}
tmp = target_df.groupby('drug_id')[columns].mean().loc[vc1]
for fold,(idx_t, idx_v)in enumerate(splits.split(tmp, tmp[columns])) :
drug_id2fold.update({k: fold for k in tmp.index[idx_v].values})
 if n_split >= 2:
splits = MultilabelStratifiedKFold(n_splits=n_split, random_state=1 + seed, shuffle=True)
 else:
splits = MultilabelStratifiedKFold(n_splits=10, random_state=1 + seed, shuffle=True)
 tmp = target_df.loc[target_df.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idx_t, idx_v)in enumerate(splits.split(tmp, tmp[columns])) :
sig_id2fold.update({k:fold for k in tmp.sig_id[idx_v].values})
 target_df['fold'] = target_df.drug_id.map(drug_id2fold)
 target_df.loc[target_df.fold.isna() , 'fold'] = \
target_df.loc[target_df.fold.isna() , 'sig_id'].map(sig_id2fold)
 target_df.fold = target_df.fold.astype('int8')
 folds = []
for fold in range(n_split):
train_index = target_df[target_df.fold != fold].index
valid_index = target_df[target_df.fold == fold].index
folds.append(( fold,(train_index, valid_index)))
 return folds 
<prepare_x_and_y>
train.k_t = text_reconstruct
train_word = train.k_t.apply(lambda x: ' '.join(x))
n_seeds = 2
n_splits = 7
epochs = 25
BASE_DIR = "/kaggle/input/lish-moa/"
pred_list = []
for seed in range(n_seeds):
splits = create_fold(BASE_DIR, n_splits, seed, include_vehicle=False)
 for fold,(index_train, index_valid)in splits:
train_X = base_train[index_train]
train_Y = targets_tr[index_train]
valid_X = base_train[index_valid]
valid_Y = targets_tr[index_valid]
pred = train_(train_X, train_Y, valid_X, valid_Y, test_X, fold, seed, epochs)
 pred_list.append(pred)
 all_pred = np.stack(pred_list)
 print(all_pred.shape )<load_from_csv>
test.keyword = test.keyword.apply(lambda x: text_cleaning_stopwords(text_cleaning_punctuation(text_cleaning_hyperlink(remove_digits(x.lower())))) if type(x)== str else x )
submission = pd.read_csv(str(Path(BASE_DIR)/ "sample_submission.csv"))
test_features = pd.read_csv(str(Path(BASE_DIR)/ "test_features.csv"))
prediction = all_pred.mean(0)
 print(prediction.shape)
 print(submission.shape)
 submission.head()<save_to_csv>
test.text = test.text.apply(lambda x: text_cleaning_stopwords(text_cleaning_punctuation(text_cleaning_hyperlink(remove_digits(x.lower())))))
 test.text = test.text.apply(lambda x: list(set(x.split(' '))))
submission.loc[test_features['cp_type']!='ctl_vehicle', submission.columns[1:]] = prediction
submission.loc[test_features['cp_type']=='ctl_vehicle', submission.columns[1:]] = 0
submission.to_csv("submission.csv", index=False )<set_options>
test['k_t'] = test.apply(lambda x : x['text'] + [x['keyword']+x['keyword']+x['keyword']] if type(x['keyword'])== str else x['text'],axis=1 )
warnings.filterwarnings('ignore' )<set_options>
ps_2 = PorterStemmer()
wnl_2 = WordNetLemmatizer()
text_reconstruct_test = []
for i,x in enumerate(test.k_t.values):
try:
a = wnl_2.lemmatize(ps_2.stem(x))
except AttributeError:
a = list(set([wnl_2.lemmatize(ps_2.stem(word)) for j,word in enumerate(x)]))
text_reconstruct_test.append(a )
def reduce_mem_usage(props):
start_mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage of properties dataframe is :",start_mem_usg," MB")
 NAlist = []
for col in props.columns:
if props[col].dtype != object:
IsInt = False
mx = props[col].max()
mn = props[col].min()
if not np.isfinite(props[col] ).all() :
NAlist.append(col)
 props[col].fillna(mn-1,inplace=True)
 asint = props[col].fillna(0 ).astype(np.int64)
 result =(props[col] - asint)
 result = result.sum()
if result > -0.01 and result < 0.01:
IsInt = True
if IsInt:
if mn >= 0:
if mx < 255:
props[col] = props[col].astype(np.uint8)
 elif mx < 65535:
props[col] = props[col].astype(np.uint16)
 elif mx < 4294967295:
props[col] = props[col].astype(np.uint32)
 else:
props[col] = props[col].astype(np.uint64)
 else:
if mn > np.iinfo(np.int8 ).min and mx < np.iinfo(np.int8 ).max:
props[col] = props[col].astype(np.int8)
 elif mn > np.iinfo(np.int16 ).min and mx < np.iinfo(np.int16 ).max:
props[col] = props[col].astype(np.int16)
 elif mn > np.iinfo(np.int32 ).min and mx < np.iinfo(np.int32 ).max:
props[col] = props[col].astype(np.int32)
 elif mn > np.iinfo(np.int64 ).min and mx < np.iinfo(np.int64 ).max:
props[col] = props[col].astype(np.int64)
 else:
props[col] = props[col].astype(np.float32)
 print("___MEMORY USAGE AFTER COMPLETION:___")
 mem_usg = props.memory_usage().sum() / 1024**2
print("Memory usage is: ",mem_usg," MB")
 print("This is ",100*mem_usg/start_mem_usg,"% of the initial size")
 return props, NAlist
def train_short_form_loader(feature_file,target_file,extra_target_file=None):

 train_features = pd.read_csv(feature_file)
 train_targets = pd.read_csv(target_file)
 train_features,_= reduce_mem_usage(train_features)
 train_targets,_ = reduce_mem_usage(train_targets)
 if extra_target_file is not None:
extra_targets = pd.read_csv(extra_target_file)
 extra_targets,_ = reduce_mem_usage(extra_targets)
 train_targets = pd.merge(train_targets,extra_targets,on ='sig_id')
 del extra_targets
targets = train_targets.columns[1:]
train_melt=train_targets.merge(train_features,how="left",on="sig_id")
 del train_features,train_targets
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt , targets.to_list()
def test_short_form_loader(feature_file):

 train_features = pd.read_csv(feature_file)
 train_features,_= reduce_mem_usage(train_features)
 train_melt = train_features.copy()
del train_features
train_melt.set_index("sig_id",inplace=True)
 train_melt["cp_type"]= train_melt["cp_type"].astype('category')
 train_melt["cp_dose"]= train_melt["cp_dose"].astype('category')
 return train_melt 
<define_variables>
test.k_t = text_reconstruct_test
test_word = test.k_t.apply(lambda x: ' '.join(x))
input_directory = '.. /input/lish-moa/'<load_from_csv>
path_for_word2vec = ".. /input/googlenewsvectorsnegative300/GoogleNews-vectors-negative300.bin"
word2vec = gensim.models.KeyedVectors.load_word2vec_format(path_for_word2vec, binary = True )
train,target_cols = train_short_form_loader(input_directory +'train_features.csv',input_directory+'train_targets_scored.csv')
 test = test_short_form_loader(input_directory +"test_features.csv")
<define_variables>
def average_w2v(list_of_tokens, vct, generate_missing = False, dimentions = 300):
if len(list_of_tokens)< 1:
return np.zeros(dimentions)
 if generate_missing:
vector = [vct[item] if item in vct else np.random.rand(dimentions)for item in list_of_tokens]
else:
vector = [vct[item] if item in vct else np.zeros(dimentions)for item in list_of_tokens]
total_length = len(vector)
 sum_of_vectors = np.sum(vector, axis=0)
 average = np.divide(sum_of_vectors, total_length)
 return average
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]<import_modules>
def word2vec_mapping(vect, our_word, generate_missing = False):
mapping = our_word.apply(lambda x: average_w2v(x, vect, generate_missing = generate_missing))
return list(mapping )
from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer
from sklearn.compose import make_column_transformer,ColumnTransformer<set_options>
our_tokenizer = RegexpTokenizer(r'\w+')
 tokenized_input_train = train_word.apply(our_tokenizer.tokenize)
 tokenized_input_test = test_word.apply(our_tokenizer.tokenize )
def seed_everything(seed=42):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )<categorify>
mapped_train = word2vec_mapping(word2vec, train.text)
 mapped_test = word2vec_mapping(word2vec,test.text )
class CatIntMapper(BaseEstimator, TransformerMixin):
def __init__(self ,col,dicti):
self.col = col
self.dicti = dicti
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 def transform(self, X):
assert X[self.col].isin(self.dicti.keys() ).all()
return pd.concat([X.drop(self.col,axis=1),X[self.col].map(self.dicti ).astype(int ).rename(self.col)],axis=1)
 class NamedOutTWrapper(BaseEstimator, TransformerMixin):
def __init__(self,transformer,columns,inplace=False,prefix='_'):
self.transformer = transformer
self.cols = columns
self.inplace = inplace
self.prefix = prefix
self.transformer_name = self._get_transformer_name()
def fit(self, X, y = None):
self.transformer = self.transformer.fit(X[self.cols] , y)
 return self
def fit_transform(self, X, y = None):
transformed_columns = self.transformer.fit_transform(X[self.cols] , y)
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols]
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def transform(self, X):
transformed_columns = self.transformer.transform(X[self.cols])
 out=pd.DataFrame(index=X.index)
 if self.inplace:
out = X[self.cols]
out[self.cols] = transformed_columns
return pd.concat([X.drop(self.cols,axis=1),out],axis=1)
 else:
for i,values in enumerate(transformed_columns.transpose()):
out[ self.transformer_name + self.prefix + str(i)] = values
return pd.concat([X,out],axis=1)
 def _get_transformer_name(self):
return str(self.transformer.__class__ ).split('.')[-1][0:-2]
class IdentityTransformer:

 def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X
def transform(self, X):
return X
class SuppressControls(BaseEstimator, TransformerMixin):
def __init__(self):
pass
def fit(self, X, y = None):
return self
def fit_transform(self, X, y = None):
return X.loc[X['cp_type']!='ctl_vehicle'].drop('cp_type', axis=1)
 def transform(self, X):
return X.loc[X['cp_type']!='ctl_vehicle'].drop('cp_type', axis=1)
<categorify>
classifier_SVM = SVC(C = 2,probability = True)
 classifier_SVM.fit(mapped_train, train.target)
 y_predicted_SVM = classifier_SVM.predict(mapped_test )
def multifold_indexer(train,target_columns,n_splits=10,random_state=12347,**kwargs):
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=n_splits,random_state=random_state,**kwargs)
 folds[ 'kfold']=0
for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=train[target_columns])) :
folds.iloc[v_idx,-1] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 return folds 
<categorify>
classifier_LogisticRegression = LogisticRegression(C=30.0, class_weight = 'balanced', solver = 'newton-cg',
multi_class = 'multinomial', n_jobs = -1, random_state = 42)
 classifier_LogisticRegression.fit(mapped_train, train.target)
 y_predicted_LogisticRegression = classifier_LogisticRegression.predict(mapped_test )
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct 
<choose_model_class>
train = train.fillna('Zilch')
 important_key_words = train.groupby('keyword' ).agg({'text':np.size, 'target':np.mean} ).rename(columns={'text':'count', 'target':'frequency'} )
class DAE_Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=1100,hidden_size2=1300):
super(DAE_Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size2))
self.batch_norm3 = nn.BatchNorm1d(hidden_size2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size2, hidden_size))
self.dense4 = nn.utils.weight_norm(nn.Linear(hidden_size, num_features))
def forward(self, x,mode='DAE'):
x1 = F.relu(self.dense1(x))
x2 = self.batch_norm2(x1)
 x2 = F.relu(self.dense2(x2))
x3 = self.batch_norm3(x2)
 x3 = F.relu(self.dense3(x3))
out = self.dense4(x3)
 if mode == 'DAE':
return out
else:
return x1,x2,x3 
<load_from_csv>
additional_list = ['bushfires','evacuated','forestfire','hostages','rescuers','sinkhole','thunderstorm']
map_controls = CatIntMapper('cp_type',{'ctl_vehicle': 0, 'trt_cp': 1})
 map_dose = CatIntMapper('cp_dose',{'D1': 1, 'D2': 0})
 map_time = CatIntMapper('cp_time',{24: 0, 48: 1, 72: 2})
 train = pd.read_csv(f'{input_directory}/train_features.csv')
 GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]
GENES
Rankg_g_tansform = NamedOutTWrapper(QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal"),columns= GENES+CELLS,inplace=True)
 PCA_g_tansform = NamedOutTWrapper(PCA(20),columns= GENES,prefix ='_g')
 PCA_c_tansform = NamedOutTWrapper(PCA(20),columns= CELLS,prefix ='_c')
 class ColumnDropper(BaseEstimator, TransformerMixin):
def __init__(self, cols):
self.cols=cols
def fit(self, X, y = None):
return self
def transform(self, X, y = None):
return X.drop(self.cols,axis=1)
 CatDropper =ColumnDropper(cols=['cp_type','cp_time','cp_dose'])
 transformers_list=[map_controls,map_dose,map_time,Rankg_g_tansform,CatDropper]<create_dataframe>
prob_disaster = 0.95
keyword_list_disaster95 = list(important_key_words[important_key_words['frequency']>prob_disaster].index)+ additional_list
exp_name = 'test_DAE_0.2_all_together'
def run_inference(X_train,y_train,X_valid,y_valid,X_test,fold, seed,inference_only=False,**kwargs):
seed_everything(seed)
 if not inference_only:
train_dataset = MoADataset(X_train, y_train)
 valid_dataset = MoADataset(X_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 testdataset = TestDataset(X_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = DAE_Model(
 num_features= X_train.shape[1] ,
num_targets= X_train.shape[1],
**kwargs 
)
 model.load_state_dict(torch.load(f".. /input/swap-dae-noise0-2/FOLD{fold}_{exp_name}.pth",map_location=torch.device('cpu')))
 model.to(DEVICE)
 if not inference_only:
oof = inference_infer_features_fn(model, validloader, DEVICE)
 else:
oof= 0
predictions = infer_features_fn(model, testloader, DEVICE)
 predictions = predictions
return oof, predictions
transformers_list
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 1000
BATCH_SIZE = 640
LEARNING_RATE = 2e-3
WEIGHT_DECAY = 1e-8
NFOLDS = 10
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
GAMMA=0.5
FACTOR=0.75
hidden_size=1100
hidden_size2=1300
PATIENCE=10
THRESHOLD = 5e-3
def infer_features_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs,mode='get_features')
 preds.append(torch.cat(outputs,axis=1 ).detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<define_variables>
numbers_of_95certain_disasters = test['id'][test.keyword.isin(keyword_list_disaster95)]
SEED = [0]
train,target_cols = train_short_form_loader('.. /input/lish-moa/train_features.csv','.. /input/lish-moa/train_targets_scored.csv')
 test = test_short_form_loader(".. /input/lish-moa/test_features.csv")
 train = pd.concat([train,test])
 train[target_cols]= train[target_cols].fillna(0)
 test = train.copy()
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
train = multifold_indexer(train,target_cols,n_splits=NFOLDS)
 for fold in range(NFOLDS):
train_df = train[train['kfold'] != fold]
valid_df = train[train['kfold'] == fold]
feature_cols = [col for col in train_df.columns if not(col in target_cols+['kfold'])]
pipeline_val = make_pipeline(*transformers_list)
 X_train, y_train = train_df[feature_cols], train_df[target_cols]
X_valid, y_valid = valid_df[feature_cols], valid_df[target_cols].values
X_train = pipeline_val.fit_transform(X_train)
 X_train = X_train.values
X_valid = pipeline_val.transform(X_valid)
 valid_index = X_valid.index
X_valid = X_valid.values
y_train = y_train.values
X_test = test[feature_cols]
X_test = pipeline_val.transform(X_test ).values
pred_ = run_inference(X_train,y_train,X_valid,y_valid,X_test,fold, seed,inference_only=True)
 break<create_dataframe>
y_predicted = np.zeros(len(y_predicted_SVM))
for i in range(0,len(y_predicted_SVM)) :
if i in numbers_of_95certain_disasters:
y_predicted[i] = 1
else:
y_predicted[i] = y_predicted_SVM[i]
transformed_features = pd.DataFrame(pred_[1],index=test.index )<choose_model_class>
sample_submission["target"] = [int(i)for i in y_predicted]
class DAE_Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size=1100,hidden_size2=1300):
super(DAE_Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size2))
self.batch_norm3 = nn.BatchNorm1d(hidden_size2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size2, hidden_size))
self.dense4 = nn.utils.weight_norm(nn.Linear(hidden_size, num_features))
def forward(self, x,mode='DAE'):
x1 = F.relu(self.dense1(x))
x2 = self.batch_norm2(x1)
 x2 = F.relu(self.dense2(x2))
x3 = self.batch_norm3(x2)
 x3 = F.relu(self.dense3(x3))
out = self.dense4(x3)
 if mode == 'DAE':
return out
else:
return x1,x2,x3 
<import_modules>
sample_submission.to_csv("submission.csv", index = False )
from sklearn.decomposition import PCA
from sklearn.preprocessing import QuantileTransformer
from sklearn.pipeline import make_pipeline,make_union 
<categorify>
holy_grail = pd.read_csv(".. /input/holygrail/submission.csv")
 y_test = holy_grail["target"]
map_controls = CatIntMapper('cp_type',{'ctl_vehicle': 0, 'trt_cp': 1})
 map_dose = CatIntMapper('cp_dose',{'D1': 1, 'D2': 0})
 map_time = CatIntMapper('cp_time',{24: 0, 48: 1, 72: 2})
<load_from_csv>
def scores(y_test, y_predicted):
accuracy = accuracy_score(y_test, y_predicted)
 f1 = f1_score(y_test, y_predicted, pos_label = None, average = 'weighted')
 precision = precision_score(y_test, y_predicted, pos_label = None, average = 'weighted')
 recall = recall_score(y_test, y_predicted, pos_label = None, average = 'weighted')
 print("accuracy = %.4f, f1 = %.4f, precision = %.4f, recall = %.4f" %(accuracy, f1, precision, recall))
return accuracy, f1, precision, recall
train = pd.read_csv(f'{input_directory}/train_features.csv' )<define_variables>
accuracy, f1, precision, recall = scores(y_test, y_predicted )
GENES = [col for col in train.columns if col.startswith('g-')]
CELLS = [col for col in train.columns if col.startswith('c-')]<choose_model_class>
probabilities_SVM = classifier_SVM.predict_proba(mapped_test)
 probabilities_LogisticRegression = classifier_LogisticRegression.predict_proba(mapped_test)
 probability_of_ones_SVM = probabilities_SVM[:,1]
probability_of_ones_LogisticRegression = probabilities_LogisticRegression[:,1]
FPrates_SVM, TPrates_SVM, thresholds_SVM = roc_curve(y_test, probability_of_ones_SVM)
 FPrates_LogisticRegression, TPrates_LogisticRegression, thresholds_LogisticRegression = roc_curve(y_test, probability_of_ones_LogisticRegression)
 sns.set_style('whitegrid')
 plt.figure(figsize =(10, 8))
plt.plot(FPrates_SVM, TPrates_SVM, label = 'SVM')
 plt.plot(FPrates_LogisticRegression, TPrates_LogisticRegression, label = 'Logistic Regression')
 plt.plot([0, 1], [0, 1], color = 'blue', linestyle = '--')
 plt.axis([0, 1, 0, 1])
 plt.xlabel('False Positive Rate', fontsize = 14)
 plt.ylabel('True Positive Rate', fontsize = 14)
 plt.title('ROC Curve', fontsize = 14)
 plt.legend(loc = "lower right")
 plt.show()