Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<drop_column>
warnings.filterwarnings('ignore')
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )<split>
class Config() :
def __init__(self):
self.work_dir = Path("/kaggle")
 self.train_path = self.work_dir / "input/nlp-getting-started/train.csv"
self.test_path = self.work_dir / "input/nlp-getting-started/test.csv"
self.submission_path = self.work_dir / "working/submission.csv"
self.model_name = "distilbert-base-uncased"
self.metric_name = "val_f1"
self.seed = 41
self.batch_size = 64
self.nfold = 5
def split_data(train, y, households, test_percentage=0.20, seed=None):
train2 = train.copy()
cv_hhs = np.random.choice(households, size=int(len(households)* test_percentage), replace=False)
 cv_idx = np.isin(households, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
return X_train, y_train, X_test, y_test<split>
class DisasterTweetsDataset(Dataset):
def __init__(self, encodings, labels=None):
self.encodings = encodings
self.labels = labels
def __len__(self):
return len(self.encodings.input_ids)
 def __getitem__(self, index):
input_id = torch.tensor(self.encodings.input_ids[index])
 attention_mask = torch.tensor(self.encodings.attention_mask[index])
 label = self.labels[index] if self.labels is not None else -1
return input_id, attention_mask, label
class DisasterTweetsDataModule(pl.LightningDataModule):
def __init__(self, config):
super().__init__()
self.config = config
self.train = pd.read_csv(config.train_path)
 self.test = pd.read_csv(config.test_path)
 self.tokenizer = DistilBertTokenizer.from_pretrained(config.model_name)
 skf = StratifiedKFold(n_splits=config.nfold, shuffle=True, random_state=config.seed)
 self.fold_indices = {fold:(t_idx, v_idx)for fold,(t_idx, v_idx)in enumerate(skf.split(list(self.train.text), list(self.train.target)))}
def _fold_data(self, fold):
train_index, val_index = self.fold_indices[fold]
t_df = self.train.loc[train_index, ["text", "target"]]
v_df = self.train.loc[val_index, ["text", "target"]]
return list(t_df.text), list(v_df.text), list(t_df.target), list(v_df.target)
 def setup(self, stage=None, fold=0, verify_train_data=False):
train_texts, val_texts, train_targets, val_targets = self._fold_data(fold)
 self.train_ds = DisasterTweetsDataset(self.tokenizer(train_texts, truncation=True, padding=True), train_targets)
 self.val_ds = DisasterTweetsDataset(self.tokenizer(val_texts, truncation=True, padding=True), val_targets)
 test_texts = list(self.test.text)if not verify_train_data else list(self.train.text)
 self.test_ds = DisasterTweetsDataset(self.tokenizer(test_texts, truncation=True, padding=True))
def train_dataloader(self):
return DataLoader(self.train_ds, batch_size=self.config.batch_size)
 def val_dataloader(self):
return DataLoader(self.val_ds, batch_size=self.config.batch_size)
 def test_dataloader(self):
return DataLoader(self.test_ds, batch_size=self.config.batch_size )
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=1977)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.15), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar<define_variables>
class DisasterTweetsNet(pl.LightningModule):
def __init__(self, train=True):
super(DisasterTweetsNet, self ).__init__()
self.model = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased", return_dict=True)
 self.F1 = F1(num_classes=2)
 if train:
self.model.train()
def forward(self, input_ids, attention_mask):
return self.model(input_ids=input_ids, attention_mask=attention_mask)
 def training_step(self, batch, batch_idx):
return self.common_step(batch, mode="train")
 def validation_step(self, batch, batch_idx):
return self.common_step(batch, mode="val")
 def test_step(self, batch, batch_idx):
input_ids, attention_masks, labels = batch
outputs = self.forward(input_ids, attention_masks)
 return torch.softmax(outputs.logits, dim=1)
 def common_step(self, batch, mode="train"):
input_ids, attention_masks, labels = batch
outputs = self.forward(input_ids, attention_masks)
 loss = F.cross_entropy(outputs.logits, labels)
 preds = torch.argmax(outputs.logits, dim=1)
 f1 = self.F1(preds, labels)
 self.log(f"{mode}_loss", loss, on_step=False, on_epoch=True, prog_bar=True, logger=True)
 self.log(f"{mode}_f1", f1, on_step=False, on_epoch=True, prog_bar=True, logger=True)
 return loss
def configure_optimizers(self):
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{"params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay)], "weight_decay": 0.01},
{"params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0}
]
return AdamW(optimizer_grouped_parameters, lr=1e-5 )
extra_drop_features = [
'geo_manual_elec_LE_2',
'agg18_estadocivil5_COUNT',
'agg18_estadocivil7_COUNT',
'geo_pared_LE_5',
'agg18_estadocivil6_COUNT',
'geo_pared_LE_4',
'agg18_parentesco10_COUNT',
'num_over_18',
'agg18_estadocivil1_MEAN',
'agg18_estadocivil4_COUNT',
'r4t3_to_tamhog',
'agg18_estadocivil3_COUNT',
'geo_pared_LE_3',
'geo_pared_LE_6',
'geo_pared_LE_0',
'geo_pared_LE_7',
'agg18_parentesco7_COUNT',
'agg18_parentesco10_MEAN',
'agg18_parentesco8_MEAN',
'rez_esc',
'hacdor',
'hacapo',
'fe_people_weird_stat',
'geo_hogar_mayor',
'geo_eviv_LE_0',
'geo_epared_LE_0',
'geo_epared_LE_2',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_0',
'agg18_parentesco9_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco11_COUNT',
'parentesco_LE',
'agg18_parentesco6_MEAN',
'agg18_parentesco6_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco4_MEAN',
'agg18_parentesco4_COUNT',
'geo_energcocinar_LE_1',
'agg18_parentesco3_COUNT',
'geo_energcocinar_LE_2',
'agg18_parentesco2_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco12_MEAN',
'agg18_parentesco12_COUNT',
] + ['geo_hogar_nin', 'geo_manual_elec_LE_3']<drop_column>
class OutputRecorder(Callback):
def __init__(self, config):
super().__init__()
self.__outputs_one_epoch = []
self.__records = []
self.config = config
@property
def outputs(self):
outputs = torch.zeros(( len(self.__records), *self.__records[0].shape), dtype=self.__records[0].dtype)
 for i, ot in enumerate(self.__records):
outputs[i] = ot
return outputs
def _record(self, outputs):
outputs = outputs.cpu().detach()
self.__outputs_one_epoch.append(outputs)
 def on_test_batch_end(self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx):
self._record(outputs)
 def on_test_epoch_end(self, trainer, pl_module):
outputs = torch.cat(self.__outputs_one_epoch)
 self.__records.append(outputs)
 self.__outputs_one_epoch = []
def make_submission(self):
results = torch.mean(self.outputs, dim=0)
 results = torch.argmax(results, dim=1)
 test = pd.read_csv(self.config.test_path)
 test["target"] = results
test = test.loc[:, ["id", "target"]]
test.to_csv(self.config.submission_path, index=False)
 return test
lgb_drop_cols = extra_drop_features + ["idhogar", 'parentesco1']
try:
X_train.drop(lgb_drop_cols, axis=1, inplace=True)
 X_test.drop(lgb_drop_cols, axis=1, inplace=True)
 except:
print("Error dropping columns" )<compute_train_metric>
class ModelSaveDir() :
def __init__(self, root_models_dir, metric_name="val_acc"):
self.root_models_dir = Path(root_models_dir)
 self.metric_name = metric_name
self.pattern = re.compile(r'vfold=(\d+)-epoch=(\d+)-' + metric_name + r'=(\d\.\d+)')
 def get_model_path_template(self, fold):
return self.root_models_dir /(f"vfold={fold}-" + "{epoch}-{" + self.metric_name + ":.4f}")
 def get_model_paths(self):
model_paths = {}
re_paths = [(self.pattern.search(str(p)) , str(p)) for p in self.root_models_dir.iterdir() if p.is_file() and p.suffix == ".ckpt"]
for fold in set([int(r.group(1)) for r,_ in re_paths]):
paths = [p for r,p in re_paths if int(r.group(1)) == fold]
metrics = np.array([float(r.group(3)) for r,_ in re_paths if int(r.group(1)) == fold])
 model_paths[fold] = paths[np.argmax(metrics)]
return model_paths
@classmethod
def make_new_dir(cls, work_dir, metric_name="val_acc"):
now = datetime.datetime.today()
root_models_dir = Path(work_dir)/ f"models/{now}"
root_models_dir.mkdir(parents=True)
 return ModelSaveDir(root_models_dir, metric_name )
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 90, 'num_leaves': 16, 'subsample': 0.94, 'reg_lambda': 0.5, }
opt_parameters = {'colsample_bytree': 0.88, 'min_child_samples': 95, 'num_leaves': 25, 'subsample': 0.94, 'reg_lambda': 0.5, }
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":800,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]<train_model>
class CVTrainer() :
def __init__(self, config, dm=None):
self.config = config
self.dm = DisasterTweetsDataModule(self.config)if not dm else dm
def fit(self, nepoch, model_save_dir, folds=[0]):
model_paths = model_save_dir.get_model_paths()
for fold in folds:
model = DisasterTweetsNet.load_from_checkpoint(checkpoint_path=str(model_paths[fold])) if fold in model_paths else DisasterTweetsNet()
template_path = model_save_dir.get_model_path_template(fold)
 checkpoint_callback = ModelCheckpoint(filepath=template_path, monitor=self.config.metric_name, save_top_k=1, mode="max")
 logger = TensorBoardLogger(model_save_dir.root_models_dir, name=f"fold={fold}")
 self.dm.setup(fold=fold)
 trainer = pl.Trainer(gpus=1, max_epochs=nepoch, progress_bar_refresh_rate=10, checkpoint_callback=checkpoint_callback, logger=logger)
 trainer.fit(model, self.dm)
 def test(self, model_save_dir, nepoch=1, verify_train_data=False):
outputRecorder = OutputRecorder(self.config)
 model_paths = model_save_dir.get_model_paths()
for fold in model_paths.keys() :
model = DisasterTweetsNet.load_from_checkpoint(checkpoint_path=str(model_paths[fold]))
self.dm.setup(verify_train_data=verify_train_data)
 trainer = pl.Trainer(gpus=1, max_epochs=1, progress_bar_refresh_rate=10, callbacks=[outputRecorder])
 for epoch in range(nepoch):
trainer.test(model, datamodule=self.dm)
 return outputRecorder
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):
X_train, y_train, X_test, y_test = split_data(X, y, households=train_households, seed=1972)
 fit_params["eval_set"] = [(X_train,y_train),(X_test,y_test)]
if sample_weight is not None:
estimator.fit(X_train, y_train, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X_train, y_train, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
config = Config()
clfs = []
for i in range(30):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_train, y_train, **fit_params)
 clf_final = vc.estimators_[0]<compute_train_metric>
model_save_dir = ModelSaveDir.make_new_dir(config.work_dir, metric_name=config.metric_name)
 model_save_dir.root_models_dir
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_train_metric>
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_test_metric>
cvTrainer = CVTrainer(config )
ranked_features = feature_importance(clf_final, X_train )<create_dataframe>
cvTrainer.fit(10, model_save_dir, [0] )
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids<predict_on_test>
y_subm['Target'] = clf_final.predict(test[X_train.columns])+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test[X_train.columns])+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test[X_train.columns])+ 1<save_to_csv>
recorder = cvTrainer.test(model_save_dir )