Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<split>
import tensorflow as tf
from transformers import BertTokenizer, TFBertModel, AdamWeightDecay
from tensorflow import keras
import matplotlib.pyplot as plt
import string
import re
import numpy as np
import pandas as pd
import emoji
import os
from sklearn.metrics import accuracy_score
from skopt.utils import use_named_args
from skopt.space import Real, Integer, Categorical
from skopt import gp_minimize
from sklearn.model_selection import train_test_split
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import StackingClassifier
from sklearn.model_selection import KFold
marking_list, train_ids_list, valid_ids_list = [], [], []
for ii in range(len(BEST_PATHS)) :
print('
** weights --
cfg.DATASETS.VALID_FOLD = ii
marking, train_ids0, valid_ids0 = split_dataset(cfg)
 if SWAP_VALID_AND_TRAIN:
print('swap!!')
 valid_ids, train_ids =train_ids0, valid_ids0
else:
train_ids, valid_ids =train_ids0, valid_ids0
marking.head(3)
 print(ii, marking.shape, df.shape)
 marking = pd.concat([marking,df])
 marking = marking.reset_index()
marking = marking.drop(['index'], axis=1)
 print(ii, marking.shape, df.shape)
 marking.tail(3)
 print('
train_ids = np.concatenate([train_ids, df.image_id.unique() ])
 print('
marking_list.append(marking)
 train_ids_list.append(train_ids)
 valid_ids_list.append(valid_ids)
 print(len(marking_list), len(train_ids_list), len(valid_ids_list))<save_to_csv>
train = pd.read_csv('.. /input/nlp-getting-started/train.csv', header = 0, encoding="utf8")
 test = pd.read_csv('.. /input/nlp-getting-started/test.csv', header = 0, encoding="utf8")
 train = train.drop(columns=['id', 'keyword', 'location'])
 test = test.drop(columns=['id', 'keyword', 'location'])
 train = train.drop_duplicates().reset_index(drop=True)
 train.sample(10, random_state=18 )
<set_options>
def remove_tweet_object(tweet):
tweet = re.sub(r"https?://\S+|www\.\S+", " ", tweet)
 tweet = re.sub(r"
tweet = re.sub(r"@\w+", " ", tweet)
 tweet = emoji.get_emoji_regexp().sub(" ", tweet)
 return tweet
def text_filter(tweet):
tweet = tweet.lower()
tweet = re.sub(r"’", "'", tweet)
 tweet = remove_tweet_object(tweet)
 return tweet
train["text"] = train["text"].apply(lambda x: text_filter(x))
test["text"] = test["text"].apply(lambda x: text_filter(x))
train.sample(10, random_state=18 )
%%writefile./data/transforms/build.py
def get_train_transforms(cfg):
return A.Compose(
 [
A.Resize(1024, 1024, p=1.0),
A.RandomSizedCrop(min_max_height=cfg.INPUT.RSC_MIN_MAX_HEIGHT, height=cfg.INPUT.RSC_HEIGHT,
width=cfg.INPUT.RSC_WIDTH, p=cfg.INPUT.RSC_PROB),
A.OneOf([
A.HueSaturationValue(hue_shift_limit=cfg.INPUT.HSV_H, sat_shift_limit=cfg.INPUT.HSV_S,
val_shift_limit=cfg.INPUT.HSV_V, p=cfg.INPUT.HSV_PROB),
A.RandomBrightnessContrast(brightness_limit=cfg.INPUT.BC_B,
contrast_limit=cfg.INPUT.BC_C, p=cfg.INPUT.BC_PROB),
],p=cfg.INPUT.COLOR_PROB),
A.ToGray(p=cfg.INPUT.TOFGRAY_PROB),
A.HorizontalFlip(p=cfg.INPUT.HFLIP_PROB),
A.VerticalFlip(p=cfg.INPUT.VFLIP_PROB),
A.Cutout(num_holes=cfg.INPUT.COTOUT_NUM_HOLES, max_h_size=cfg.INPUT.COTOUT_MAX_H_SIZE,
max_w_size=cfg.INPUT.COTOUT_MAX_W_SIZE, fill_value=cfg.INPUT.COTOUT_FILL_VALUE, p=cfg.INPUT.COTOUT_PROB),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms(cfg):
return A.Compose(
 [
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_test_transform() :
return A.Compose([
ToTensorV2(p=1.0)
 ])
 def build_transforms(cfg, is_train=True):
if is_train:
transform = get_train_transforms(cfg)
 else:
transform = get_valid_transforms(cfg)
 return transform 
<create_dataframe>
X_train, X_val, y_train, y_val = train_test_split(train.text, train.target, test_size=0.2, random_state=42 )
warnings.filterwarnings("ignore")
 def build_dataset(cfg, marking,train_ids, valid_ids):
train_dataset = train_wheat(
 root = cfg.DATASETS.ROOT_DIR,
image_ids=train_ids,
marking=marking,
transforms=build_transforms(cfg, is_train=True),
test=False, 
)
 validation_dataset = train_wheat(
 root=cfg.DATASETS.ROOT_DIR,
image_ids=valid_ids,
marking=marking,
transforms=build_transforms(cfg, is_train=False),
test=True, 
)
 return train_dataset, validation_dataset
def make_data_loader(cfg, marking,train_ids, valid_ids, is_train=True):
if is_train:
batch_size = cfg.SOLVER.IMS_PER_BATCH
else:
batch_size = cfg.TEST.IMS_PER_BATCH
train_dataset, validation_dataset = build_dataset(cfg, marking,train_ids, valid_ids)
 num_workers = cfg.DATALOADER.NUM_WORKERS
train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=batch_size,
sampler=torch.utils.data.sampler.RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=num_workers,
collate_fn=collate_batch, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=batch_size,
num_workers=num_workers,
shuffle=False,
sampler=torch.utils.data.sampler.SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_batch, 
)
 return train_loader, val_loader 
<train_model>
name_bert = "bert-base-uncased"
tokenizer = BertTokenizer.from_pretrained(name_bert, do_lower_case=True)
 bert_model = TFBertModel.from_pretrained(name_bert, output_hidden_states=True, trainable=True)
 bert_w = bert_model.get_weights()
class Fitter:
def __init__(self, model, device, cfg, train_loader, val_loader, logger, mixed_precision=APEX, accum=ACCUM):
self.config = cfg
self.epoch = 0
self.train_loader = train_loader
self.val_loader = val_loader
self.base_dir = f'{self.config.OUTPUT_DIR}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.logger = logger
self.best_final_score = 0.0
self.best_score_threshold = SCORE_THRESHOLD
self.mixed_precision = mixed_precision
self.accumulate = accum
self.model = model
self.device = device
self.model.to(self.device)
 self.optimizer = make_optimizer(cfg, model)
 if self.mixed_precision:
self.model, self.optimizer = amp.initialize(self.model, self.optimizer, opt_level="O1", verbosity=0)
 self.scheduler = make_scheduler(cfg, self.optimizer, train_loader)
 self.logger.info(f'Fitter prepared.Device is {self.device}')
 self.all_predictions = []
self.early_stop_epochs = 0
self.early_stop_patience = self.config.SOLVER.EARLY_STOP_PATIENCE
self.do_scheduler = True
self.logger.info("Start training")
 def fit(self):
for epoch in range(self.epoch, self.config.SOLVER.MAX_EPOCHS):
if epoch < self.config.SOLVER.WARMUP_EPOCHS:
lr_scale = min(1., float(epoch + 1)/ float(self.config.SOLVER.WARMUP_EPOCHS))
for pg in self.optimizer.param_groups:
pg['lr'] = lr_scale * self.config.SOLVER.BASE_LR
self.do_scheduler = False
else:
self.do_scheduler = True
if self.config.VERBOSE:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.logger.info(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch()
self.logger.info(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
best_score_threshold, best_final_score = self.validation()
self.logger.info(f'[RESULT]: Val.Epoch: {self.epoch}, Best Score Threshold: {best_score_threshold:.2f}, Best Score: {best_final_score:.5f}, time: {(time.time() - t):.5f}')
 if best_final_score > self.best_final_score:
self.logger.info('** UPDATE best weights **!')
 self.best_final_score = best_final_score
self.best_score_threshold = best_score_threshold
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint.bin')
 self.save_model(f'{self.base_dir}/best-model.bin')
 self.save_predictions(f'{self.base_dir}/all_predictions.csv')
 self.early_stop(best_final_score)
 if self.early_stop_epochs > self.config.SOLVER.EARLY_STOP_PATIENCE:
self.logger.info('Early Stopping!')
 break
if self.epoch % self.config.SOLVER.CLEAR_OUTPUT == 0:
pass
self.epoch += 1
def validation(self):
self.model.eval()
t = time.time()
self.all_predictions = []
torch.cuda.empty_cache()
valid_loader = tqdm(self.val_loader, total=len(self.val_loader), desc="Validating")
 with torch.no_grad() :
for step,(images, targets, image_ids)in enumerate(valid_loader):
images = list(image.cuda() for image in images)
 outputs = self.model(images)
 inference(self.all_predictions, images, outputs, targets, image_ids)
 valid_loader.set_description(f'Validate Step {step}/{len(self.val_loader)}, ' + \
f'time: {(time.time() - t):.5f}')
 best_score_threshold, best_final_score = evaluate(self.all_predictions)
 return best_score_threshold, best_final_score
def train_one_epoch(self):
self.model.train()
summary_loss = AverageMeter()
loss_box_reg = AverageMeter()
loss_classifier = AverageMeter()
loss_objectness = AverageMeter()
loss_rpn_box_reg = AverageMeter()
t = time.time()
train_loader = tqdm(self.train_loader, total=len(self.train_loader), desc="Training")
 for step,(images, targets, image_ids)in enumerate(train_loader):
images = torch.stack(images)
 if self.mixed_precision == False:
images = images.to(self.device ).float()
else:
images = images.to(self.device ).half()
batch_size = images.shape[0]
targets = [{k: v.to(self.device)for k, v in t.items() } for t in targets]
for i in range(len(targets)) :
if self.mixed_precision == False:
targets[i]['boxes'] = targets[i]['boxes'].float()
else:
targets[i]['boxes'] = targets[i]['boxes'].half()
loss_dict = self.model(images, targets)
 loss = sum(loss for loss in loss_dict.values())
 box_reg = loss_dict['loss_box_reg']
classifier = loss_dict['loss_classifier']
objectness = loss_dict['loss_objectness']
rpn_box_reg = loss_dict['loss_rpn_box_reg']
if self.mixed_precision:
with amp.scale_loss(loss, self.optimizer)as scaled_loss:
scaled_loss = scaled_loss/ self.accumulate
scaled_loss.backward()
else:
loss = loss/ self.accumulate
loss.backward()
summary_loss.update(loss.item() , batch_size)
 loss_box_reg.update(box_reg.item() , batch_size)
 loss_classifier.update(classifier.item() , batch_size)
 loss_objectness.update(objectness.item() , batch_size)
 loss_rpn_box_reg.update(rpn_box_reg.item() , batch_size)
 if step % self.accumulate == 0:
self.optimizer.step()
self.optimizer.zero_grad()
if self.do_scheduler:
self.scheduler.step()
train_loader.set_description(f'Train Step {step}/{len(self.train_loader)}, ' + \
f'Learning rate {self.optimizer.param_groups[0]["lr"]}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'loss_box_reg: {loss_box_reg.avg:.5f}, ' + \
f'loss_classifier: {loss_classifier.avg:.5f}, ' + \
f'loss_objectness: {loss_objectness.avg:.5f}, ' + \
f'loss_rpn_box_reg: {loss_rpn_box_reg.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}')
 return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_score_threshold': self.best_score_threshold,
'best_final_score': self.best_final_score,
'epoch': self.epoch,
}, path)
 def save_model(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.state_dict() ,
'best_score_threshold': self.best_score_threshold,
'best_final_score': self.best_final_score,
}, path)
 def save_predictions(self, path):
df = pd.DataFrame(self.all_predictions)
 df.to_csv(path, index=False)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_score_threshold = SCORE_THRESHOLD
self.best_final_score = 0.69
self.epoch = checkpoint['epoch'] + 1
def early_stop(self, score):
if score < self.best_final_score:
self.early_stop_epochs += 1
else:
self.early_stop_epochs = 0<set_options>
combined = pd.concat([X_train, X_val, test.text], axis=0)
 combined = tokenizer(combined.values.tolist() , padding=True, truncation=True, return_tensors='tf')
 train_input =(combined["input_ids"][:len(X_train)], combined["attention_mask"][:len(X_train)], combined["token_type_ids"][:len(X_train)])
 val_input =(combined["input_ids"][len(X_train):len(X_train)+len(X_val)], combined["attention_mask"][len(X_train):len(X_train)+len(X_val)], combined["token_type_ids"][len(X_train):len(X_train)+len(X_val)])
 test_input =(combined["input_ids"][len(X_train)+len(X_val):], combined["attention_mask"][len(X_train)+len(X_val):], combined["token_type_ids"][len(X_train)+len(X_val):] )
cfg.defrost()
cfg['DATASETS']['ROOT_DIR'] = NEW_INPUT_PATH
cfg.INPUT.HSV_H = HSV_H
cfg.INPUT.HSV_S = HSV_S
cfg.INPUT.HSV_V = HSV_V
cfg.INPUT.BC_B = BC_B
cfg.INPUT.BC_C = BC_C
cfg.INPUT.COTOUT_NUM_HOLES=0
cfg.SOLVER.BASE_LR = BASE_LR
cfg.SOLVER.BIAS_LR_FACTOR = BIAS_LR_FACTOR
cfg.SOLVER.MOMENTUM=MOMENTUM
cfg.SOLVER.WARMUP_EPOCHS=WARMUP_EPOCHS
OUTPUT_DIRS = []
for ii in range(len(models)) :
OUTPUT_DIRS.append("/kaggle/working/weights_%d/"%ii)
 if os.path.exists(OUTPUT_DIRS[-1])== False:
print('create ',OUTPUT_DIRS[-1])
 !mkdir {OUTPUT_DIRS[-1]}
!ls /kaggle/working/<init_hyperparams>
bert_model.set_weights(bert_w)
 optimizer = AdamWeightDecay(learning_rate=3e-5, weight_decay_rate=0.01, exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"])
 loss = keras.losses.BinaryCrossentropy(from_logits=False, label_smoothing=0.0)
 input_word_ids = keras.layers.Input(shape=(train_input[0].shape[1],), dtype=tf.int32, name="input_ids")
 input_mask = keras.layers.Input(shape=(train_input[0].shape[1],), dtype=tf.int32, name="attention_mask")
 segment_ids = keras.layers.Input(shape=(train_input[0].shape[1],), dtype=tf.int32, name="token_type_ids")
 outputs = bert_model([input_word_ids, input_mask, segment_ids])
 drop = keras.layers.Dropout(0.1 )(outputs[1])
 relu = keras.layers.Dense(96, "relu" )(drop)
 drop = keras.layers.Dropout(0.1 )(relu)
 out = keras.layers.Dense(1, "sigmoid" )(drop)
 model = keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(optimizer=optimizer, loss=loss, metrics=["accuracy"])
 model.summary()
history_bert = model.fit(
 train_input,
y_train,
validation_data=(val_input, y_val),
epochs=1,
batch_size=32,
callbacks=[tf.keras.callbacks.EarlyStopping(monitor="val_accuracy", patience=1, mode='auto', restore_best_weights=True)]
)
fitters=[]
for ii,path in enumerate(cfg['TEST']['WEIGHT']):
cfg['OUTPUT_DIR'] = OUTPUT_DIRS[ii]
checkpoint = torch.load(path)
 cfg.SOLVER.MAX_EPOCHS = checkpoint['epoch']+PSEUDO_EPOCHS+1
if n_test <11:
cfg.SOLVER.MAX_EPOCHS = checkpoint['epoch']+PSEUDO_EPOCHS_COMMIT+1
print('epochs = %d+%d+%d'%(checkpoint['epoch'],PSEUDO_EPOCHS_COMMIT,1))
print(cfg)
 print('*** weight path ***', path)
 model = build_model(cfg)
 model.load_state_dict(checkpoint['model_state_dict'])
 train_loader, val_loader = make_data_loader(cfg,marking_list[ii],train_ids_list[ii], valid_ids_list[ii])
 logger = setup_logger("logger", cfg['OUTPUT_DIR'], 0)
 fitter = Fitter(model=model, device="cuda", cfg=cfg, train_loader=train_loader, val_loader=val_loader, logger=logger)
 fitter.load(path)
 fitter.fit()
!rm -f {OUTPUT_DIRS[ii]+'last-checkpoint.bin'}
seed_everything(ii+1)
 cfg['SEED'] = ii+1<load_pretrained>
model_bert_enc = keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=relu)
 train_text_vect = model_bert_enc.predict(train_input)
 val_text_vect = model_bert_enc.predict(val_input)
 test_text_vect = model_bert_enc.predict(test_input)
 print("Done" )
best_paths = []
for ii in range(len(OUTPUT_DIRS)) :
if os.path.exists(OUTPUT_DIRS[ii]+'best-checkpoint.bin'):
best_path = OUTPUT_DIRS[ii]+'best-checkpoint.bin'
elif os.path.exists(OUTPUT_DIRS[ii]+'last-checkpoint.bin'):
best_path = OUTPUT_DIRS[ii]+'last-checkpoint.bin'
else:
best_path = BEST_PATHS[ii]
best_paths.append(best_path)
 !ls -lh {OUTPUT_DIRS[ii]}
print(' ----
')
 print('best paths are ', best_paths )<load_pretrained>
space = [Real(1, 1e3, prior="log-uniform", name="C_svm", transform="identity"),
Real(1, 1e3, prior="log-uniform", name="C_lr", transform="identity"),
Categorical(['hinge', 'squared_hinge'], name="loss_svm", transform="identity"),
Categorical(['l1', 'l2', 'elasticnet'], name="penalty_sgd", transform="identity"),
Categorical(['hinge', 'log', 'modified_huber', 'squared_hinge', 'perceptron'], name="loss_sgd", transform="identity"),
]
@use_named_args(space)
 def opt_model(C_svm, loss_svm, penalty_sgd, loss_sgd, C_lr):
estimators = [('1', make_pipeline(MinMaxScaler() , LinearSVC(C=C_svm, loss=loss_svm, random_state=42))), 
('2', make_pipeline(MinMaxScaler() , GaussianNB())) , 
('3', make_pipeline(MinMaxScaler() , SGDClassifier(penalty=penalty_sgd, loss=loss_sgd, random_state=42))),
]
clf = StackingClassifier(estimators=estimators, final_estimator=LogisticRegression(C=C_lr, random_state=42), n_jobs=-1, cv=KFold(n_splits=5, shuffle=True, random_state=42), stack_method='predict')
 clf.fit(train_text_vect, y_train)
 return -clf.score(train_text_vect, y_train)
 res = gp_minimize(opt_model, space, n_initial_points=10, n_calls=50, verbose=True, random_state=0, n_jobs=-1 )
if True:
cfg['OUTPUT_DIR'] = "/kaggle/working/"
cfg['DATASETS']['ROOT_DIR'] = "/kaggle/input/global-wheat-detection"
cfg['TEST']['WEIGHT'] = best_paths
cfg['TEST']['IMS_PER_BATCH'] = 1
print(cfg)
 test_loader = make_test_data_loader(cfg)
 tester = Tester(models=models, device=device, cfg=cfg, test_loader=test_loader, n_viz=N_VIZ)
 tester.load(best_paths)
 print('success load weights!', best_paths)
 tester.test(pp_shrink=PP_SHRINK[1] )<install_modules>
estimators = [ 
('1', make_pipeline(MinMaxScaler() , LinearSVC(C=res.x[0], loss=res.x[2] ,random_state=42))), 
('2', make_pipeline(MinMaxScaler() , GaussianNB())) , 
('3', make_pipeline(MinMaxScaler() , SGDClassifier(penalty=res.x[3], loss=res.x[4],random_state=42))),
]
clf = StackingClassifier(estimators=estimators, final_estimator=LogisticRegression(C=res.x[1],random_state=42), n_jobs=-1, cv=KFold(n_splits=5, shuffle=True, random_state=42), stack_method='predict')
 clf.fit(train_text_vect, y_train)
 print(clf.score(train_text_vect, y_train))
print(clf.score(val_text_vect, y_val))
<set_options><EOS> 
df_submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv', index_col=0 ).fillna('')
 df_submission['target'] = clf.predict(test_text_vect)
 df_submission.to_csv('submission.csv')
 df_submission