Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<import_modules>
print(tf.__version__ )
set_seed,
create_logging,
WheatDataset,
FastDataLoader,
collate,
ModleWithLoss,
CtdetLoss,
ModelEMA,
get_constant_schedule_with_warmup,
train_one_epoch,
get_train_transforms,
freeze_bn
)<define_variables>
df = pd.read_csv('.. /input/disaster-tweets-cleaned/df.csv')
 test_df = pd.read_csv('.. /input/disaster-tweets-cleaned/test_df.csv')
 print(df.shape, test_df.shape )
bifpn_path_0 = '.. /input/wheat-weights/model_centernet_effnetb5_bifpn_00099.pth'
bifpn_path_1 = '.. /input/wheat-weights/model_centernet_effnetb5_bifpn_fold1_00099.pth'
bifpn_path_3 = '.. /input/wheat-weights/model_centernet_effnetb5_bifpn_fold3_lb_ema_00099.pth'<init_hyperparams>
train_txts, val_txts, y_train, y_val = train_test_split(
 df[col].values, df['target'].values,
shuffle = True, test_size = 0.15,
stratify = df['target'].values, 
)
 test_txts = test_df[col].values
y_test = test_df['target'].values
print('Train size:', train_txts.shape)
 print('Validation size:', val_txts.shape)
 print('Test size:', test_txts.shape )
class Config:
arch = 'timm-efficientnet-b5'
heads = {'hm': 1,
'wh': 2,
'reg': 2}
head_conv = 64
reg_offset = True
cat_spec_wh = False
img_size = 1024
in_scale = 1024 / img_size
down_ratio = 4
mean = [0.315290, 0.317253, 0.214556],
std = [0.245211, 0.238036, 0.193879]
num_classes = 1
pad = 63
batch_size = 8
K = 128
max_per_image = 128
fix_res = False
test_scales = [1]
flip_test = False
nms = False
gpus = [0]
amp = True
opt = Config()
device = torch.device('cuda')if opt.gpus[0] >= 0 else torch.device('cpu' )<feature_engineering>
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased' )
def change_key(d):
for _ in range(len(d)) :
k, v = d.popitem(False)
 d['.'.join(k.split('.')[1:])] = v<load_from_csv>
def tokenize_txts(txts, max_len = 40):
res = tokenizer(
 text = [tokenizer.tokenize(txt)for txt in txts],
max_length = max_len,
padding = 'max_length',
truncation = True,
is_split_into_words = True, 
)
 return {
'input_word_ids': res['input_ids'],
'input_mask': res['attention_mask'],
'input_type_ids': res['token_type_ids'],
}
def detokenize_txt(token):
res = tokenizer.decode(token)
 return res
DIR_INPUT = '.. /input/global-wheat-detection'
DIR_TRAIN = f'{DIR_INPUT}/train'
DIR_TEST = f'{DIR_INPUT}/test'
train_df = pd.read_csv(f'{DIR_INPUT}/train.csv')
 train_df.shape<data_type_conversions>
MAX_LEN = 35
BATCH_SIZE = 32
train_df['x'] = -1
train_df['y'] = -1
train_df['w'] = -1
train_df['h'] = -1
def expand_bbox(x):
r = np.array(re.findall("([0-9]+[.]?[0-9]*)", x))
if len(r)== 0:
r = [-1, -1, -1, -1]
return r
train_df[['x', 'y', 'w', 'h']] = np.stack(train_df['bbox'].apply(lambda x: expand_bbox(x)))
 train_df.drop(columns=['bbox'], inplace=True)
 train_df['x'] = train_df['x'].astype(np.float)
 train_df['y'] = train_df['y'].astype(np.float)
 train_df['w'] = train_df['w'].astype(np.float)
 train_df['h'] = train_df['h'].astype(np.float)
 train_df.head()<categorify>
train_tokens = tokenize_txts(train_txts, MAX_LEN)
 val_tokens = tokenize_txts(val_txts, MAX_LEN)
 test_tokens = tokenize_txts(test_txts, MAX_LEN)
class WheatDatasetTest(torch.utils.data.Dataset):
def __init__(self, opt, image_dir, transforms=None,
mean=[0.315290, 0.317253, 0.214556],
std=[0.245211, 0.238036, 0.193879]):
self.opt = opt
self.image_dir = image_dir
self.img_id = os.listdir(self.image_dir)
 self.transforms = transforms
self.mean = np.array(mean, dtype=np.float32 ).reshape(1, 1, 3)
 self.std = np.array(std, dtype=np.float32 ).reshape(1, 1, 3)
 def __len__(self):
return len(self.img_id)
 def __getitem__(self, idx):
img_path = os.path.join(self.image_dir, self.img_id[idx])
 image = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB)
 h0, w0 = image.shape[0:2]
if self.transforms is not None:
image = self.transforms(image)
 image_resized = cv2.resize(image,(self.opt.img_size, self.opt.img_size))
return image_resized, self.img_id[idx], image, h0, w0<concatenate>
print('Orginal txt: ', train_txts[0])
 print()
sample = train_tokens['input_word_ids'][0]
print('Tokenized txt:', sample)
 print()
print('Detokenizd txt:', detokenize_txt(sample))
def flip_lr(img):
return np.ascontiguousarray(img[:, ::-1, :])
 def deaug_lr(img, boxes):
h, w = img.shape[:2]
boxes[:,(0, 2)] = w - boxes[:,(2, 0)]
return boxes
def flip_ud(img):
return np.ascontiguousarray(img[::-1, :, :])
 def deaug_ud(img, boxes):
h, w = img.shape[:2]
boxes[:,(1, 3)] = w - boxes[:,(3, 1)]
return boxes<create_dataframe>
train_ds = tf.data.Dataset.from_tensor_slices(( train_tokens, y_train))
val_ds = tf.data.Dataset.from_tensor_slices(( val_tokens, y_val))
test_ds = tf.data.Dataset.from_tensor_slices(( test_tokens, y_test))
train_ds = train_ds.batch(BATCH_SIZE)
 val_ds = val_ds.batch(BATCH_SIZE)
 test_ds = test_ds.batch(BATCH_SIZE )
testdataset = WheatDatasetTest(opt, DIR_TEST)
 print('Total number of images in test set: {}'.format(len(testdataset)))
 testdataset_lr = WheatDatasetTest(opt, DIR_TEST, transforms=flip_lr)
 testdataset_ud = WheatDatasetTest(opt, DIR_TEST, transforms=flip_ud )<find_best_model_class>
class MyF1(tf.keras.metrics.Metric):
def __init__(self, name = 'mf1_score'):
super(MyF1, self ).__init__(name)
 self.p = tf.metrics.Precision()
self.r = tf.metrics.Recall()
self.f1 = self.add_weight(name="f1", initializer="zeros")
 def update_state(self, actual, predicted, sample_weight = None):
self.p.update_state(actual, predicted)
 self.r.update_state(actual, predicted)
 self.f1.assign(2 * self.p.result() * self.r.result() /(self.p.result() + self.r.result()))
def reset_states(self,):
self.p.reset_states()
self.r.reset_states()
self.f1.assign(0.0)
 def result(self,):
return self.f1
def do_predict(opt, model, threshold, flip_type=0, return_ids=False, return_shapes=False):
if flip_type == 0:
test_dataset = testdataset
deaug_transform = None
elif flip_type == 1:
test_dataset = testdataset_lr
deaug_transform = deaug_lr
elif flip_type == 2:
test_dataset = testdataset_ud
deaug_transform = deaug_ud
detector = CtdetDetector(opt, model)
 pred_boxes = []
pred_scores = []
height_list = []
width_list = []
if return_ids:
img_ids = []
for img, img_id, img0, h0, w0 in tqdm(test_dataset):
ret = detector.run(img)
 results = ret['results'][1]
results = results[results[:, 4] > threshold]
pred_box = results[:, :4]
if flip_type != 0:
pred_box = deaug_transform(img, pred_box)
 pred_box[:, 0] = np.clip(pred_box[:, 0] / opt.img_size * w0, 0, w0-1)
 pred_box[:, 1] = np.clip(pred_box[:, 1] / opt.img_size * h0, 0 ,h0-1)
 pred_box[:, 2] = np.clip(pred_box[:, 2] / opt.img_size * w0, 0, w0-1)
 pred_box[:, 3] = np.clip(pred_box[:, 3] / opt.img_size * h0, 0 ,h0-1)
 pred_boxes.append(pred_box)
 pred_scores.append(results[:, 4])
 if return_ids:
img_ids.append(os.path.splitext(img_id)[0])
 if return_shapes:
height_list.append(h0)
 width_list.append(w0)
 if return_shapes:
if return_ids:
return pred_boxes, pred_scores, height_list, width_list, img_ids
else:
return pred_boxes, pred_scores, height_list, width_list
else:
if return_ids:
return pred_boxes, pred_scores, img_ids
else:
return pred_boxes, pred_scores<load_pretrained>
class CSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
def __init__(self, lr, freeze_epoch ,batch_size, data_size):
super(CSchedule, self ).__init__()
self.lr = lr
self.bs = batch_size
self.ds = data_size
self.freeze_epoch = freeze_epoch
def __call__(self, step):
epoch = step /(self.ds / self.bs)+ 1
if not self.freeze_epoch or epoch < self.freeze_epoch:
return self.lr /(tf.cast(epoch, tf.float32)+1)
 else:
return self.lr * 10 /(tf.cast(epoch, tf.float32)+1)
 clr = CSchedule(2e-5, 3, BATCH_SIZE, len(train_txts))
plt.plot([clr(x)for x in tf.range(1, 1457, dtype=tf.float32)])
 plt.xlabel('steps')
 plt.ylabel('learning rate')
 plt.title('Learning Rate Schedule')
 plt.show()
bifpn_model = PoseBiFPNNet(opt.arch, opt.heads, opt.head_conv)
 checkpoint = torch.load(bifpn_path_0, map_location=device)
 change_key(checkpoint['model'])
 bifpn_model.load_state_dict(checkpoint['model'])
 bifpn_model.to(device)
 del checkpoint
gc.collect()<predict_on_test>
bert_handler = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3'
opt.pad = 63
opt.test_scales = [1.1, ]
threshold = 0.30
bifpn0_pred_boxes_0 , bifpn0_pred_scores_0, h0_list, w0_list, img_ids = do_predict(opt, bifpn_model, threshold=threshold, flip_type=0, return_ids=True, return_shapes=True)
 bifpn0_pred_boxes_0_lr, bifpn0_pred_scores_0_lr = do_predict(opt, bifpn_model, threshold=threshold, flip_type=1, return_ids=False, return_shapes=False)
 bifpn0_pred_boxes_0_ud, bifpn0_pred_scores_0_ud = do_predict(opt, bifpn_model, threshold=threshold, flip_type=2, return_ids=False, return_shapes=False)
 opt.test_scales = [1.25, ]
threshold = 0.28
bifpn0_pred_boxes_l , bifpn0_pred_scores_l = do_predict(opt, bifpn_model, threshold=threshold, flip_type=0, return_ids=False, return_shapes=False)
 bifpn0_pred_boxes_l_lr, bifpn0_pred_scores_l_lr = do_predict(opt, bifpn_model, threshold=threshold, flip_type=1, return_ids=False, return_shapes=False)
 bifpn0_pred_boxes_l_ud, bifpn0_pred_scores_l_ud = do_predict(opt, bifpn_model, threshold=threshold, flip_type=2, return_ids=False, return_shapes=False )<set_options>
class MClassifier(tf.keras.Model):
def __init__(self, dropout_rate):
super(MClassifier, self ).__init__()
self.bert_layer = hub.KerasLayer(
 bert_handler,
name = 'feature_ext',
trainable = True, 
)
 self.dropout = tf.keras.layers.Dropout(dropout_rate)
 self.proba = tf.keras.layers.Dense(1, activation = 'sigmoid')
 def call(self, X_in, training):
x = self.bert_layer(X_in, training = training)
 x = x['pooled_output']
x = self.dropout(x, training = training)
 output = self.proba(x)
 return output
del bifpn_model
gc.collect()
torch.cuda.empty_cache()<load_pretrained>
LEARNING_RATE = 2e-5
EPOCHS = 5
DP_RATE = 0.3
loss_objective = tf.keras.losses.BinaryCrossentropy()
bifpn_model = PoseBiFPNNet(opt.arch, opt.heads, opt.head_conv)
 checkpoint = torch.load(bifpn_path_1, map_location=device)
 change_key(checkpoint['model'])
 bifpn_model.load_state_dict(checkpoint['model'])
 bifpn_model.to(device)
 del checkpoint
gc.collect()<predict_on_test>
def model_evaluation(model, ds, name):
acc = tf.keras.metrics.BinaryAccuracy()
f1 = MyF1()
total_loss = []
y_hats = []
for X, y in ds:
y_hat = model(X, training = False)
 loss = loss_objective(y, y_hat)
 acc.update_state(y, y_hat)
 f1.update_state(y, y_hat)
 total_loss.append(loss.numpy())
 y_hats.append(y_hat)
 y_hats = tf.concat(y_hats, axis = 0 ).numpy()
print(f'{name}, loss {np.mean(total_loss):.3f}, acc {acc.result().numpy() :.3f}, f1 {f1.result().numpy() :.3f}')
 return y_hats, acc.result().numpy() , f1.result().numpy()
opt.pad = 63
opt.test_scales = [1.1, ]
threshold = 0.30
bifpn1_pred_boxes_0 , bifpn1_pred_scores_0 = do_predict(opt, bifpn_model, threshold=threshold, flip_type=0, return_ids=False, return_shapes=False)
 bifpn1_pred_boxes_0_lr, bifpn1_pred_scores_0_lr = do_predict(opt, bifpn_model, threshold=threshold, flip_type=1, return_ids=False, return_shapes=False)
 bifpn1_pred_boxes_0_ud, bifpn1_pred_scores_0_ud = do_predict(opt, bifpn_model, threshold=threshold, flip_type=2, return_ids=False, return_shapes=False)
 opt.test_scales = [1.25, ]
threshold = 0.28
bifpn1_pred_boxes_l , bifpn1_pred_scores_l = do_predict(opt, bifpn_model, threshold=threshold, flip_type=0, return_ids=False, return_shapes=False)
 bifpn1_pred_boxes_l_lr, bifpn1_pred_scores_l_lr = do_predict(opt, bifpn_model, threshold=threshold, flip_type=1, return_ids=False, return_shapes=False)
 bifpn1_pred_boxes_l_ud, bifpn1_pred_scores_l_ud = do_predict(opt, bifpn_model, threshold=threshold, flip_type=2, return_ids=False, return_shapes=False )<set_options>
@tf.function
def train_step(model, tr_vars, X, y):
with tf.GradientTape() as tape:
y_hat = model(X, training = True)
 loss = loss_objective(y, y_hat)
 grads = tape.gradient(loss, tr_vars)
 return loss, grads, y_hat
def train_model(model, epochs, freeze_bert_on_epoch = None):
clr = CSchedule(LEARNING_RATE, freeze_bert_on_epoch, BATCH_SIZE, len(train_txts))
optimizer = tf.keras.optimizers.Adam(clr)
 acc = tf.keras.metrics.BinaryAccuracy()
f1 = MyF1()
for epoch in range(1, epochs + 1):
if freeze_bert_on_epoch is not None and epoch == freeze_bert_on_epoch:
model.get_layer('feature_ext' ).trainable = False
acc.reset_states()
f1.reset_states()
total_loss = []
for X, y in train_ds:
tr_vars = model.trainable_variables
loss, grads, y_hat = train_step(model, tr_vars, X, y)
 optimizer.apply_gradients(zip(grads, tr_vars))
acc.update_state(y, y_hat)
 f1.update_state(y, y_hat)
 total_loss.append(loss.numpy())
 print(f'epoch {epoch}, loss {np.mean(total_loss):.3f}, acc {acc.result().numpy() :.3f}, f1 {f1.result().numpy() :.3f}')
 model_evaluation(model, val_ds, 'val')
 print()
y_test_hat, _, _ = model_evaluation(model, test_ds, 'test')
 return y_test_hat
del bifpn_model
gc.collect()
torch.cuda.empty_cache()<load_pretrained>
model = MClassifier(DP_RATE,)
 y_test_hat = train_model(model, EPOCHS, freeze_bert_on_epoch = 3 )
bifpn_model = PoseBiFPNNet(opt.arch, opt.heads, opt.head_conv)
 checkpoint = torch.load(bifpn_path_3, map_location=device)
 change_key(checkpoint['model'])
 bifpn_model.load_state_dict(checkpoint['model'])
 bifpn_model.to(device)
 del checkpoint
gc.collect()<predict_on_test>
y_model_hat = np.array([1 if x[0] >0.5 else 0 for x in y_test_hat])
 print(classification_report(y_test, y_model_hat))
opt.pad = 63
opt.test_scales = [1.1, ]
threshold = 0.30
bifpn3_pred_boxes_0 , bifpn3_pred_scores_0 = do_predict(opt, bifpn_model, threshold=threshold, flip_type=0, return_ids=False, return_shapes=False)
 bifpn3_pred_boxes_0_lr, bifpn3_pred_scores_0_lr = do_predict(opt, bifpn_model, threshold=threshold, flip_type=1, return_ids=False, return_shapes=False)
 bifpn3_pred_boxes_0_ud, bifpn3_pred_scores_0_ud = do_predict(opt, bifpn_model, threshold=threshold, flip_type=2, return_ids=False, return_shapes=False)
 opt.test_scales = [1.25, ]
threshold = 0.28
bifpn3_pred_boxes_l , bifpn3_pred_scores_l = do_predict(opt, bifpn_model, threshold=threshold, flip_type=0, return_ids=False, return_shapes=False)
 bifpn3_pred_boxes_l_lr, bifpn3_pred_scores_l_lr = do_predict(opt, bifpn_model, threshold=threshold, flip_type=1, return_ids=False, return_shapes=False)
 bifpn3_pred_boxes_l_ud, bifpn3_pred_scores_l_ud = do_predict(opt, bifpn_model, threshold=threshold, flip_type=2, return_ids=False, return_shapes=False )<set_options>
def train_gbm_cls(X_tr, y_tr, X_val, y_val, X_test, y_test):
gbm_cls = LGBMClassifier(
 objective = 'binary',
class_weight = 'balanced' 
)
 gbm_cls.fit(
 X_tr, y_tr,
eval_set =(X_val, y_val),
early_stopping_rounds = 20,
verbose = 0, 
)
 print('Train')
 print(classification_report(y_train, gbm_cls.predict(X_tr)))
 print('Validation')
 print(classification_report(y_val, gbm_cls.predict(X_val)))
 print('Test')
 gbm_y_hat = gbm_cls.predict(X_test)
 print(classification_report(y_test, gbm_y_hat))
return gbm_cls, gbm_y_hat
train_feat = tr_mean_max
val_feat = val_mean_max
test_feat = test_mean_max
gbm_cls, gbm_y_hat = train_gbm_cls(train_feat, y_train, val_feat, y_val , test_feat, y_test )
del bifpn_model
gc.collect()
torch.cuda.empty_cache()<categorify>
sub = pd.DataFrame(columns = ['id', 'target'])
 sub['id'] = test_df.id
sub['target'] = gbm_y_hat
def normalize_boxes(boxes, h0, w0):
boxes[:, 0] = boxes[:, 0] / w0
boxes[:, 1] = boxes[:, 1] / h0
boxes[:, 2] = boxes[:, 2] / w0
boxes[:, 3] = boxes[:, 3] / h0
return boxes
def denormalize_clip_boxes(boxes, h0, w0):
boxes[:, 0] = np.clip(boxes[:, 0] * w0, 0, w0-1)
 boxes[:, 1] = np.clip(boxes[:, 1] * h0, 0, h0-1)
 boxes[:, 2] = np.clip(boxes[:, 2] * w0, 0, w0-1)
 boxes[:, 3] = np.clip(boxes[:, 3] * h0, 0, h0-1)
 return boxes<define_variables>
nsub = pd.DataFrame(columns = ['id', 'target'])
 nsub['id'] = test_df.id
nsub['target'] = y_model_hat
nsub.to_csv('submission.csv', index = False )