Kaggle competition: Natural Language Processing with Disaster Tweets
sys.path.insert(0, ".. /input/weightedboxesfusion")
 iou_thr = 0.44
skip_box_thr = 0.00001
pred_boxes_ensemble = []
pred_scores_ensemble = []
for(b00, b01, b02, b03, b04, b05,
b10, b11, b12, b13, b14, b15,
b20, b21, b22, b23, b24, b25,
s00, s01, s02, s03, s04, s05,
s10, s11, s12, s13, s14, s15,
s20, s21, s22, s23, s24, s25,
h0, w0)in zip(
 tqdm(bifpn0_pred_boxes_0),
bifpn0_pred_boxes_0_lr,
bifpn0_pred_boxes_0_ud,
bifpn0_pred_boxes_l,
bifpn0_pred_boxes_l_lr,
bifpn0_pred_boxes_l_ud,
bifpn1_pred_boxes_0,
bifpn1_pred_boxes_0_lr,
bifpn1_pred_boxes_0_ud,
bifpn1_pred_boxes_l,
bifpn1_pred_boxes_l_lr,
bifpn1_pred_boxes_l_ud,
bifpn3_pred_boxes_0,
bifpn3_pred_boxes_0_lr,
bifpn3_pred_boxes_0_ud,
bifpn3_pred_boxes_l,
bifpn3_pred_boxes_l_lr,
bifpn3_pred_boxes_l_ud,
bifpn0_pred_scores_0,
bifpn0_pred_scores_0_lr,
bifpn0_pred_scores_0_ud,
bifpn0_pred_scores_l,
bifpn0_pred_scores_l_lr,
bifpn0_pred_scores_l_ud,
bifpn1_pred_scores_0,
bifpn1_pred_scores_0_lr,
bifpn1_pred_scores_0_ud,
bifpn1_pred_scores_l,
bifpn1_pred_scores_l_lr,
bifpn1_pred_scores_l_ud,
bifpn3_pred_scores_0,
bifpn3_pred_scores_0_lr,
bifpn3_pred_scores_0_ud,
bifpn3_pred_scores_l,
bifpn3_pred_scores_l_lr,
bifpn3_pred_scores_l_ud,
h0_list,
w0_list):
boxes_list = [
normalize_boxes(b00, h0, w0 ).tolist() ,
normalize_boxes(b01, h0, w0 ).tolist() ,
normalize_boxes(b02, h0, w0 ).tolist() ,
normalize_boxes(b03, h0, w0 ).tolist() ,
normalize_boxes(b04, h0, w0 ).tolist() ,
normalize_boxes(b05, h0, w0 ).tolist() ,
normalize_boxes(b10, h0, w0 ).tolist() ,
normalize_boxes(b11, h0, w0 ).tolist() ,
normalize_boxes(b12, h0, w0 ).tolist() ,
normalize_boxes(b13, h0, w0 ).tolist() ,
normalize_boxes(b14, h0, w0 ).tolist() ,
normalize_boxes(b15, h0, w0 ).tolist() ,
normalize_boxes(b20, h0, w0 ).tolist() ,
normalize_boxes(b21, h0, w0 ).tolist() ,
normalize_boxes(b22, h0, w0 ).tolist() ,
normalize_boxes(b23, h0, w0 ).tolist() ,
normalize_boxes(b24, h0, w0 ).tolist() ,
normalize_boxes(b25, h0, w0 ).tolist()
]
scores_list = [
s00.tolist() ,
s01.tolist() ,
s02.tolist() ,
s03.tolist() ,
s04.tolist() ,
s05.tolist() ,
s10.tolist() ,
s11.tolist() ,
s12.tolist() ,
s13.tolist() ,
s14.tolist() ,
s15.tolist() ,
s20.tolist() ,
s21.tolist() ,
s22.tolist() ,
s23.tolist() ,
s24.tolist() ,
s25.tolist()
]
labels_list = [
[0] * len(b00),
[0] * len(b01),
[0] * len(b02),
[0] * len(b03),
[0] * len(b04),
[0] * len(b05),
[0] * len(b10),
[0] * len(b11),
[0] * len(b12),
[0] * len(b13),
[0] * len(b14),
[0] * len(b15),
[0] * len(b20),
[0] * len(b21),
[0] * len(b22),
[0] * len(b23),
[0] * len(b24),
[0] * len(b25)
 ]
boxes, scores, _ = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes_list, scores_list, labels_list, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes_ensemble.append(boxes)
 pred_scores_ensemble.append(scores )<concatenate>
!pip install tweet_preprocessor
!pip install datasets
pred_boxes_ensemble = [denormalize_clip_boxes(a, h0, w0)for a, h0, w0 in zip(pred_boxes_ensemble, h0_list, w0_list)]
pred_scores_ensemble = [a for a in pred_scores_ensemble]<categorify>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
data_dict_pseudo = []
id_generator = 20201000000
for idx,(bboxes, h0, w0)in enumerate(zip(tqdm(pred_boxes_ensemble), h0_list, w0_list)) :
img_dict = {
'file_name': os.path.join(DIR_TEST, testdataset[idx][1]),
'height': h0,
'width': w0,
'id': id_generator,
}
annotations = []
for bbox in bboxes:
xywh = np.round([bbox[0], bbox[1], bbox[2]-bbox[0], bbox[3]-bbox[1]] ).astype(float ).tolist()
annotations.append({
'area': xywh[2] * xywh[3],
'bbox': xywh,
'category_id': 0,
'bbox_mode': 1
})
 img_dict['annotations'] = annotations
id_generator += 1
data_dict_pseudo.append(img_dict )<define_variables>
sample_submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 sample_submission.head(20 )
data_dict_pseudo = [d for d in data_dict_pseudo if len(d['annotations'])> 0]<init_hyperparams>
train.dropna()
class TrainConfig:
seed = 2519
arch = 'timm-efficientnet-b5'
heads = {
'hm': 1,
'wh': 2,
'reg': 2}
head_conv = 64
reg_offset = True
data_root = '.. /input/global-wheat-detection'
crop_size = 896
scale = 0.
 shift = 0.
 rotate = 15.
 shear = 5.
 down_ratio = 4
debug = False
hm_weight = 1
off_weight = 1
wh_weight = 0.1
batch_size = 4
base_lr = 0.25e-4
warmup_iters = 0
total_epochs = 9
stage_epochs = 9
freeze_bn = True
accumulate = 3
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 ema = False
amp = True
output_dir = './'
logs_dir = os.path.join(output_dir, 'logs')
 log_interval = 10
checkpoint = 1
load_model = '.. /input/wheat-weights/model_centernet_effnetb5_bifpn_00099.pth'
resume = ''
train_opt = TrainConfig()<load_from_disk>
print('train positive samples: %d' % train[train['target'] == 1].shape[0])
 print('train negative samples: %d' % train[train['target'] == 0].shape[0] )
with open('.. /input/wheat-splits/wheat_train_3.json', 'r')as f:
data_dict_train = json.load(f)
 with open('.. /input/wheat-splits/wheat_valid_3.json', 'r')as f:
data_dict_valid = json.load(f )<create_dataframe>
def preprocess(text):
text = text.replace("
text = p.clean(text)
 return text
train['text'] = train['text'].apply(lambda x: preprocess(x))
test['text'] = test['text'].apply(lambda x: preprocess(x))
print(train['text'].values.tolist() [:5])
 print(test['text'].values.tolist() [:5] )
def main(opt):
set_seed(opt.seed)
 torch.backends.cudnn.benchmark = True
create_logging(opt.logs_dir, 'w')
 train_dataset = WheatDataset(
 opt,
opt.data_root,
data_dict_train,
data_dict_pseudo=data_dict_pseudo,
img_size=1024,
transforms=get_train_transforms(opt.crop_size),
is_train=True,
load_to_ram=False)
 logging.info('{} images in training set'.format(len(train_dataset.data_dict)))
 train_loader = FastDataLoader(
 train_dataset,
opt.batch_size,
collate_fn=collate,
shuffle=True,
drop_last=True,
pin_memory=True,
num_workers=2)
 model = ModleWithLoss(PoseBiFPNNet(opt.arch, opt.heads, opt.head_conv), CtdetLoss(opt)).to(opt.device)
 if opt.freeze_bn:
model.apply(freeze_bn)
 if opt.ema:
logging.info('Training with EMA')
 ema = ModelEMA(model)
 else:
ema = None
optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=opt.base_lr)
 num_training_steps = int(opt.total_epochs * len(train_dataset)/ opt.batch_size / opt.accumulate)
 scheduler = get_constant_schedule_with_warmup(optimizer, opt.warmup_iters)
 current_epoch = 0
if opt.load_model != '':
checkpoint = torch.load(opt.load_model)
 model.load_state_dict(checkpoint['model'])
 elif opt.resume != '':
checkpoint = torch.load(opt.resume)
 model.load_state_dict(checkpoint['model'])
 optimizer.load_state_dict(checkpoint['optimizer'])
 scheduler.load_state_dict(checkpoint['scheduler'])
 current_epoch = checkpoint['epoch'] + 1
scaler = GradScaler() if opt.amp else None
for epoch in tqdm(range(current_epoch, current_epoch + opt.stage_epochs)) :
epoch_start_time = time.time()
train_one_epoch(opt, model, optimizer, scheduler, train_loader, epoch, ema=ema, scaler=scaler)
 logging.info('-' * 89)
 logging.info('end of epoch {:4d} | time: {:5.2f}s |'.format(epoch,(time.time() - epoch_start_time)))
 logging.info('-' * 89)
 return model.state_dict()<load_pretrained>
X_train, X_val, y_train, y_val = train_test_split(train['text'], train['target'], test_size=.1, random_state=42)
 X_test = test['text'].values.tolist()
X_train = X_train.tolist()
X_val = X_val.tolist()
y_train = y_train.tolist()
y_val = y_val.tolist()
gc.collect()
torch.cuda.empty_cache()
pseudo_model = PoseBiFPNNet(train_opt.arch, train_opt.heads, train_opt.head_conv)
 if len(os.listdir(DIR_TEST)) < 20:
train_opt.total_epochs = 2
train_opt.stage_epochs = 2
data_dict_train = data_dict_train[:300]
state_dict = main(train_opt)
 change_key(state_dict)
 pseudo_model.load_state_dict(state_dict)
 pseudo_model.to(device)
 del state_dict
gc.collect()
else:
state_dict = main(train_opt)
 change_key(state_dict)
 pseudo_model.load_state_dict(state_dict)
 pseudo_model.to(device)
 del state_dict
gc.collect()<predict_on_test>
model = AutoModelForSequenceClassification.from_pretrained('vinai/bertweet-large')
 tokenizer = AutoTokenizer.from_pretrained("vinai/bertweet-large", use_fast=False )
opt.pad = 63
opt.test_scales = [1.1, ]
threshold = 0.32
pseudo_pred_boxes_0 , pseudo_pred_scores_0, h0_list, w0_list, img_ids = do_predict(opt, pseudo_model, threshold=threshold, flip_type=0, return_ids=True, return_shapes=True)
 pseudo_pred_boxes_0_lr, pseudo_pred_scores_0_lr = do_predict(opt, pseudo_model, threshold=threshold, flip_type=1, return_ids=False, return_shapes=False)
 pseudo_pred_boxes_0_ud, pseudo_pred_scores_0_ud = do_predict(opt, pseudo_model, threshold=threshold, flip_type=2, return_ids=False, return_shapes=False)
 opt.test_scales = [1.25, ]
threshold = 0.30
pseudo_pred_boxes_l , pseudo_pred_scores_l = do_predict(opt, pseudo_model, threshold=threshold, flip_type=0, return_ids=False, return_shapes=False)
 pseudo_pred_boxes_l_lr, pseudo_pred_scores_l_lr = do_predict(opt, pseudo_model, threshold=threshold, flip_type=1, return_ids=False, return_shapes=False)
 pseudo_pred_boxes_l_ud, pseudo_pred_scores_l_ud = do_predict(opt, pseudo_model, threshold=threshold, flip_type=2, return_ids=False, return_shapes=False )<concatenate>
train_encodings = tokenizer(X_train, truncation=True, padding=True)
 val_encodings = tokenizer(X_val, truncation=True, padding=True )
pred_boxes_pseudo = [denormalize_clip_boxes(a, h0, w0)for a, h0, w0 in zip(pred_boxes_pseudo, h0_list, w0_list)]
pred_scores_pseudo = [a for a in pred_scores_pseudo]<categorify>
class DisasterDataset(torch.utils.data.Dataset):
def __init__(self, encodings, labels):
self.encodings = encodings
self.labels = labels
def __getitem__(self, idx):
item = {key: torch.tensor(val[idx])for key, val in self.encodings.items() }
item['labels'] = torch.tensor(self.labels[idx])
 return item
def __len__(self):
return len(self.labels)
 train_dataset = DisasterDataset(train_encodings, y_train)
 val_dataset = DisasterDataset(val_encodings, y_val )
def format_prediction_string(boxes, scores):
pred_strings = []
for s, b in zip(scores, boxes.astype(int)) :
pred_strings.append(f'{s:.4f} {b[0]} {b[1]} {b[2]} {b[3]}')
 return " ".join(pred_strings )<compute_test_metric>
acc = load_metric('accuracy')
 precision = load_metric('precision')
 recall = load_metric('recall')
 f1 = load_metric('f1')
 def compute_metrics(eval_pred):
predictions, labels = eval_pred
predictions = np.argmax(predictions, axis=1)
 acc_result = acc.compute(predictions=predictions, references=labels)
 precision_result = precision.compute(predictions=predictions, references=labels)
 recall_result = recall.compute(predictions=predictions, references=labels)
 f1_result = f1.compute(predictions=predictions, references=labels)
 result = {**acc_result, **precision_result, **recall_result, **f1_result}
return result
training_args = TrainingArguments(
 output_dir='./results',
num_train_epochs=2,
per_device_train_batch_size=64,
per_device_eval_batch_size=128,
warmup_steps=500,
weight_decay=0.01,
logging_dir='./logs',
logging_steps=20,
evaluation_strategy='steps' 
)
 trainer = Trainer(
 model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=val_dataset,
compute_metrics=compute_metrics 
)
 trainer.train()
pred_strs = []
for bboxes, scores in zip(pred_boxes_pseudo, pred_scores_pseudo):
if len(bboxes)> 0:
bboxes[:, 2] -= bboxes[:, 0]
bboxes[:, 3] -= bboxes[:, 1]
bboxes = bboxes.round()
pred_strs.append(format_prediction_string(bboxes, scores))
else:
pred_strs.append('' )<create_dataframe>
nlp=pipeline("sentiment-analysis", model=model.to('cpu'), tokenizer=tokenizer)
 test_preds = []
for index, text in enumerate(test['text'].values.tolist()):
if index % 10 == 0:
print(index)
 if nlp(text)[0]['label'] == 'LABEL_0':
test_preds.append(0)
 else:
test_preds.append(1)
 test['target'] = test_preds
<save_to_csv><EOS> 
submissions = test.drop(labels = ["keyword", "location", "text"], axis = 1)
 submissions.to_csv("submissions.csv", index = False )