Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 train.head()
class WheatDataset(Dataset):
def __init__(self , dataframe , image_ids, transforms = None):
super().__init__()
self.image_ids = image_ids
self.dataframe = dataframe
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image, boxes = self.load_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['labels'] = torch.stack(sample['labels'])
 break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{dir[1]}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.dataframe[self.dataframe['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
area =(boxes[:, 2] * boxes[:, 3])
 boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
area = torch.as_tensor(area, dtype=torch.float32)
 return image, boxes<concatenate>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
!pip install sentencepiece
def collate_fn(batch):
return tuple(zip(*batch))<create_dataframe>
import tensorflow_hub as hub
import tokenization
from sklearn.model_selection import train_test_split
import tensorflow as tf
bs = 2
fold_number = 0
train_set = WheatDataset(dataframe=df, image_ids=df_folds[df_folds['fold'] != fold_number].index.values , transforms=transforms_train)
 valid_set = WheatDataset(dataframe=df, image_ids=df_folds[df_folds['fold'] == fold_number].index.values , transforms=transforms_valid)
 train_loader = DataLoader(train_set,batch_size = bs,sampler=RandomSampler(train_set),pin_memory=False,
drop_last=True,collate_fn=collate_fn , num_workers=4)
 valid_loader = DataLoader(valid_set,batch_size = bs ,sampler=SequentialSampler(valid_set),pin_memory=False,
drop_last=True,collate_fn=collate_fn , num_workers=4)
 images , targets , path_images = next(iter(train_loader))
img = visualizeTarget(images[0],targets[0])
 visualize(Example_one_image_from_dataloader = img )<choose_model_class>
X = np.array(train['text'])
 y = np.array(train['target'] )
class Training:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.best_calc_loss = 10**5
self.model = model
self.device = device
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 def train_loop(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
print(f'
{timestamp}
LR: {lr}')
 t = time.time()
calc_loss = self.train_one_epoch(train_loader)
 print(f'Train.Epoch: {self.epoch}, Loss: {calc_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save_model(f'{self.base_dir}/last-epoch.bin')
 t = time.time()
calc_loss = self.valid_one_epoch(validation_loader)
 print(f'Val.Epoch: {self.epoch}, Loss: {calc_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if calc_loss.avg < self.best_calc_loss:
self.best_calc_loss = calc_loss.avg
self.model.eval()
self.save_model(f'{self.base_dir}/efficient-best-{str(self.epoch ).zfill(2)}epoch.bin')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=calc_loss.avg)
 self.epoch += 1
def train_one_epoch(self, train_loader):
self.model.train()
calc_loss = Calculator()
t = time.time()
target_res = {}
for images, targets, image_ids in tqdm_notebook(train_loader):
images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
target_res['bbox'] = boxes
target_res['cls'] = labels
self.optimizer.zero_grad()
outputs = self.model(images, target_res)
 loss = outputs['loss']
calc_loss.update(loss.detach().item() , batch_size)
 loss.backward()
self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return calc_loss
def valid_one_epoch(self, val_loader):
self.model.eval()
calc_loss = Calculator()
t = time.time()
with torch.no_grad() :
for images, targets, image_ids in tqdm_notebook(val_loader):
images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
target_res = {}
target_res['bbox'] = boxes
target_res['cls'] = labels
target_res["img_scale"] = torch.tensor([1.0] * batch_size, dtype=torch.float ).to(self.device)
 target_res["img_size"] = torch.tensor([images[0].shape[-2:]] * batch_size, dtype=torch.float ).to(self.device)
 outputs = self.model(images, target_res)
 loss = outputs['loss']
calc_loss.update(loss.detach().item() , batch_size)
 return calc_loss
def save_model(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_calc_loss': self.best_calc_loss,
'epoch': self.epoch,
}, path)
 def load_model(self, path):
model = torch.load(path)
 self.model.model.load_state_dict(model['model_state_dict'])
 self.optimizer.load_state_dict(model['optimizer_state_dict'])
 self.scheduler.load_state_dict(model['scheduler_state_dict'])
 self.best_calc_loss = model['best_calc_loss']
self.epoch = model['epoch'] + 1<load_pretrained>
def bert_encode(texts,tokenizer, max_len = 512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 pad_len = max_len - len(input_sequence)
 tokens += [0]*pad_len
pad_mask = [1]*len(input_sequence)+[0]*pad_len
segment_id = [0]*max_len
all_tokens.append(tokens)
 all_masks.append(pad_mask)
 all_segments.append(segment_id)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
def get_model(num_classes = 1):
config = get_efficientdet_config('tf_efficientdet_d7x')
 model = EfficientDet(config, pretrained_backbone=False)
 checkpoint = torch.load('.. /input/efficientdetd7x/tf_efficientdet_d7x-f390b87c.pth')
 model.load_state_dict(checkpoint)
 config.num_classes = num_classes
config.image_size = 512
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
return DetBenchTrain(model, config )<init_hyperparams>
def build_model(bert_layer,max_len=512):
input_word_ids = tf.keras.layers.Input(shape=(max_len,),dtype=tf.int32,name="input_word_ids")
 input_mask = tf.keras.layers.Input(shape=(max_len,),dtype=tf.int32,name="input_mask")
 input_segment_ids = tf.keras.layers.Input(shape=(max_len,),dtype=tf.int32,name="input_segment_ids")
 _,sequence_output = bert_layer([input_word_ids,input_mask,input_segment_ids])
 clf_output = sequence_output[:,0,:]
model_X = tf.keras.layers.Dense(100,activation='relu' )(clf_output)
 model_X = tf.keras.layers.BatchNormalization()(model_X)
 model_X = tf.keras.layers.Dropout(0.5 )(model_X)
 model_X = tf.keras.layers.Dense(100,activation='relu' )(model_X)
 model_X = tf.keras.layers.BatchNormalization()(model_X)
 model_X = tf.keras.layers.Dropout(0.5 )(model_X)
 model_X = tf.keras.layers.Dense(100,activation='relu' )(model_X)
 model_X = tf.keras.layers.BatchNormalization()(model_X)
 model_X = tf.keras.layers.Dropout(0.5 )(model_X)
 out = tf.keras.layers.Dense(1,activation='sigmoid' )(model_X)
 model = tf.keras.models.Model(inputs=[input_word_ids,input_mask,input_segment_ids],outputs=out)
 return model
class GlobalParametersTrain:
lr = 0.0002
n_epochs = 20
folder = '.. /input/modelfasterrcnn'
verbose = True
verbose_step = 10
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(mode='min',factor=0.5,patience=1,verbose=False, threshold=0.0001,threshold_mode='abs',cooldown=0, min_lr=1e-8,eps=1e-08 )<choose_model_class>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
def load_model(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d7x')
 model = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 model.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
model = DetBenchPredict(model, config)
 model.eval() ;
return model.cuda()
model = load_model('.. /input/efficientdet7xbestmodel/efficient-best-11epoch.bin' )<predict_on_test>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def make_predictions(images , score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
outputs = model(images , torch.tensor([1.0] * images.shape[0], dtype=torch.float ).to(device), torch.tensor([images[0].shape[-2:]] * images.shape[0], dtype=torch.float ).to(device))
for i in range(images.shape[0]):
boxes = outputs[i].detach().cpu().numpy() [:,:4]
scores = outputs[i].detach().cpu().numpy() [:,4]
labels = outputs[i].detach().cpu().numpy() [:,5]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
'labels': labels[indexes],
})
 return [predictions]<compute_train_metric>
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2)
 train_input = bert_encode(X_train,tokenizer,max_len=264)
 val_input = bert_encode(X_test,tokenizer,max_len=264)
 test_input = bert_encode(test.text.values,tokenizer,max_len=264)
 train_labels = y_train
val_labels = y_test
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [prediction[image_index]['labels'].tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<predict_on_test>
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 es = EarlyStopping(monitor='val_loss',patience=3,verbose=1,restore_best_weights=True,min_delta=0.01)
 model.compile(optimizer=Adam(lr=1e-5),loss='binary_crossentropy',metrics=['accuracy'])
 train_history = model.fit(
 train_input, train_labels,
validation_data=(val_input,val_labels),
epochs=50,
callbacks=[checkpoint,es],
batch_size=12
)
for j,(images, targets , image_ids)in enumerate(valid_loader):
break
predictions = make_predictions(images)
 i = 0
sample = images[i].permute(1,2,0 ).cpu().numpy()
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=511 )<categorify>
y_pred = model.predict(test_input)
 ans = pd.DataFrame({'id':np.array(test['id']),'target':np.array(y_pred.round().astype(int)).reshape(-1)})
 ans.to_csv('submission.csv',index=False)
 ans