Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<set_options>
import numpy as np
import pandas as pd
import tensorflow as tf
import csv
import re
%%writefile./modeling/wheat_detector.py
class WheatDetector(nn.Module):
def __init__(self, cfg, **kwargs):
super(WheatDetector, self ).__init__()
self.backbone = resnest_fpn_backbone(pretrained=False)
 self.base = FasterRCNN(self.backbone, num_classes=cfg.MODEL.NUM_CLASSES, **kwargs)
 def forward(self, images, targets=None):
return self.base(images, targets )<set_options>
def convert_test_attributes_to_onehot(df, value_list):
one_hots = np.zeros(( len(df), len(value_list)))
 for index, row in df.iterrows() :
if row['keyword'] != '':
try:
one_hots[index, value_list.index(row['keyword'])] = 1
except ValueError:
continue
if row['keyword'] != '':
try:
one_hots[index, value_list.index(row['location'])] = 1
except ValueError:
continue
return one_hots
def convert_attributes_to_onehot(df):
one_hot_keywords = pd.get_dummies(train_data['keyword'])
 one_hot_locations = pd.get_dummies(train_data['location'])
 return pd.concat([one_hot_keywords, one_hot_locations.reindex(one_hot_keywords.index)], axis=1)
 def vectorize_tweets(dataframe, df_glove):
vectorized_data = np.zeros(( len(dataframe), MAX_TWEET_LENGTH, VECTORS_PER_WORD))
for index, row in dataframe.iterrows() :
i = 0
for word in re.sub(r'\W+', ' ', row['text'].lower() ).split(' '):
try:
vectorized_data[index, i, :] = df_glove.loc[word]
except KeyError:
continue
i += 1
if index % 100 == 99:
print('Processing text number ' + str(index + 1)+ ' of ' + str(len(dataframe)))
 return vectorized_data
sys.path.insert(0, "./external/wbf")
 warnings.filterwarnings("ignore")
 class BaseWheatTTA:

 image_size = IMG_SIZE
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAReduceSaturation(BaseWheatTTA):

 def augment(self, image):
image = TF.adjust_saturation(image, 0.9)
 return image
def batch_augment(self, images):
images_new = []
for image in images:
images_new.append(self.augment(image))
return torch.stack(images_new)
 def deaugment_boxes(self, boxes):
return boxes
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1, 3]] = self.image_size - boxes[:, [3, 1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0, 2]] = self.image_size - boxes[:, [2, 0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0, 2]] = self.image_size - boxes[:, [3, 1]]
res_boxes[:, [1, 3]] = boxes[:, [0, 2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:, 0] = np.min(boxes[:, [0, 2]], axis=1)
 result_boxes[:, 2] = np.max(boxes[:, [0, 2]], axis=1)
 result_boxes[:, 1] = np.min(boxes[:, [1, 3]], axis=1)
 result_boxes[:, 3] = np.max(boxes[:, [1, 3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<define_variables>
def build_model(lstm_shape, dense_shape):
dropout = 0.8
dense_input = tf.keras.layers.Input(shape=(dense_shape))
dense1 = tf.keras.layers.Dense(50 )(dense_input)
 lstm_input = tf.keras.layers.Input(shape=(lstm_shape))
lstm1 = tf.keras.layers.GaussianNoise(0.075 )(lstm_input)
 lstm2 = tf.keras.layers.LSTM(units=500, return_sequences=True )(lstm1)
 lstm3 = tf.keras.layers.Dropout(dropout )(lstm2)
 lstm4 = tf.keras.layers.LSTM(units=100, return_sequences=True )(lstm3)
 lstm5 = tf.keras.layers.Flatten()(lstm4)
 lstm6 = tf.keras.layers.Dense(20 )(lstm5)
 concat1 = tf.keras.layers.Concatenate()([dense1, lstm5])
 concat2 = tf.keras.layers.Dropout(dropout )(concat1)
 concat3 = tf.keras.layers.Dense(80 )(concat2)
 outputs = tf.keras.layers.Dense(1, activation='sigmoid' )(concat3)
 model = tf.keras.Model(inputs=[dense_input, lstm_input], outputs=outputs)
 model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.00005, beta_1=0.9, beta_2=0.998,),
loss=tf.keras.losses.BinaryCrossentropy() ,
metrics=['acc'])
 return model
class Tester:
def __init__(self, models, device, cfg, test_loader, n_viz=N_VIZ):
self.config = cfg
self.test_loader = test_loader
self.base_dir = f'{self.config.OUTPUT_DIR}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.score_threshold = SCORE_THRESHOLD
self.iou_threshold = NMS_IOU_THRESHOLD
self.use_nms = USE_NMS
self.n_viz = n_viz
self.device = device
self.models = []
for i,model in enumerate(models):
self.models.append(model)
 self.models[-1].eval()
self.models[-1].to(self.device)
 self.log(f'Tester prepared.Device is {self.device}')
 def test(self, pp_shrink=0):
results,all_predictions = self.infer(pp_shrink)
 self.save_predictions(results)
 return all_predictions
def process_det(self, index, outputs):
boxes = outputs[index]['boxes'].data.cpu().numpy()
scores = outputs[index]['scores'].data.cpu().numpy()
boxes =(boxes ).clip(min=0, max=1023 ).astype(int)
 indexes = np.where(scores > self.score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores
def make_tta_predictions(self, tta_transforms, images, model_id):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
outputs = self.models[model_id](tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
indexes = np.where(scores > self.score_threshold)[0]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 if self.use_nms:
labels = np.ones(scores.shape[0] ).astype(int ).tolist()
boxes, scores, labels = ensemble_boxes.ensemble_boxes_nms.nms_method([boxes], [scores], [labels], method=3,
weights=None, iou_thr=self.iou_threshold,
thresh=self.score_threshold)
 else:
boxes = boxes[indexes]
scores = scores[indexes]
result.append({
'boxes': boxes,
'scores': scores,
})
 predictions.append(result)
 return predictions

 def run_wbf(self, predictions, image_index, image_shape=(IMG_SIZE,IMG_SIZE), iou_thr=WBF_IOU, skip_box_thr=WBF_SKIP_BOX, weights=None):
new_boxes_multi_models = []
for prediction in predictions:
pred_boxes = prediction[image_index]['boxes']
pred_boxes[:, 0] = pred_boxes[:, 0]/(image_shape[0]-1)
 pred_boxes[:, 1] = pred_boxes[:, 1]/(image_shape[1]-1)
 pred_boxes[:, 2] = pred_boxes[:, 2]/(image_shape[0]-1)
 pred_boxes[:, 3] = pred_boxes[:, 3]/(image_shape[1]-1)
 new_boxes_multi_models.append(pred_boxes.tolist())
 boxes = new_boxes_multi_models
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in
predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels,
weights=None, iou_thr=iou_thr,
skip_box_thr=skip_box_thr)
 boxes[:, 0] = boxes[:, 0]*(image_shape[0]-1)
 boxes[:, 1] = boxes[:, 1]*(image_shape[1]-1)
 boxes[:, 2] = boxes[:, 2]*(image_shape[0]-1)
 boxes[:, 3] = boxes[:, 3]*(image_shape[1]-1)
 return boxes, scores, labels
def format_prediction_string(self, boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def infer(self, pp_shrink=0):
for i in range(len(self.models)) :
self.models[i].eval()
torch.cuda.empty_cache()
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
test_loader = tqdm(self.test_loader, total=len(self.test_loader), desc="Testing")
 results = []
all_predictions = []
boxes10 = []
for images, image_ids in test_loader:
predictions=[]
for ii in range(len(self.models)) :
predictions.append(self.make_tta_predictions(tta_transforms, images,model_id=ii))
predictions = np.vstack(predictions)
 for i, image in enumerate(images):
boxes, scores, labels = self.run_wbf(predictions, image_index=i,image_shape=[image.shape[1],image.shape[2]])
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
if len(boxes10)< self.n_viz:
print('writing...',i,image_id)
 sample = image.permute(1,2,0 ).cpu().numpy()
fig, ax = plt.subplots(1, 1, figsize=(16, 8))
boxes10.append(( sample, boxes))
for box, score in zip(boxes,scores):
cv2.rectangle(sample,(box[0], box[1]),(box[2], box[3]),(1, 0, 0), 5)
 cv2.putText(sample, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX ,
1,(255,255,255), 3, cv2.LINE_AA)
 ax.set_axis_off()
ax.imshow(sample);
plt.show()
boxes[:, 0] = boxes[:, 0] + pp_shrink
boxes[:, 1] = boxes[:, 1] + pp_shrink
boxes[:, 2] = boxes[:, 2] - pp_shrink
boxes[:, 3] = boxes[:, 3] - pp_shrink
img_shape = image.cpu().numpy().shape
boxes[:, 0] = [max(min(x, img_shape[1]-1), 0)for x in boxes[:, 0]]
boxes[:, 1] = [max(min(x, img_shape[2]-1), 0)for x in boxes[:, 1]]
boxes[:, 2] = [max(min(x, img_shape[1]-1), 0)for x in boxes[:, 2]]
boxes[:, 3] = [max(min(x, img_shape[2]-1), 0)for x in boxes[:, 3]]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
areas = boxes[:, 2]*boxes[:, 3]
if USE_BOUNDS_FILTER==True:
print("Filter by area bounds")
 print("Length of unfiltered boxes: " + str(len(boxes)))
 print("Length of unfiltered scores: " + str(len(scores)))
 filtered_boxes = []
filtered_scores = []
for i in range(len(boxes)) :
if areas[i] > LOWER_BOUND and areas[i] < UPPER_BOUND and scores[i] >= WBF_SCORE_THRESHOLD:
filtered_boxes.append(boxes[i])
 filtered_scores.append(scores[i])
 print("Length of filtered boxes: " + str(len(filtered_boxes)))
 print("Length of filtered scores: " + str(len(filtered_scores)))
 boxes = filtered_boxes
scores = filtered_scores
else:
print("area filtering not applied")
 result = {
'image_id': image_id,
'PredictionString': self.format_prediction_string(boxes, scores),
}
all_prediction = {
'image_id': image_id,
'pred_boxes': boxes,
'scores': scores,
'img_shape' : img_shape
}
results.append(result)
 all_predictions.append(all_prediction)
 return results, all_predictions
def format_prediction_string(self, boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def save_predictions(self, results):
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv(f'{self.config.OUTPUT_DIR}/submission.csv', index=False)
 def load(self, paths):
print(paths)
 for i in range(len(paths)) :
checkpoint = torch.load(paths[i])
 self.models[i].load_state_dict(checkpoint['model_state_dict'])
 print('finish loading ',paths[i])
 def log(self, message):
if self.config.VERBOSE:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<feature_engineering>
MAX_TWEET_LENGTH = 280
VECTORS_PER_WORD = 50
BATCH_SIZE = 256
NUM_EPOCHS = 40
cfg['OUTPUT_DIR'] = "/kaggle/working/"
cfg['DATASETS']['ROOT_DIR'] = "/kaggle/input/global-wheat-detection"
cfg['TEST']['IMS_PER_BATCH'] = 1
cfg['TEST']['WEIGHT'] = BEST_PATHS
cfg<load_from_csv>
print('Loading test and training data')
 test_data = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 train_data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 print('Loading GloVe file')
 glove_data = pd.read_csv('/kaggle/input/glove50d/glove.6B.50d.txt', sep=' ', index_col=0, header = None, quoting=csv.QUOTE_NONE, encoding='utf-8')
 glove_data.head()
df_one_hot_attributes = convert_attributes_to_onehot(train_data)
 df_test_one_hot_attributes = convert_test_attributes_to_onehot(test_data, df_one_hot_attributes.columns.to_list())
 df_one_hot_attributes.head()
vectorized_training_data = vectorize_tweets(train_data, glove_data)
 vectorized_testing_data = vectorize_tweets(test_data, glove_data )
<categorify><EOS> 
model = build_model([MAX_TWEET_LENGTH, VECTORS_PER_WORD], len(df_one_hot_attributes.columns))
model.summary()
checkpoint_save = tf.keras.callbacks.ModelCheckpoint('saved_model.h5', save_best_only=True, monitor='val_acc', mode='min')
 model.fit(x=[df_one_hot_attributes, vectorized_training_data], y=train_data['target'], batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, verbose=1, shuffle=True)
 print('Training complete.Commencing creation of submission file')
 df_submission = pd.DataFrame()
df_submission['id'] = test_data['id']
predictions = model.predict([df_test_one_hot_attributes, vectorized_testing_data])
 prediction_list = []
for prediction in predictions:
if prediction < 0.5:
prediction_list += [0]
else:
prediction_list += [1]
df_submission['target'] = prediction_list
df_submission.head(5)
 df_submission.to_csv('submission.csv', index=False)
 print('All done.' )