Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
def load_dataset(root):
csv = pd.read_csv(os.path.join(root, "train.csv"))
data = {}
for i in csv.index:
key = csv["image_id"][i]
bbox = json.loads(csv["bbox"][i])
 bbox = [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3], 0.0]
if key in data:
data[key].append(bbox)
 else:
data[key] = [bbox]
return sorted(
 [(k, os.path.join(root, "train", k + ".jpg"), v)for k, v in data.items() ],
key=lambda x: x[0] 
)
 def load_image(path):
with open(path, "rb")as f:
buf = f.read()
return mx.image.imdecode(buf)
 def get_batches(dataset, batch_size, width=512, height=512, net=None, ctx=mx.cpu()):
batches = len(dataset)// batch_size
sampler = Sampler(dataset, width, height, net)
 stack_fn = [gcv.data.batchify.Stack() ]
pad_fn = [gcv.data.batchify.Pad(pad_val=-1)]
if net is None:
batchify_fn = gcv.data.batchify.Tuple(*(stack_fn + pad_fn))
else:
batchify_fn = gcv.data.batchify.Tuple(*(stack_fn * 6 + pad_fn))
with Pool(cpu_count() * 2)as p:
for i in range(batches):
start = i * batch_size
samples = p.map(sampler, range(start, start + batch_size))
batch = batchify_fn(samples)
 yield [x.as_in_context(ctx)for x in batch]
def gauss_blur(image, level):
return cv2.blur(image,(level * 2 + 1, level * 2 + 1))
def gauss_noise(image):
for i in range(image.shape[2]):
c = image[:, :, i]
diff = 255 - c.max() ;
noise = np.random.normal(0, random.randint(1, 6), c.shape)
 noise =(noise - noise.min())/(noise.max() - noise.min())
 noise = diff * noise
image[:, :, i] = c + noise.astype(np.uint8)
 return image
class YOLO3TrainTransform:
def __init__(self, width, height, net, mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225), **kwargs):
self._width = width
self._height = height
self._mean = mean
self._std = std
self._fake_x = mx.nd.zeros(( 1, 3, height, width))
net = copy.deepcopy(net)
 net.collect_params().reset_ctx(None)
 with mx.autograd.train_mode() :
_, self._anchors, self._offsets, self._feat_maps, _, _, _, _ = net(self._fake_x)
 self._target_generator = gcv.model_zoo.yolo.yolo_target.YOLOV3PrefetchTargetGenerator(
 num_class=len(net.classes), **kwargs)
 def __call__(self, img, label):
if np.random.uniform(0, 1)> 0.5:
img, expand = gcv.data.transforms.image.random_expand(img, max_ratio=1.5, fill=114, keep_ratio=False)
 bbox = gcv.data.transforms.bbox.translate(label, x_offset=expand[0], y_offset=expand[1])
 else:
img, bbox = img, label
h, w, _ = img.shape
bbox, crop = gcv.data.transforms.experimental.bbox.random_crop_with_constraints(bbox,(w, h))
x0, y0, w, h = crop
img = mx.image.fixed_crop(img, x0, y0, w, h)
 h, w, _ = img.shape
interp = np.random.randint(0, 5)
 img = gcv.data.transforms.image.imresize(img, self._width, self._height, interp=interp)
 bbox = gcv.data.transforms.bbox.resize(bbox,(w, h),(self._width, self._height))
h, w, _ = img.shape
img, flips = gcv.data.transforms.image.random_flip(img, px=0.5, py=0.5)
 bbox = gcv.data.transforms.bbox.flip(bbox,(w, h), flip_x=flips[0], flip_y=flips[1])
 img = gcv.data.transforms.experimental.image.random_color_distort(img)
 img = mx.nd.image.to_tensor(img)
 img = mx.nd.image.normalize(img, mean=self._mean, std=self._std)
 gt_bboxes = mx.nd.array(bbox[np.newaxis, :, :4])
 gt_ids = mx.nd.array(bbox[np.newaxis, :, 4:5])
 gt_mixratio = mx.nd.array(bbox[np.newaxis, :, -1:])
 objectness, center_targets, scale_targets, weights, class_targets = self._target_generator(
 self._fake_x, self._feat_maps, self._anchors, self._offsets,
gt_bboxes, gt_ids, gt_mixratio)
 return(img, objectness[0], center_targets[0], scale_targets[0], weights[0],
class_targets[0], gt_bboxes[0])
 class Sampler:
def __init__(self, dataset, width, height, net=None, **kwargs):
self._dataset = dataset
if net is None:
self._training_mode = False
self._transform = gcv.data.transforms.presets.yolo.YOLO3DefaultValTransform(width, height, **kwargs)
 else:
self._training_mode = True
self._transform = YOLO3TrainTransform(width, height, net, **kwargs)
 def __call__(self, idx):
if self._training_mode:
raw, bboxes = self._load_mixup(idx)
 raw = raw.asnumpy()
blur = random.randint(0, 3)
 if blur > 0:
raw = gauss_blur(raw, blur)
 raw = gauss_noise(raw)
 h, w, _ = raw.shape
rot = random.randint(0, 3)
 if rot > 0:
raw = np.rot90(raw, k=rot)
 if rot == 1:
raw_bboxes = bboxes.copy()
bboxes[:, [0, 2]] = raw_bboxes[:, [1, 3]]
bboxes[:, [1, 3]] = w - raw_bboxes[:, [2, 0]]
elif rot == 2:
bboxes[:, [0, 1, 2, 3]] = np.array([[w, h, w, h]])- bboxes[:, [2, 3, 0, 1]]
elif rot == 3:
raw_bboxes = bboxes.copy()
bboxes[:, [0, 2]] = h - raw_bboxes[:, [1, 3]]
bboxes[:, [1, 3]] = raw_bboxes[:, [2, 0]]
raw_bboxes = bboxes.copy()
bboxes[:, 0] = np.min(raw_bboxes[:, [0, 2]], axis=1)
 bboxes[:, 1] = np.min(raw_bboxes[:, [1, 3]], axis=1)
 bboxes[:, 2] = np.max(raw_bboxes[:, [0, 2]], axis=1)
 bboxes[:, 3] = np.max(raw_bboxes[:, [1, 3]], axis=1)
 raw = mx.nd.array(raw)
 else:
raw = load_image(self._dataset[idx][1])
 bboxes = np.array(self._dataset[idx][2])
 res = self._transform(raw, bboxes)
 return [mx.nd.array(x)for x in res]
def _load_mixup(self, idx1):
r = random.gauss(0.5, 0.5 / 1.96)
 if r > 0.0:
raw1 = load_image(self._dataset[idx1][1])
 bboxes1 = np.array(self._dataset[idx1][2])
 if r >= 1.0:
return raw1, np.hstack([bboxes1, np.full(( bboxes1.shape[0], 1), 1.0)])
 idx2 = random.randint(0, len(self._dataset)- 1)
 raw2 = load_image(self._dataset[idx2][1])
 bboxes2 = np.array(self._dataset[idx2][2])
 if r <= 0.0:
return raw2, np.hstack([bboxes2, np.full(( bboxes2.shape[0], 1), 1.0)])
 h = max(raw1.shape[0], raw2.shape[0])
 w = max(raw1.shape[1], raw2.shape[1])
 mix_raw = mx.nd.zeros(shape=(h, w, 3), dtype="float32")
 mix_raw[:raw1.shape[0], :raw1.shape[1], :] += raw1.astype("float32")* r
mix_raw[:raw2.shape[0], :raw2.shape[1], :] += raw2.astype("float32")*(1.0 - r)
 mix_bboxes = np.vstack([
np.hstack([bboxes1, np.full(( bboxes1.shape[0], 1), r)]),
np.hstack([bboxes2, np.full(( bboxes2.shape[0], 1), 1.0 - r)])
 ])
 return mix_raw.astype("uint8"), mix_bboxes 
<load_pretrained>
tweets = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
def load_model(path, ctx=mx.cpu()):
net = gcv.model_zoo.yolo3_darknet53_custom(["wheat"], pretrained_base=False)
 net.set_nms(post_nms=150)
 net.load_parameters(path, ctx=ctx)
 return net 
<normalization>
stop = set(stopwords.words('english'))
corpus0 = []
[corpus0.append(word.lower())for tweet in tweets[tweets.target == 0].text for word in word_tokenize(tweet)]
corpus1 = []
[corpus1.append(word.lower())for tweet in tweets[tweets.target == 1].text for word in word_tokenize(tweet)]
def count_top_stopwords(corpus):
stopwords_freq = {}
for word in corpus:
if word in stop:
if word in stopwords_freq:
stopwords_freq[word] += 1
else:
stopwords_freq[word] = 1
topwords = sorted(stopwords_freq.items() , key=lambda item: item[1], reverse=True)[:10]
x,y = zip(*topwords)
 return x,y
x0,y0 = count_top_stopwords(corpus0)
 x1,y1 = count_top_stopwords(corpus1)
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 plt.bar(x0, y0, color=color[0])
 plt.title('Top Stopwords for Non-Disaster Tweets')
 plt.subplot(1,2,2)
 plt.bar(x1, y1, color=color[1])
 plt.title('Top Stopwords for Disaster Tweets' )
def inference(models, path):
raw = load_image(path)
 rh, rw, _ = raw.shape
classes_list = []
scores_list = []
bboxes_list = []
for _ in range(16):
img, flips = gcv.data.transforms.image.random_flip(raw, px=0.5, py=0.5)
 x, _ = gcv.data.transforms.presets.yolo.transform_test(img, short=img_s)
 _, _, xh, xw = x.shape
rot = random.randint(0, 3)
 if rot > 0:
x = np.rot90(x.asnumpy() , k=rot, axes=(2, 3))
for model in models:
classes, scores, bboxes = model(mx.nd.array(x, ctx=context))
if rot > 0:
if rot == 1:
raw_bboxes = bboxes.copy()
bboxes[0, :, [0, 2]] = xh - raw_bboxes[0, :, [1, 3]]
bboxes[0, :, [1, 3]] = raw_bboxes[0, :, [2, 0]]
elif rot == 2:
bboxes[0, :, [0, 1, 2, 3]] = mx.nd.array([[xw], [xh], [xw], [xh]], ctx=context)- bboxes[0, :, [2, 3, 0, 1]]
elif rot == 3:
raw_bboxes = bboxes.copy()
bboxes[0, :, [0, 2]] = raw_bboxes[0, :, [1, 3]]
bboxes[0, :, [1, 3]] = xw - raw_bboxes[0, :, [2, 0]]
raw_bboxes = bboxes.copy()
bboxes[0, :, 0] = raw_bboxes[0, :, [0, 2]].min(axis=0)
 bboxes[0, :, 1] = raw_bboxes[0, :, [1, 3]].min(axis=0)
 bboxes[0, :, 2] = raw_bboxes[0, :, [0, 2]].max(axis=0)
 bboxes[0, :, 3] = raw_bboxes[0, :, [1, 3]].max(axis=0)
 bboxes[0, :, :] = gcv.data.transforms.bbox.flip(bboxes[0, :, :],(xw, xh), flip_x=flips[0], flip_y=flips[1])
 bboxes[0, :, 0::2] =(bboxes[0, :, 0::2] /(xw - 1)).clip(0.0, 1.0)
 bboxes[0, :, 1::2] =(bboxes[0, :, 1::2] /(xh - 1)).clip(0.0, 1.0)
 classes_list.append([
int(classes[0, i].asscalar())for i in range(classes.shape[1])
 if classes[0, i].asscalar() >= 0.0
])
 scores_list.append([
scores[0, i].asscalar() for i in range(classes.shape[1])
 if classes[0, i].asscalar() >= 0.0
])
 bboxes_list.append([
bboxes[0, i].asnumpy().tolist() for i in range(classes.shape[1])
 if classes[0, i].asscalar() >= 0.0
])
 bboxes, scores, classes = weighted_boxes_fusion(bboxes_list, scores_list, classes_list)
 bboxes[:, 0::2] *= rw - 1
bboxes[:, 1::2] *= rh - 1
return bboxes, scores, classes 
<load_pretrained>
stop = ENGLISH_STOP_WORDS.union(stop)
 def remove_url(txt):
return ' '.join(re.sub('([^0-9A-Za-z \t])|(\w+:\/\/\S+)', '', txt ).split())
 corpus0 = []
[corpus0.append(word.lower())for tweet in tweets[tweets.target == 0].text for word in word_tokenize(remove_url(tweet)) ]
corpus0 = list(filter(lambda x: x not in stop, corpus0))
corpus1 = []
[corpus1.append(word.lower())for tweet in tweets[tweets.target == 1].text for word in word_tokenize(remove_url(tweet)) ]
corpus1 = list(filter(lambda x: x not in stop, corpus1))
a = Counter(corpus0 ).most_common()
df0 = pd.DataFrame(a, columns=['Word','Count'])
 b = Counter(corpus1 ).most_common()
df1 = pd.DataFrame(b, columns=['Word','Count'])
 plt.figure(figsize=(15,4))
plt.subplot(1,2,1)
 sns.barplot(x='Word',y='Count',data=df0.head(10), color=color[1] ).set_title('Most Common Words for Non-Disasters')
 plt.xticks(rotation=45)
 plt.subplot(1,2,2)
 sns.barplot(x='Word',y='Count',data=df1.head(10), color=color[0] ).set_title('Most Common Words for Disasters')
 plt.xticks(rotation=45 )
rounds = 2
max_epochs = 5
learning_rate = 0.001
batch_size = 2
img_s = 1024
threshold = 0.1
context = mx.gpu()
print("Loading pre-trained model...")
 model = load_model(".. /input/yolov3/global-wheat-yolo3-darknet53_best2.params", ctx=context)
 print("Loading training set...")
 dataset = load_dataset("/kaggle/input/global-wheat-detection")
 print("Loading test images...")
 test_images = [ 
(os.path.join(dirname, filename), os.path.splitext(filename)[0])
 for dirname, _, filenames in os.walk('/kaggle/input/global-wheat-detection/test')for filename in filenames
]
if len(test_images)> 10:
best_score = 0.0
metrics = [gcv.utils.metrics.VOCMApMetric(iou_thresh=iou)for iou in [0.5, 0.55, 0.6]]
for pseudo_models in [[model]] * rounds:
print("Pseudo labaling...")
 pseudo_set = []
for path, image_id in test_images:
print(path)
 bboxes, scores, classes = inference(pseudo_models, path)
 label = [
[round(x)for x in bboxes[i].tolist() ] + [0.0] for i in range(classes.shape[0])
 if model.classes[int(classes[i])] == "wheat" and scores[i] > threshold
]
if len(label)> 0:
pseudo_set.append(( image_id, path, label))
training_set = pseudo_set
validation_set = dataset[int(len(dataset)* 0.9):]
print("Training set: ", len(training_set))
print("Validation set: ", len(validation_set))
print("Re-training...")
 trainer = mx.gluon.Trainer(model.collect_params() , "Nadam", {
"learning_rate": learning_rate
})
 for epoch in range(max_epochs):
ts = time.time()
random.shuffle(training_set)
 training_total_L = 0.0
training_batches = 0
for x, objectness, center_targets, scale_targets, weights, class_targets, gt_bboxes in get_batches(training_set, batch_size, width=img_s, height=img_s, net=model, ctx=context):
training_batches += 1
with mx.autograd.record() :
obj_loss, center_loss, scale_loss, cls_loss = model(x, gt_bboxes, objectness, center_targets, scale_targets, weights, class_targets)
 L = obj_loss + center_loss + scale_loss + cls_loss
L.backward()
trainer.step(x.shape[0])
 training_batch_L = mx.nd.mean(L ).asscalar()
if training_batch_L != training_batch_L:
raise ValueError()
training_total_L += training_batch_L
print("[Epoch %d Batch %d] batch_loss %.10f average_loss %.10f elapsed %.2fs" %(
 epoch, training_batches, training_batch_L, training_total_L / training_batches, time.time() - ts 
))
training_avg_L = training_total_L / training_batches
for metric in metrics:
metric.reset()
for x, label in get_batches(validation_set, batch_size, width=img_s, height=img_s, ctx=context):
classes, scores, bboxes = model(x)
 for metric in metrics:
metric.update(
 bboxes,
classes.reshape(( 0, -1)) ,
scores.reshape(( 0, -1)) ,
label[:, :, :4],
label[:, :, 4:5].reshape(( 0, -1)) 
)
 score = mx.nd.array([metric.get() [1] for metric in metrics], ctx=context ).mean()
print("[Epoch %d] training_loss %.10f validation_score %.10f best_score %.10f duration %.2fs" %(
 epoch + 1, training_avg_L, score.asscalar() , best_score, time.time() - ts 
))
if score.asscalar() > best_score:
best_score = score.asscalar()
model.save_parameters("global-wheat-yolo3-darknet53.params")
 print("Loading re-trained model...")
 model = load_model("global-wheat-yolo3-darknet53.params", ctx=context)
 print("Inference...")
 results = []
for path, image_id in test_images:
print(path)
 bboxes, scores, classes = inference([model], path)
 bboxes[:, 2:4] -= bboxes[:, 0:2]
results.append({
"image_id": image_id,
"PredictionString": " ".join([
" ".join([str(x)for x in [scores[i]] + [round(x)for x in bboxes[i].tolist() ]])
 for i in range(classes.shape[0])
 if model.classes[int(classes[i])] == "wheat" and scores[i] > threshold
])
 })
 pd.DataFrame(results, columns=['image_id', 'PredictionString'] ).to_csv('submission.csv', index=False)
<import_modules>
tweets['polarity'] = [TextBlob(tweet ).sentiment.polarity for tweet in tweets.text]
tweets['subjectivity'] = [TextBlob(tweet ).sentiment.subjectivity for tweet in tweets.text]
tweets['exclaimation_num'] = [tweet.count('!')for tweet in tweets.text]
tweets['questionmark_num'] = [tweet.count('?')for tweet in tweets.text]
def count_url_hashtag_mention(text):
urls_num = len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', text))
word_tokens = text.split()
hash_num = len([word for word in word_tokens if word[0] == '
mention_num = len([word for word in word_tokens if word[0] == '@' and word.count('@')== 1])
 return urls_num, hash_num, mention_num
url_num, hash_num, mention_num = zip(*[count_url_hashtag_mention(tweet)for tweet in tweets.text])
 tweets = tweets.assign(url_num = url_num, hash_num = hash_num, mention_num = mention_num)
 contractions = ["'t", "'re", "'s", "'d", "'ll", "'ve", "'m"]
tweets['contraction_num'] = [sum([tweet.count(cont)for cont in contractions])for tweet in tweets.text]
sys.path.insert(0, "/kaggle/input/weightedboxesfusion")
<define_variables>
tweets.keyword.fillna('None', inplace=True)
 def decontraction(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
tweets.text = [decontraction(tweet)for tweet in tweets.text]
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 print(remove_emoji('OMG there is a volcano eruption!!! 😭😱😷'))
tweets.text = tweets.text.apply(lambda x: remove_emoji(x))
DATA_DIR = "/kaggle/input/global-wheat-detection"
MODELS_IN_DIR = "/kaggle/input/frcnn152foldthree"<load_from_csv>
tweets.text = tweets.text.apply(lambda x: remove_url(x))
def remove_punct(text):
new_punct = re.sub('\ |\!|\?', '', punctuation)
 table = str.maketrans('','', new_punct)
 return text.translate(table)
 tweets.text = tweets.text.apply(lambda x: remove_punct(x))
def replace_amp(text):
text = re.sub(r' amp ', ' and ', text)
 return text
tweets.text = tweets.text.apply(lambda x: replace_amp(x))
test_df = pd.read_csv(os.path.join(DATA_DIR, "sample_submission.csv"))
test_df.shape<feature_engineering>
lemmatizer = WordNetLemmatizer()
def lemma(text):
words = word_tokenize(text)
 return ' '.join([lemmatizer.lemmatize(w.lower() , pos='v')for w in words])
 tweets.text = tweets.text.apply(lambda x: lemma(x))
class WheatDataset(Dataset):
def __init__(self, dataframe, image_dir, transforms=None):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.transforms = transforms
def __len__(self)-> int:
return len(self.image_ids)
 def __getitem__(self, idx: int):
image_id = self.image_ids[idx]
image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.df[self.df['image_id'] == image_id]
if self.transforms:
sample = {"image": image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id 
<choose_model_class>
def generate_ngrams(text, n):
words = word_tokenize(text)
 return [' '.join(ngram)for ngram in list(get_data(ngrams(words, n)))if not all(w in stop for w in ngram)]
def get_data(gen):
try:
for elem in gen:
yield elem
except(RuntimeError, StopIteration):
return
def get_model() :

 backbone = resnet_fpn_backbone('resnet152', pretrained=False)
 model = FasterRCNN(backbone, num_classes=2)
 return model<load_pretrained>
bigrams_disaster = tweets[tweets.target==1].text.apply(lambda x: generate_ngrams(x, 2))
bigrams_ndisaster = tweets[tweets.target==0].text.apply(lambda x: generate_ngrams(x, 2))
bigrams_d_dict = {}
for bgs in bigrams_disaster:
for bg in bgs:
if bg in bigrams_d_dict:
bigrams_d_dict[bg] += 1
else:
bigrams_d_dict[bg] = 1
bigrams_d_df = pd.DataFrame(bigrams_d_dict.items() , columns=['Bigrams','Count'])
 bigrams_nd_dict = {}
for bgs in bigrams_ndisaster:
for bg in bgs:
if bg in bigrams_nd_dict:
bigrams_nd_dict[bg] += 1
else:
bigrams_nd_dict[bg] = 1
bigrams_nd_df = pd.DataFrame(bigrams_nd_dict.items() , columns=['Bigrams','Count'] )
DEVICE = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = get_model()
model.load_state_dict(torch.load(os.path.join(MODELS_IN_DIR, "best_model.pth")))
 model.eval()
model.to(DEVICE)
 1 == 1<categorify>
trigrams_disaster = tweets[tweets.target==1].text.apply(lambda x: generate_ngrams(x, 3))
trigrams_ndisaster = tweets[tweets.target==0].text.apply(lambda x: generate_ngrams(x, 3))
trigrams_d_dict = {}
for tgs in trigrams_disaster:
for tg in tgs:
if tg in trigrams_d_dict:
trigrams_d_dict[tg] += 1
else:
trigrams_d_dict[tg] = 1
trigrams_d_df = pd.DataFrame(trigrams_d_dict.items() , columns=['Trigrams','Count'])
 trigrams_nd_dict = {}
for tgs in trigrams_ndisaster:
for tg in tgs:
if tg in trigrams_nd_dict:
trigrams_nd_dict[tg] += 1
else:
trigrams_nd_dict[tg] = 1
trigrams_nd_df = pd.DataFrame(trigrams_nd_dict.items() , columns=['Trigrams','Count'] )
def get_test_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<create_dataframe>
def remove_stopwords(text):
word_tokens = word_tokenize(text)
 return ' '.join([w.lower() for w in word_tokens if not w.lower() in stop])
 tweets['text_nostopwords'] = tweets.text.apply(lambda x: remove_stopwords(x))
def collate_fn(batch):
return tuple(zip(*batch))
test_dataset = WheatDataset(test_df, os.path.join(DATA_DIR, "test"), get_test_transforms())
 test_data_loader = DataLoader(
 test_dataset,
batch_size=4,
shuffle=False,
num_workers=1,
drop_last=False,
collate_fn=collate_fn
)<define_variables>
mask = np.array(Image.open('.. /input/twitterlogo3/twitter-logo-png-transparent.png'))
reverse = mask[...,::-1,:]
def wc_words(target, mask=mask):
words = [word.lower() for tweet in tweets[tweets.target == target].text_nostopwords for word in tweet.split() ]
words = list(filter(lambda w: w != 'like', words))
words = list(filter(lambda w: w != 'new', words))
words = list(filter(lambda w: w != 'people', words))
dict = {}
for w in words:
if w in dict:
dict[w] += 1
else:
dict[w] = 1
my_cloud = WordCloud(background_color='white', stopwords=stop, mask=mask, random_state=0 ).generate_from_frequencies(dict)
 plt.subplot(1,2,target+1)
 plt.imshow(my_cloud, interpolation='bilinear')
 plt.axis("off")
 plt.figure(figsize=(15,10))
wc_words(0)
 plt.title('Non-Disaster')
 wc_words(1, reverse)
 plt.title('Disaster')
 plt.show()
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
pd.reset_option('max_colwidth')
 tweets.drop('text_nostopwords', axis=1, inplace=True)
 tweets.head()
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]]
res_boxes[:, [1,3]] = boxes[:, [0,2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
X_train, X_val, y_train, y_val = train_test_split(tweets.drop(['keyword','location','target'],axis=1), tweets[['target']], test_size=0.2, stratify=tweets[['target']], random_state=0)
 X_train_text = X_train['text']
X_val_text = X_val['text']
print('X_train shape: ', X_train.shape)
 print('X_val shape: ', X_val.shape)
 print('y_train shape: ', y_train.shape)
 print('y_val shape: ', y_val.shape )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
print('Train Class Proportion:
', y_train['target'].value_counts() / len(y_train)* 100)
 print('
Validation Class Proportion:
', y_val['target'].value_counts() / len(y_val)* 100 )
def make_tta_predictions(images, score_threshold=0.5):
with torch.no_grad() :
images = torch.stack(images ).float().to(DEVICE)
 predictions = []
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<compute_train_metric>
tokenizer_1 = Tokenizer(num_words=5000, oov_token='<UNK>')
 tokenizer_1.fit_on_texts(X_train_text )
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.5, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<predict_on_test>
X_train_text = tokenizer_1.texts_to_sequences(X_train_text)
 X_val_text = tokenizer_1.texts_to_sequences(X_val_text)
 print(X_train_text[:10])
 print('')
 print(X_val_text[:10] )
results = []
for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<prepare_output>
tokenizer_1.sequences_to_texts([X_train_text[1]] )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.head()<save_to_csv>
vocab_size = len(tokenizer_1.word_index)+ 1
embeddings_index = dict()
f = open('.. /input/glovetwitter27b100dtxt/glove.twitter.27B.200d.txt')
 for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))
test_df.to_csv('submission.csv', index=False )<install_modules>
embedding_matrix = np.zeros(( vocab_size, 200))
for word, i in tokenizer_1.word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print('Embedding Matrix Shape:', embedding_matrix.shape )
!pip install --no-deps '.. /input/timm0130/timm-0.1.30-py3-none-any.whl' > /dev/null<install_modules>
num_epochs=15
dropout=0.2
recurrent_dropout=0.2
lr=0.0005
batch_size=128
class_weight = {0: y_train['target'].value_counts() [1]/len(y_train), 1: y_train['target'].value_counts() [0]/len(y_train)}
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<set_options>
lstm_model = Sequential()
embedding_layer = Embedding(vocab_size, 200, weights=[embedding_matrix], input_length=maxlen, trainable=False)
 lstm_model.add(embedding_layer)
 lstm_model.add(LSTM(128, return_sequences=True, dropout=dropout, recurrent_dropout=recurrent_dropout))
lstm_model.add(LSTM(128))
lstm_model.add(Dense(1, activation='sigmoid'))
adam = optimizers.Adam(lr=lr)
 lstm_model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['acc'])
 print(lstm_model.summary() )
%matplotlib inline
VerticalFlip, HorizontalFlip, IAASharpen,
OneOf, Compose , BboxParams, Resize, HueSaturationValue
,RandomBrightnessContrast, ToGray , Cutout , RandomSizedCrop )<import_modules>
checkpoint = ModelCheckpoint('lstm_model.h5', monitor='val_acc', save_best_only=True)
 history = lstm_model.fit(X_train_text, y_train, batch_size=batch_size, callbacks=[checkpoint], epochs=num_epochs,
class_weight=class_weight, validation_data=(X_val_text, y_val), verbose=1)
 plot_model_performance(history )
from effdet import get_efficientdet_config, EfficientDet, DetBenchTrain , DetBenchPredict
from effdet.efficientdet import HeadNet<define_variables>
test['char_len'] = test.text.str.len()
word_tokens = [len(word_tokenize(tweet)) for tweet in test.text]
test['word_len'] = word_tokens
sent_tokens = [len(sent_tokenize(tweet)) for tweet in test.text]
test['sent_len'] = sent_tokens
DIR_PATH = '/kaggle/input/global-wheat-detection/'
dir = glob.glob(os.path.join(DIR_PATH , '*'))
dir.sort(reverse=True)
 train_paths = glob.glob(os.path.join(dir[1] , '*'))
test_paths = glob.glob(os.path.join(dir[2] , '*'))<feature_engineering>
test['polarity'] = [TextBlob(tweet ).sentiment.polarity for tweet in test.text]
test['subjectivity'] = [TextBlob(tweet ).sentiment.subjectivity for tweet in test.text]
test['exclaimation_num'] = [tweet.count('!')for tweet in test.text]
test['questionmark_num'] = [tweet.count('?')for tweet in test.text]
def count_url_hashtag_mention(text):
urls_num = len(re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F])) +', text))
word_tokens = text.split()
hash_num = len([word for word in word_tokens if word[0] == '
mention_num = len([word for word in word_tokens if word[0] == '@' and word.count('@')== 1])
 return urls_num, hash_num, mention_num
url_num, hash_num, mention_num = zip(*[count_url_hashtag_mention(tweet)for tweet in test.text])
 test = test.assign(url_num = url_num, hash_num = hash_num, mention_num = mention_num)
 contractions = ["'t", "'re", "'s", "'d", "'ll", "'ve", "'m"]
test['contraction_num'] = [sum([tweet.count(cont)for cont in contractions])for tweet in test.text]
df = pd.read_csv(dir[0])
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df<feature_engineering>
test.keyword.fillna('None', inplace=True)
 def decontraction(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
test.text = [decontraction(tweet)for tweet in test.text]
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 print(remove_emoji("OMG there is a volcano eruption!!! 😭😱😷"))
test.text = test.text.apply(lambda x: remove_emoji(x))
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = df[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = df[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
df_folds<train_model>
test.text = test.text.apply(lambda x: remove_url(x))
def remove_punct(text):
new_punct = re.sub('\ |\!|\?', '', punctuation)
 table=str.maketrans('','',new_punct)
 return text.translate(table)
 test.text = test.text.apply(lambda x: remove_punct(x))
def replace_amp(text):
text = re.sub(r" amp ", " and ", text)
 return text
test.text = test.text.apply(lambda x: replace_amp(x))
image_id = '8425a537b.jpg'
image_path = glob.glob(os.path.join(dir[1] , image_id))
image , boxes = load_image_and_boxes(image_path[0])
 show_image(image, boxes, "Image without bounding box" )<train_model>
lemmatizer = WordNetLemmatizer()
def lemma(text):
words = word_tokenize(text)
 return ' '.join([lemmatizer.lemmatize(w.lower() , pos='v')for w in words])
 test.text = test.text.apply(lambda x: lemma(x))
image_id = 'b3c96d5ad.jpg'
image_path = glob.glob(os.path.join(dir[1] , image_id))
image , boxes = load_image_and_boxes(image_path[0])
 show_image(image, boxes, "Image with bounding box" )<normalization>
test_text = test['text']
test_text = tokenizer_1.texts_to_sequences(test_text)
 test_text = pad_sequences(test_text, padding='post', maxlen=50)
 print('X_test shape:', test_text.shape )
<normalization><EOS> 
lstm_model.load_weights('lstm_model.h5')
 submission = test.copy() [['id']]
submission['target'] = lstm_model.predict_classes(test_text)
 submission.to_csv('submission.csv', index=False)
 display(submission.head() )