Kaggle competition: Natural Language Processing with Disaster Tweets
import os
import datetime
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.optim as optim
import torch.optim.lr_scheduler as lr_scheduler
import torchvision.transforms as transforms
from PIL import Image<define_variables>
!pip install nlpaug
CLASS_NAMES =('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
 data_root = '.. /input/Kannada-MNIST'
train_file_name = 'train.csv'
val_file_name = 'Dig-MNIST.csv'
test_file_name = 'test.csv'
NORM_MEAN = [0.485, 0.456, 0.406]
NORM_STD = [0.229, 0.224, 0.225]<normalization>
!kaggle datasets download -d rtatman/glove-global-vectors-for-word-representation
class KannadaMNISTTransforms(transforms.Compose):
def __init__(self, in_channels=1, out_channels=1, size=(28, 28)) :
if out_channels not in(3, 1)or in_channels not in(3, 1):
raise ValueError("Images must have 1 or 3 channels")
 mean = NORM_MEAN if out_channels == 3 else [sum(NORM_MEAN)/ 3]
std = NORM_STD if out_channels == 3 else [sum(NORM_STD)/ 3]
transforms_list = []
if in_channels != out_channels:
transforms_list.append(transforms.Grayscale(out_channels))
transforms_list.extend([
transforms.RandomAffine(10.0, translate=(0.25, 0.25), shear=(-10, 10, -10, 10)) ,
transforms.Resize(size),
transforms.ToTensor() ,
transforms.RandomErasing(p=0.2, scale=(0.05, 0.1), ratio=(0.3, 3.3), value=0),
transforms.Normalize(mean=mean, std=std)
 ])
 super(KannadaMNISTTransforms, self ).__init__(transforms_list )<categorify>
!pip install nltk
!pip install gensim
class KannadaMNISTDataset(torch.utils.data.Dataset):
def __init__(self, images, targets=None, transform=None):
super(KannadaMNISTDataset, self ).__init__()
self.images = [Image.fromarray(image)for image in images]
self.targets = np.zeros(len(images)) if targets is None else targets.astype(int)
 self.transform = transform
def __len__(self):
return len(self.images)
 def __getitem__(self, index):
return self.images[index] if self.transform is None else self.transform(self.images[index]), torch.tensor(self.targets[index], dtype=torch.long )<load_from_csv>
nltk.download('all' )
train_df = pd.read_csv(os.path.join(data_root, train_file_name))
val_df = pd.read_csv(os.path.join(data_root, val_file_name))
test_df = pd.read_csv(os.path.join(data_root, test_file_name))<data_type_conversions>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
train_targets = train_df.label.values.astype(int)
 train_images =(train_df.drop('label', axis=1 ).values.astype(float)/ 255 ).reshape(-1, 28, 28)
 val_targets = val_df.label.values.astype(int)
 val_images =(val_df.drop('label', axis=1 ).values.astype(float)/ 255 ).reshape(-1, 28, 28)
 test_ids = test_df.id.values.astype(int)
 test_images =(test_df.drop('id', axis=1 ).values.astype(float)/ 255 ).reshape(-1, 28, 28)
 print("Origin trainset images/targets shapes", train_images.shape, train_targets.shape)
 print("Origin valset images/targets shapes", val_images.shape, val_targets.shape)
 print("Origin testset images/ids shape", test_images.shape, test_ids.shape)
 print()<create_dataframe>
plt.style.use('ggplot')
 stop=set(stopwords.words('english'))
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 plt.style.use('ggplot')
 stop=set(stopwords.words('english'))
warnings.filterwarnings("ignore")
 nltk.download('brown', quiet=True)
 nltk.download('universal_tagset', quiet=True)
 SEED = 1337
batch_size = 1000
size =(28, 28)
 origin_channels = 1
in_channels = 1
train_dataset = KannadaMNISTDataset(train_images, train_targets, transform=KannadaMNISTTransforms(in_channels=origin_channels, out_channels=in_channels, size=size))
val_dataset = KannadaMNISTDataset(val_images, val_targets, transform=Transforms(in_channels=origin_channels, out_channels=in_channels, size=size))
test_dataset = KannadaMNISTDataset(test_images, transform=Transforms(in_channels=origin_channels, out_channels=in_channels, size=size))
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4)
 val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=4)
 test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4 )<choose_model_class>
df_train = pd.read_csv('.. /input/nlp-getting-started/train.csv', dtype={'id': np.int16, 'target': np.int8})
 df_test = pd.read_csv('.. /input/nlp-getting-started/test.csv', dtype={'id': np.int16})
 print('Training Set Shape = {}'.format(df_train.shape))
print('Training Set Memory Usage = {:.2f} MB'.format(df_train.memory_usage().sum() / 1024**2))
print('Test Set Shape = {}'.format(df_test.shape))
print('Test Set Memory Usage = {:.2f} MB'.format(df_test.memory_usage().sum() / 1024**2))
class Conv2dBNReLU(nn.Sequential):

 def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, groups=1, bias=True):
super(Conv2dBNReLU, self ).__init__(
 nn.Conv2d(in_channels, out_channels,
kernel_size=kernel_size, stride=stride, padding=padding, groups=groups, bias=bias),
nn.BatchNorm2d(out_channels),
nn.ReLU(inplace=True)
)
 class DOLinearBNReLU(nn.Sequential):

 def __init__(self, in_features, out_features, bias=True):
super(DOLinearBNReLU, self ).__init__(
 nn.Dropout(0.2),
nn.Linear(in_features, out_features, bias=bias),
nn.BatchNorm1d(out_features),
nn.ReLU(inplace=True)
)
 class DOLinear(nn.Sequential):

 def __init__(self, in_features, out_features, bias=True):
super(DOLinear, self ).__init__(
 nn.Dropout(0.2),
nn.Linear(in_features, out_features, bias=bias)
)
 class KannadaMNISTNet(nn.Module):
def __init__(self, in_channels=1, classes=10):
super(KannadaMNISTNet, self ).__init__()
self.feature_extractor = nn.Sequential(
 Conv2dBNReLU(in_channels, 64, kernel_size=3, padding=1),
Conv2dBNReLU(64, 64, kernel_size=3, padding=1),
Conv2dBNReLU(64, 64, kernel_size=3, padding=1),
nn.MaxPool2d(2),
Conv2dBNReLU(64, 128, kernel_size=3, padding=1),
Conv2dBNReLU(128, 128, kernel_size=3, padding=1),
Conv2dBNReLU(128, 128, kernel_size=3, padding=1),
nn.MaxPool2d(2),
Conv2dBNReLU(128, 256, kernel_size=3, padding=1),
Conv2dBNReLU(256, 256, kernel_size=3, padding=1),
Conv2dBNReLU(256, 512, kernel_size=3, padding=1),
Conv2dBNReLU(512, 512, kernel_size=3, padding=1), 
)
 self.pool = nn.AdaptiveAvgPool2d(( 7, 7))
self.classifier = nn.Sequential(
 DOLinearBNReLU(7 * 7 * 512, 512),
DOLinear(512, classes)
)
 self.sm = nn.Softmax()
def forward(self, x):
x = self.feature_extractor(x)
 x = self.pool(x)
 x = x.view(-1, 7 * 7 * 512)
 x = self.classifier(x)
 return x
def predict(self, x):
x = self.forward(x)
 x = self.sm(x)
 return x<choose_model_class>
print(f'Number of unique values in keyword = {df_train["keyword"].nunique() }(Training)- {df_test["keyword"].nunique() }(Test)')
 print(f'Number of unique values in location = {df_train["location"].nunique() }(Training)- {df_test["location"].nunique() }(Test)' )
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
net = KannadaMNISTNet(in_channels=in_channels, classes=10)
 net = net.to(torch.device(device))
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters() , lr=0.002, weight_decay=0.00005)
 scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.25, patience=2, verbose=False, mode='min', threshold=0.0001, cooldown=0, min_lr=0.000001)
 trainer = PyTorchTrainer(device=device)
 trainer.train(net, optimizer, criterion, train_dataloader, val_dataloader, scheduler=scheduler, epochs=110 )<find_best_params>
def create_corpus(target):
corpus=[]
for x in df_train[df_train['target']==target]['text'].str.split() :
for i in x:
corpus.append(i)
 return corpus
net.eval()
for param in net.parameters() :
param.requires_grad = False<categorify>
counter=Counter(corpus)
 most=counter.most_common()
x=[]
y=[]
for word,count in most[:40]:
if(word not in stop):
x.append(word)
 y.append(count )
test_predictions = None
for batch in test_dataloader:
inputs = batch[0]
inputs = inputs.to(torch.device(device))
output = net.forward(inputs)
 predictions = output.argmax(dim=1 ).data
test_predictions = predictions if test_predictions is None else torch.cat(( test_predictions, predictions))
test_predictions = test_predictions.cpu().numpy()<prepare_output>
def get_top_tweet_bigrams(corpus, n=None):
vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 return words_freq[:n]
submission_df = pd.DataFrame(np.c_[test_ids[:,None], test_predictions], columns=['id', 'label'])
 submission_df.head()<save_to_csv>
df_train['word_count'] = df_train['text'].apply(lambda x: len(str(x ).split()))
df_test['word_count'] = df_test['text'].apply(lambda x: len(str(x ).split()))
df_train['unique_word_count'] = df_train['text'].apply(lambda x: len(set(str(x ).split())))
 df_test['unique_word_count'] = df_test['text'].apply(lambda x: len(set(str(x ).split())))
 df_train['stop_word_count'] = df_train['text'].apply(lambda x: len([w for w in str(x ).lower().split() if w in STOPWORDS]))
df_test['stop_word_count'] = df_test['text'].apply(lambda x: len([w for w in str(x ).lower().split() if w in STOPWORDS]))
df_train['url_count'] = df_train['text'].apply(lambda x: len([w for w in str(x ).lower().split() if 'http' in w or 'https' in w]))
df_test['url_count'] = df_test['text'].apply(lambda x: len([w for w in str(x ).lower().split() if 'http' in w or 'https' in w]))
df_train['mean_word_length'] = df_train['text'].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
df_test['mean_word_length'] = df_test['text'].apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
df_train['char_count'] = df_train['text'].apply(lambda x: len(str(x)))
 df_test['char_count'] = df_test['text'].apply(lambda x: len(str(x)))
 df_train['punctuation_count'] = df_train['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
df_test['punctuation_count'] = df_test['text'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
df_train['hashtag_count'] = df_train['text'].apply(lambda x: len([c for c in str(x)if c == '
df_test['hashtag_count'] = df_test['text'].apply(lambda x: len([c for c in str(x)if c == '
df_train['mention_count'] = df_train['text'].apply(lambda x: len([c for c in str(x)if c == '@']))
df_test['mention_count'] = df_test['text'].apply(lambda x: len([c for c in str(x)if c == '@']))
submission_df.to_csv('submission.csv', index=False )<install_modules>
def generate_ngrams(text, n_gram=1):
token = [token for token in text.lower().split(' ')if token != '' if token not in STOPWORDS]
ngrams = zip(*[token[i:] for i in range(n_gram)])
 return [' '.join(ngram)for ngram in ngrams]
N = 100
disaster_unigrams = defaultdict(int)
 nondisaster_unigrams = defaultdict(int)
 for tweet in df_train[DISASTER_TWEETS]['text']:
for word in generate_ngrams(tweet):
disaster_unigrams[word] += 1
for tweet in df_train[~DISASTER_TWEETS]['text']:
for word in generate_ngrams(tweet):
nondisaster_unigrams[word] += 1
df_disaster_unigrams = pd.DataFrame(sorted(disaster_unigrams.items() , key=lambda x: x[1])[::-1])
 df_nondisaster_unigrams = pd.DataFrame(sorted(nondisaster_unigrams.items() , key=lambda x: x[1])[::-1])
 disaster_bigrams = defaultdict(int)
 nondisaster_bigrams = defaultdict(int)
 for tweet in df_train[DISASTER_TWEETS]['text']:
for word in generate_ngrams(tweet, n_gram=2):
disaster_bigrams[word] += 1
for tweet in df_train[~DISASTER_TWEETS]['text']:
for word in generate_ngrams(tweet, n_gram=2):
nondisaster_bigrams[word] += 1
df_disaster_bigrams = pd.DataFrame(sorted(disaster_bigrams.items() , key=lambda x: x[1])[::-1])
 df_nondisaster_bigrams = pd.DataFrame(sorted(nondisaster_bigrams.items() , key=lambda x: x[1])[::-1])
 disaster_trigrams = defaultdict(int)
 nondisaster_trigrams = defaultdict(int)
 for tweet in df_train[DISASTER_TWEETS]['text']:
for word in generate_ngrams(tweet, n_gram=3):
disaster_trigrams[word] += 1
for tweet in df_train[~DISASTER_TWEETS]['text']:
for word in generate_ngrams(tweet, n_gram=3):
nondisaster_trigrams[word] += 1
df_disaster_trigrams = pd.DataFrame(sorted(disaster_trigrams.items() , key=lambda x: x[1])[::-1])
 df_nondisaster_trigrams = pd.DataFrame(sorted(nondisaster_trigrams.items() , key=lambda x: x[1])[::-1] )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<set_options>
sw = stopwords.words('english')
 stw = sw + ['lot','frog','ppl','tldr','time','nan','thing', 'subject', 're', 'edu', 'use','good','really','quite','nice','well','little','need','keep','make','important','take','get','very','course','instructor','example']
ps = PorterStemmer()
lemmatizer = nltk.stem.WordNetLemmatizer()
SEED = 100
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<categorify>
def lower(df):
df['com_token'] = df['text'].str.lower().str.split()
df["com_"] = df["com_token"].apply(' '.join)
 return df
def collate_fn(batch):
return tuple(zip(*batch))
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]]
res_boxes[:, [1,3]] = boxes[:, [0,2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes)
 tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<choose_model_class>
df_train = lower(df_train)
 df_train["Orig_comment"] = df_train["text"]
df_train["text"] = df_train["com_"]
MODELS_PATHS = ['.. /input/frcnnfold012best/frcnn-fold0-best100.pth',
'.. /input/frcnnfold012best/frcnn_fold1_best50.pth',
'.. /input/frcnnfold012best/frcnn_fold2_best50.pth']
def get_model() :
backbone = resnet_fpn_backbone('resnet101', pretrained=False)
 model = FasterRCNN(backbone, num_classes=2)
 return model
frcnn_models = []
DEVICE = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 for path in MODELS_PATHS:
model = get_model()
model.load_state_dict(torch.load(path))
model.eval()
model.to(DEVICE)
 frcnn_models.append(model )<feature_engineering>
def decontracted(tweet):

 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"can't", "can not", tweet)
 tweet = re.sub(r"he\ ’ s", "he is", tweet)
 tweet = re.sub(r"i\ ’ m", "he is", tweet)
 tweet=re.sub("(<.*?>)","",tweet)
 tweet=re.sub("(\\W|\\d)"," ",tweet)
 tweet = re.sub(r"n't", " not", tweet)
 tweet = re.sub(r"'re", " are", tweet)
 tweet = re.sub(r"'s", " is", tweet)
 tweet = re.sub(r"'d", " would", tweet)
 tweet = re.sub(r"'ll", " will", tweet)
 tweet = re.sub(r"'t", " not", tweet)
 tweet = re.sub(r"'ve", " have", tweet)
 tweet = re.sub(r"'m", " am", tweet)
 tweet = re.sub(r"'didnt", " did not", tweet)
 tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015-08-05", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015-08-06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"OBLITERATION", "obliteration", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"megynkelly", "Megyn Kelly", tweet)
 tweet = re.sub(r"cnewslive", "C News Live", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"cjoyner", "Chris Joyner", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"S3XLEAK", "sex leak", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"foodscare", "food scare", tweet)
 tweet = re.sub(r"MNPDNashville", "Metropolitan Nashville Police Department", tweet)
 tweet = re.sub(r"TfLBusAlerts", "TfL Bus Alerts", tweet)
 tweet = re.sub(r"GamerGate", "Gamer Gate", tweet)
 tweet = re.sub(r"IHHen", "Humanitarian Relief", tweet)
 tweet = re.sub(r"spinningbot", "spinning bot", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"po_st", "po.st", tweet)
 tweet = re.sub(r"scoopit", "scoop.it", tweet)
 tweet = re.sub(r"UltimaLucha", "Ultima Lucha", tweet)
 tweet = re.sub(r"JonathanFerrell", "Jonathan Ferrell", tweet)
 tweet = re.sub(r"aria_ahrary", "Aria Ahrary", tweet)
 tweet = re.sub(r"rapidcity", "Rapid City", tweet)
 tweet = re.sub(r"OutBid", "outbid", tweet)
 tweet = re.sub(r"lavenderpoetrycafe", "lavender poetry cafe", tweet)
 tweet = re.sub(r"EudryLantiqua", "Eudry Lantiqua", tweet)
 tweet = re.sub(r"15PM", "15 PM", tweet)
 tweet = re.sub(r"OriginalFunko", "Funko", tweet)
 tweet = re.sub(r"rightwaystan", "Richard Tan", tweet)
 tweet = re.sub(r"CindyNoonan", "Cindy Noonan", tweet)
 tweet = re.sub(r"RT_America", "RT America", tweet)
 tweet = re.sub(r"narendramodi", "Narendra Modi", tweet)
 tweet = re.sub(r"BakeOffFriends", "Bake Off Friends", tweet)
 tweet = re.sub(r"TeamHendrick", "Hendrick Motorsports", tweet)
 tweet = re.sub(r"alexbelloli", "Alex Belloli", tweet)
 tweet = re.sub(r"itsjustinstuart", "Justin Stuart", tweet)
 tweet = re.sub(r"gunsense", "gun sense", tweet)
 tweet = re.sub(r"DebateQuestionsWeWantToHear", "debate questions we want to hear", tweet)
 tweet = re.sub(r"RoyalCarribean", "Royal Carribean", tweet)
 tweet = re.sub(r"samanthaturne19", "Samantha Turner", tweet)
 tweet = re.sub(r"JonVoyage", "Jon Stewart", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"SuryaRay", "Surya Ray", tweet)
 tweet = re.sub(r"pattonoswalt", "Patton Oswalt", tweet)
 tweet = re.sub(r"minhazmerchant", "Minhaz Merchant", tweet)
 tweet = re.sub(r"TLVFaces", "Israel Diaspora Coalition", tweet)
 tweet = re.sub(r"pmarca", "Marc Andreessen", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"jamaicaplain", "Jamaica Plain", tweet)
 tweet = re.sub(r"Japton", "Arkansas", tweet)
 tweet = re.sub(r"RouteComplex", "Route Complex", tweet)
 tweet = re.sub(r"INSubcontinent", "Indian Subcontinent", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Hiroshima70", "Hiroshima", tweet)
 tweet = re.sub(r"GMMBC", "Greater Mt Moriah Baptist Church", tweet)
 tweet = re.sub(r"versethe", "verse the", tweet)
 tweet = re.sub(r"TubeStrike", "Tube Strike", tweet)
 tweet = re.sub(r"MissionHills", "Mission Hills", tweet)
 tweet = re.sub(r"ProtectDenaliWolves", "Protect Denali Wolves", tweet)
 tweet = re.sub(r"NANKANA", "Nankana", tweet)
 tweet = re.sub(r"SAHIB", "Sahib", tweet)
 tweet = re.sub(r"PAKPATTAN", "Pakpattan", tweet)
 tweet = re.sub(r"Newz_Sacramento", "News Sacramento", tweet)
 tweet = re.sub(r"gofundme", "go fund me", tweet)
 tweet = re.sub(r"pmharper", "Stephen Harper", tweet)
 tweet = re.sub(r"IvanBerroa", "Ivan Berroa", tweet)
 tweet = re.sub(r"LosDelSonido", "Los Del Sonido", tweet)
 tweet = re.sub(r"bancodeseries", "banco de series", tweet)
 tweet = re.sub(r"timkaine", "Tim Kaine", tweet)
 tweet = re.sub(r"IdentityTheft", "Identity Theft", tweet)
 tweet = re.sub(r"AllLivesMatter", "All Lives Matter", tweet)
 tweet = re.sub(r"mishacollins", "Misha Collins", tweet)
 tweet = re.sub(r"BillNeelyNBC", "Bill Neely", tweet)
 tweet = re.sub(r"BeClearOnCancer", "be clear on cancer", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"ScreamQueens", "Scream Queens", tweet)
 tweet = re.sub(r"AskCharley", "Ask Charley", tweet)
 tweet = re.sub(r"BlizzHeroes", "Heroes of the Storm", tweet)
 tweet = re.sub(r"BradleyBrad47", "Bradley Brad", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"meinlcymbals", "MEINL Cymbals", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"savebees", "save bees", tweet)
 tweet = re.sub(r"GreenHarvard", "Green Harvard", tweet)
 tweet = re.sub(r"StandwithPP", "Stand with planned parenthood", tweet)
 tweet = re.sub(r"hermancranston", "Herman Cranston", tweet)
 tweet = re.sub(r"WMUR9", "WMUR-TV", tweet)
 tweet = re.sub(r"RockBottomRadFM", "Rock Bottom Radio", tweet)
 tweet = re.sub(r"ameenshaikh3", "Ameen Shaikh", tweet)
 tweet = re.sub(r"ProSyn", "Project Syndicate", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"s2g", "swear to god", tweet)
 tweet = re.sub(r"listenlive", "listen live", tweet)
 tweet = re.sub(r"CDCgov", "Centers for Disease Control and Prevention", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"CBSBigBrother", "Big Brother", tweet)
 tweet = re.sub(r"JulieDiCaro", "Julie DiCaro", tweet)
 tweet = re.sub(r"theadvocatemag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park Police Department", tweet)
 tweet = re.sub(r"THISIZBWRIGHT", "Bonnie Wright", tweet)
 tweet = re.sub(r"Popularmmos", "Popular MMOs", tweet)
 tweet = re.sub(r"WildHorses", "Wild Horses", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"HORNDALE", "Horndale", tweet)
 tweet = re.sub(r"PINER", "Piner", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"thatswhatfriendsarefor", "that is what friends are for", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"MissCharleyWebb", "Charley Webb", tweet)
 tweet = re.sub(r"shoalstraffic", "shoals traffic", tweet)
 tweet = re.sub(r"GeorgeFoster72", "George Foster", tweet)
 tweet = re.sub(r"pop2015", "pop 2015", tweet)
 tweet = re.sub(r"_PokemonCards_", "Pokemon Cards", tweet)
 tweet = re.sub(r"DianneG", "Dianne Gallagher", tweet)
 tweet = re.sub(r"KashmirConflict", "Kashmir Conflict", tweet)
 tweet = re.sub(r"BritishBakeOff", "British Bake Off", tweet)
 tweet = re.sub(r"FreeKashmir", "Free Kashmir", tweet)
 tweet = re.sub(r"mattmosley", "Matt Mosley", tweet)
 tweet = re.sub(r"BishopFred", "Bishop Fred", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"UNHEALED", "unhealed", tweet)
 tweet = re.sub(r"CharlesDagnall", "Charles Dagnall", tweet)
 tweet = re.sub(r"Latestnews", "Latest news", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"datingtips", "dating tips", tweet)
 tweet = re.sub(r"charlesadler", "Charles Adler", tweet)
 tweet = re.sub(r"twia", "Texas Windstorm Insurance Association", tweet)
 tweet = re.sub(r"txlege", "Texas Legislature", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"hempoil", "hemp oil", tweet)
 tweet = re.sub(r"CommoditiesAre", "Commodities are", tweet)
 tweet = re.sub(r"tubestrike", "tube strike", tweet)
 tweet = re.sub(r"JoeNBC", "Joe Scarborough", tweet)
 tweet = re.sub(r"LiteraryCakes", "Literary Cakes", tweet)
 tweet = re.sub(r"TI5", "The International 5", tweet)
 tweet = re.sub(r"thehill", "the hill", tweet)
 tweet = re.sub(r"3others", "3 others", tweet)
 tweet = re.sub(r"stighefootball", "Sam Tighe", tweet)
 tweet = re.sub(r"whatstheimportantvideo", "what is the important video", tweet)
 tweet = re.sub(r"ClaudioMeloni", "Claudio Meloni", tweet)
 tweet = re.sub(r"DukeSkywalker", "Duke Skywalker", tweet)
 tweet = re.sub(r"carsonmwr", "Fort Carson", tweet)
 tweet = re.sub(r"offdishduty", "off dish duty", tweet)
 tweet = re.sub(r"andword", "and word", tweet)
 tweet = re.sub(r"rhodeisland", "Rhode Island", tweet)
 tweet = re.sub(r"easternoregon", "Eastern Oregon", tweet)
 tweet = re.sub(r"WAwildfire", "Washington Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"57am", "57 am", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"JacobHoggard", "Jacob Hoggard", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"under50", "under 50", tweet)
 tweet = re.sub(r"getitbeforeitsgone", "get it before it is gone", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"amwriting", "am writing", tweet)
 tweet = re.sub(r"Bokoharm", "Boko Haram", tweet)
 tweet = re.sub(r"Nowlike", "Now like", tweet)
 tweet = re.sub(r"seasonfrom", "season from", tweet)
 tweet = re.sub(r"epicente", "epicenter", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"sicklife", "sick life", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"EllenPompeo", "Ellen Pompeo", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"ABCNetwork", "ABC Network", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"SummerUnderTheStars", "Summer Under The Stars", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"onbeingwithKristaTippett", "on being with Krista Tippett", tweet)
 tweet = re.sub(r"Beingtweets", "Being tweets", tweet)
 tweet = re.sub(r"newauthors", "new authors", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"44PM", "44 PM", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"40PM", "40 PM", tweet)
 tweet = re.sub(r"myswc", "Severe Weather Center", tweet)
 tweet = re.sub(r"ithats", "that is", tweet)
 tweet = re.sub(r"icouldsitinthismomentforever", "I could sit in this moment forever", tweet)
 tweet = re.sub(r"FatLoss", "Fat Loss", tweet)
 tweet = re.sub(r"02PM", "02 PM", tweet)
 tweet = re.sub(r"MetroFmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"MetrofmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"terrorismturn", "terrorism turn", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BehindTheScenes", "Behind The Scenes", tweet)
 tweet = re.sub(r"GeorgeTakei", "George Takei", tweet)
 tweet = re.sub(r"WomensWeeklyMag", "Womens Weekly Magazine", tweet)
 tweet = re.sub(r"SurvivorsGuidetoEarth", "Survivors Guide to Earth", tweet)
 tweet = re.sub(r"incubusband", "incubus band", tweet)
 tweet = re.sub(r"Babypicturethis", "Baby picture this", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"win10", "Windows 10", tweet)
 tweet = re.sub(r"idkidk", "I do not know I do not know", tweet)
 tweet = re.sub(r"TheWalkingDead", "The Walking Dead", tweet)
 tweet = re.sub(r"amyschumer", "Amy Schumer", tweet)
 tweet = re.sub(r"crewlist", "crew list", tweet)
 tweet = re.sub(r"Erdogans", "Erdogan", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"TonyAbbottMHR", "Tony Abbott", tweet)
 tweet = re.sub(r"paulmyerscough", "Paul Myerscough", tweet)
 tweet = re.sub(r"georgegallagher", "George Gallagher", tweet)
 tweet = re.sub(r"JimmieJohnson", "Jimmie Johnson", tweet)
 tweet = re.sub(r"pctool", "pc tool", tweet)
 tweet = re.sub(r"DoingHashtagsRight", "Doing Hashtags Right", tweet)
 tweet = re.sub(r"ThrowbackThursday", "Throwback Thursday", tweet)
 tweet = re.sub(r"SnowBackSunday", "Snowback Sunday", tweet)
 tweet = re.sub(r"LakeEffect", "Lake Effect", tweet)
 tweet = re.sub(r"RTphotographyUK", "Richard Thomas Photography UK", tweet)
 tweet = re.sub(r"BigBang_CBS", "Big Bang CBS", tweet)
 tweet = re.sub(r"writerslife", "writers life", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"UnusualWords", "Unusual Words", tweet)
 tweet = re.sub(r"wizkhalifa", "Wiz Khalifa", tweet)
 tweet = re.sub(r"acreativedc", "a creative DC", tweet)
 tweet = re.sub(r"vscodc", "vsco DC", tweet)
 tweet = re.sub(r"VSCOcam", "vsco camera", tweet)
 tweet = re.sub(r"TheBEACHDC", "The beach DC", tweet)
 tweet = re.sub(r"buildingmuseum", "building museum", tweet)
 tweet = re.sub(r"WorldOil", "World Oil", tweet)
 tweet = re.sub(r"redwedding", "red wedding", tweet)
 tweet = re.sub(r"AmazingRaceCanada", "Amazing Race Canada", tweet)
 tweet = re.sub(r"WakeUpAmerica", "Wake Up America", tweet)
 tweet = re.sub(r"\\Allahuakbar\", "Allahu Akbar", tweet)
 tweet = re.sub(r"bleased", "blessed", tweet)
 tweet = re.sub(r"nigeriantribune", "Nigerian Tribune", tweet)
 tweet = re.sub(r"HIDEO_KOJIMA_EN", "Hideo Kojima", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"changetheworld", "change the world", tweet)
 tweet = re.sub(r"Ebolacase", "Ebola case", tweet)
 tweet = re.sub(r"mcgtech", "mcg technologies", tweet)
 tweet = re.sub(r"withweapons", "with weapons", tweet)
 tweet = re.sub(r"advancedwarfare", "advanced warfare", tweet)
 tweet = re.sub(r"letsFootball", "let us Football", tweet)
 tweet = re.sub(r"LateNiteMix", "late night mix", tweet)
 tweet = re.sub(r"PhilCollinsFeed", "Phil Collins", tweet)
 tweet = re.sub(r"RudyHavenstein", "Rudy Havenstein", tweet)
 tweet = re.sub(r"22PM", "22 PM", tweet)
 tweet = re.sub(r"54am", "54 AM", tweet)
 tweet = re.sub(r"38am", "38 AM", tweet)
 tweet = re.sub(r"OldFolkExplainStuff", "Old Folk Explain Stuff", tweet)
 tweet = re.sub(r"BlacklivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"InsaneLimits", "Insane Limits", tweet)
 tweet = re.sub(r"youcantsitwithus", "you cannot sit with us", tweet)
 tweet = re.sub(r"2k15", "2015", tweet)
 tweet = re.sub(r"TheIran", "Iran", tweet)
 tweet = re.sub(r"JimmyFallon", "Jimmy Fallon", tweet)
 tweet = re.sub(r"AlbertBrooks", "Albert Brooks", tweet)
 tweet = re.sub(r"defense_news", "defense news", tweet)
 tweet = re.sub(r"nuclearrcSA", "Nuclear Risk Control Self Assessment", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"NuclearPower", "Nuclear Power", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"ProBonoNews", "Pro Bono News", tweet)
 tweet = re.sub(r"JakartaPost", "Jakarta Post", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"ineedcake", "I need cake", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"AlexxPage", "Alex Page", tweet)
 tweet = re.sub(r"jonathanserrie", "Jonathan Serrie", tweet)
 tweet = re.sub(r"SocialJerkBlog", "Social Jerk Blog", tweet)
 tweet = re.sub(r"ChelseaVPeretti", "Chelsea Peretti", tweet)
 tweet = re.sub(r"irongiant", "iron giant", tweet)
 tweet = re.sub(r"RonFunches", "Ron Funches", tweet)
 tweet = re.sub(r"TimCook", "Tim Cook", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"Madsummer", "Mad summer", tweet)
 tweet = re.sub(r"NowYouKnow", "Now you know", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TomLandry", "Tom Landry", tweet)
 tweet = re.sub(r"showgirldayoff", "show girl day off", tweet)
 tweet = re.sub(r"Yougslavia", "Yugoslavia", tweet)
 tweet = re.sub(r"QuantumDataInformatics", "Quantum Data Informatics", tweet)
 tweet = re.sub(r"FromTheDesk", "From The Desk", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"CatoInstitute", "Cato Institute", tweet)
 tweet = re.sub(r"EmekaGift", "Emeka Gift", tweet)
 tweet = re.sub(r"LetsBe_Rational", "Let us be rational", tweet)
 tweet = re.sub(r"Cynicalreality", "Cynical reality", tweet)
 tweet = re.sub(r"FredOlsenCruise", "Fred Olsen Cruise", tweet)
 tweet = re.sub(r"NotSorry", "not sorry", tweet)
 tweet = re.sub(r"UseYourWords", "use your words", tweet)
 tweet = re.sub(r"WordoftheDay", "word of the day", tweet)
 tweet = re.sub(r"Dictionarycom", "Dictionary.com", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"uiseful", "useful", tweet)
 tweet = re.sub(r"JusticeDotOrg", "The American Association for Justice", tweet)
 tweet = re.sub(r"autoaccidents", "auto accidents", tweet)
 tweet = re.sub(r"SteveGursten", "Steve Gursten", tweet)
 tweet = re.sub(r"MichiganAutoLaw", "Michigan Auto Law", tweet)
 tweet = re.sub(r"birdgang", "bird gang", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"RVacchianoNYDN", "Ralph Vacchiano NY Daily News", tweet)
 tweet = re.sub(r"EdmontonEsks", "Edmonton Eskimos", tweet)
 tweet = re.sub(r"david_brelsford", "David Brelsford", tweet)
 tweet = re.sub(r"TOI_India", "The Times of India", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"SkinsOn9", "Skins on 9", tweet)
 tweet = re.sub(r"sothathappened", "so that happened", tweet)
 tweet = re.sub(r"LCOutOfDoors", "LC Out Of Doors", tweet)
 tweet = re.sub(r"NationFirst", "Nation First", tweet)
 tweet = re.sub(r"IndiaToday", "India Today", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"HOSTAGESTHROSW", "hostages throw", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"BidTime", "Bid Time", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"eatshit", "eat shit", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"facilitiesmanagement", "facilities management", tweet)
 tweet = re.sub(r"facilitydude", "facility dude", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"TheBodyShopAust", "The Body Shop Australia", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"OldsFireDept", "Olds Fire Department", tweet)
 tweet = re.sub(r"RiverComplex", "River Complex", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"slownewsday", "slow news day", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"AdilGhumro", "Adil Ghumro", tweet)
 tweet = re.sub(r"netbots", "net bots", tweet)
 tweet = re.sub(r"byebyeroad", "bye bye road", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"EndofUS", "End of United States", tweet)
 tweet = re.sub(r"35PM", "35 PM", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"76mins", "76 minutes", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"livesmatter", "lives matter", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"instaxbooty", "instagram booty", tweet)
 tweet = re.sub(r"Jerusalem_Post", "Jerusalem Post", tweet)
 tweet = re.sub(r"WayneRooney_INA", "Wayne Rooney", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"OculusRift", "Oculus Rift", tweet)
 tweet = re.sub(r"OwenJones84", "Owen Jones", tweet)
 tweet = re.sub(r"jeremycorbyn", "Jeremy Corbyn", tweet)
 tweet = re.sub(r"paulrogers002", "Paul Rogers", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"FilipeCoelho92", "Filipe Coelho", tweet)
 tweet = re.sub(r"OnlyQuakeNews", "Only Quake News", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"YEEESSSS", "yes", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"IntlDevelopment", "Intl Development", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"NewsThousands", "News Thousands", tweet)
 tweet = re.sub(r"EdmundAdamus", "Edmund Adamus", tweet)
 tweet = re.sub(r"EyewitnessWV", "Eye witness WV", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"DublinComicCon", "Dublin Comic Con", tweet)
 tweet = re.sub(r"NicholasBrendon", "Nicholas Brendon", tweet)
 tweet = re.sub(r"Alltheway80s", "All the way 80s", tweet)
 tweet = re.sub(r"FromTheField", "From the field", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"MadRiverComplex", "Mad River Complex", tweet)
 tweet = re.sub(r"feelingmanly", "feeling manly", tweet)
 tweet = re.sub(r"stillnotoverit", "still not over it", tweet)
 tweet = re.sub(r"FortitudeValley", "Fortitude Valley", tweet)
 tweet = re.sub(r"CoastpowerlineTramTr", "Coast powerline", tweet)
 tweet = re.sub(r"ServicesGold", "Services Gold", tweet)
 tweet = re.sub(r"NewsbrokenEmergency", "News broken emergency", tweet)
 tweet = re.sub(r"Evaucation", "evacuation", tweet)
 tweet = re.sub(r"leaveevacuateexitbe", "leave evacuate exit be", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"Tubestrike", "tube strike", tweet)
 tweet = re.sub(r"CLASS_SICK", "CLASS SICK", tweet)
 tweet = re.sub(r"localplumber", "local plumber", tweet)
 tweet = re.sub(r"awesomejobsiri", "awesome job siri", tweet)
 tweet = re.sub(r"PayForItHow", "Pay for it how", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"crimeairnetwork", "crime air network", tweet)
 tweet = re.sub(r"KimAcheson", "Kim Acheson", tweet)
 tweet = re.sub(r"cityofcalgary", "City of Calgary", tweet)
 tweet = re.sub(r"prosyndicate", "pro syndicate", tweet)
 tweet = re.sub(r"660NEWS", "660 NEWS", tweet)
 tweet = re.sub(r"BusInsMagazine", "Business Insurance Magazine", tweet)
 tweet = re.sub(r"wfocus", "focus", tweet)
 tweet = re.sub(r"ShastaDam", "Shasta Dam", tweet)
 tweet = re.sub(r"go2MarkFranco", "Mark Franco", tweet)
 tweet = re.sub(r"StephGHinojosa", "Steph Hinojosa", tweet)
 tweet = re.sub(r"Nashgrier", "Nash Grier", tweet)
 tweet = re.sub(r"NashNewVideo", "Nash new video", tweet)
 tweet = re.sub(r"IWouldntGetElectedBecause", "I would not get elected because", tweet)
 tweet = re.sub(r"SHGames", "Sledgehammer Games", tweet)
 tweet = re.sub(r"bedhair", "bed hair", tweet)
 tweet = re.sub(r"JoelHeyman", "Joel Heyman", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 tweet=tweet.strip()
return tweet
class WheatDataset(Dataset):
def __init__(self, dataframe, image_dir, transforms=None):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.transforms = transforms
def __len__(self)-> int:
return len(self.image_ids)
 def __getitem__(self, idx: int):
image_id = self.image_ids[idx]
image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.df[self.df['image_id'] == image_id]
if self.transforms:
sample = {"image": image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id<categorify>
def remove_punct(text):
table=str.maketrans('','',string.punctuation)
 return text.translate(table )
def get_train_transform() :
return A.Compose([
A.Flip(0.5),
ToTensorV2(p=1.0)
 ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})
 def get_valid_transform() :
return A.Compose([
ToTensorV2(p=1.0)
 ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})
 def get_test_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<categorify>
df_train['text']=df_train['text'].apply(reduce_lengthening, 0)
 df_train['text']=df_train['text'].apply(decontracted, 0)
 df_train['text']=df_train['text'].apply(lambda x : remove_punct(x))
def make_tta_predictions(images, model, score_threshold=0.40):
with torch.no_grad() :
images = torch.stack(images ).float().to(DEVICE)
 predictions = []
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<compute_train_metric>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'',text )
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.45, skip_box_thr=0.45, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<load_from_csv>
df_train['text']=df_train['text'].apply(lambda x : remove_URL(x))
DATA_DIR = '.. /input/global-wheat-detection'
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 test_df = pd.read_csv(os.path.join(DATA_DIR, "sample_submission.csv"))
testdf_pseudos = []
for model in frcnn_models:
results = []
testdf_pseudo = []
test_dataset = WheatDataset(test_df, os.path.join(DATA_DIR, "test"), get_test_transforms())
 test_data_loader = DataLoader(
 test_dataset,
batch_size=4,
shuffle=False,
num_workers=0,
drop_last=False,
collate_fn=collate_fn 
)
 for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images,model,score_threshold=0.70)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': 'nvnn'+image_id,
'width': 1024,
'height': 1024,
'source': 'nvnn',
'x': box[0],
'y': box[1],
'w': box[2],
'h': box[3]
}
testdf_pseudo.append(result)
 testdf_pseudos.append(testdf_pseudo )<create_dataframe>
aug_w2v = naw.WordEmbsAug(
 model_type='glove', model_path='.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt',
action="substitute")
new_train_dfs = []
for result in testdf_pseudos:
df = pd.DataFrame(result, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h'])
 new_train_dfs.append(df )<data_type_conversions>
aug_w2v.aug_p=0.2
print("Augmented Text:")
 for ii in range(5):
augmented_text = aug_w2v.augment(text)
 print(augmented_text )
train_df = pd.read_csv(f'{DATA_DIR}/train.csv')
 train_df['x'] = -1
train_df['y'] = -1
train_df['w'] = -1
train_df['h'] = -1
def expand_bbox(x):
r = np.array(re.findall("([0-9]+[.]?[0-9]*)", x))
if len(r)== 0:
r = [-1, -1, -1, -1]
return r
train_df[['x', 'y', 'w', 'h']] = np.stack(train_df['bbox'].apply(lambda x: expand_bbox(x)))
 train_df.drop(columns=['bbox'], inplace=True)
 train_df['x'] = train_df['x'].astype(np.float)
 train_df['y'] = train_df['y'].astype(np.float)
 train_df['w'] = train_df['w'].astype(np.float)
 train_df['h'] = train_df['h'].astype(np.float)
 skf = StratifiedKFold(n_splits=30, shuffle=True, random_state=10)
 fold_df = train_df[['image_id']].copy()
fold_df.loc[:, 'bbox_count'] = 1
fold_df = fold_df.groupby('image_id' ).count()
fold_df.loc[:, 'source'] = train_df[['image_id', 'source']].groupby('image_id' ).min() ['source']
fold_df.loc[:, 'group'] = np.char.add(fold_df['source'].values.astype('str'),
fold_df['bbox_count'].apply(lambda x: f'_{x//15}' ).values.astype('str'))
fold_df.loc[:, 'fold'] = 0
for i,(train_index, val_index)in enumerate(skf.split(X=fold_df.index, y=fold_df['group'])) :
fold_df.loc[fold_df.iloc[val_index].index, 'fold'] = i
fold_number = 0
valid_image_ids = fold_df[fold_df['fold']==fold_number].index.values
valid_df = train_df[train_df['image_id'].isin(valid_image_ids)]
valid_df.shape<feature_engineering>
train,valid=train_test_split(df_train,test_size=0.15)
 print('Shape of train',train.shape)
 print("Shape of Validation ",valid.shape )
class WheatDataset(Dataset):
def __init__(self, dataframe, image_dir=DATA_DIR, transforms=None):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
records = self.df[self.df['image_id'] == image_id]
if 'nvnn' in image_id:
image_id = image_id[4:]
image = cv2.imread(f'{self.image_dir}/test/{image_id}.jpg', cv2.IMREAD_COLOR)
 else:
image = cv2.imread(f'{self.image_dir}/train/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
area =(boxes[:, 3] - boxes[:, 1])*(boxes[:, 2] - boxes[:, 0])
 area = torch.as_tensor(area, dtype=torch.float32)
 labels = torch.ones(( records.shape[0],), dtype=torch.int64)
 iscrowd = torch.zeros(( records.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 target['area'] = area
target['iscrowd'] = iscrowd
if self.transforms:
sample = {
'image': image,
'bboxes': target['boxes'],
'labels': labels
}
sample = self.transforms(**sample)
 image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'] = target['boxes'].float()
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<init_hyperparams>
def augment_text(df,samples=300,pr=0.2):
aug_w2v.aug_p=pr
new_text=[]
df_n=df[df.target==1].reset_index(drop=True)
 for i in tqdm(np.random.randint(0,len(df_n),samples)) :
text = df_n.iloc[i]['text']
augmented_text = aug_w2v.augment(text)
 new_text.append(augmented_text)
 new=pd.DataFrame({'text':new_text,'target':1})
 df=shuffle(df.append(new ).reset_index(drop=True))
return df
class Averager:
def __init__(self):
self.current_total = 0.0
self.iterations = 0.0
def send(self, value):
self.current_total += value
self.iterations += 1
@property
def value(self):
if self.iterations == 0:
return 0
else:
return 1.0 * self.current_total / self.iterations
def reset(self):
self.current_total = 0.0
self.iterations = 0.0<create_dataframe>
train = augment_text(train,samples=400)
 tweet = train.append(valid ).reset_index(drop=True )
frcnn_models_trained = []
for model, train_df in zip(frcnn_models, new_train_dfs):
train_dataset = WheatDataset(train_df, image_dir=DATA_DIR, transforms=get_train_transform())
 valid_dataset = WheatDataset(valid_df, image_dir=DATA_DIR, transforms=get_valid_transform())
 indices = torch.randperm(len(train_dataset)).tolist()
train_data_loader = DataLoader(
 train_dataset,
batch_size=8,
shuffle=False,
num_workers=0,
collate_fn=collate_fn 
)
 valid_data_loader = DataLoader(
 valid_dataset,
batch_size=4,
shuffle=False,
num_workers=0,
collate_fn=collate_fn 
)
 model.train()
model.to(device)
 params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params, lr=0.01, momentum=0.9, weight_decay=0.0001)
 lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.5)
 num_epochs = 10
loss_hist = Averager()
itr = 1
for epoch in range(num_epochs):
loss_hist.reset()
for images, targets, image_ids in train_data_loader:
images = list(image.to(device)for image in images)
 targets = [{k: v.to(device)for k, v in t.items() } for t in targets]
loss_dict = model(images, targets)
 losses = sum(loss for loss in loss_dict.values())
 loss_value = losses.item()
loss_hist.send(loss_value)
 optimizer.zero_grad()
losses.backward()
optimizer.step()
if itr % 50 == 0:
print(f"Iteration
itr += 1
if lr_scheduler is not None:
lr_scheduler.step()
print(f"Epoch
frcnn_models_trained.append(model)
 frcnn_models_trained = [model.eval() for model in frcnn_models_trained]<define_variables>
df=pd.concat([tweet,df_test] )
USE_OPTIMIZE = False<load_from_csv>
def create_corpus(df):
corpus=[]
for tweet in tqdm(df['text']):
words=[word.lower() for word in word_tokenize(tweet)if(( word.isalpha() ==1)&(word not in stop)) ]
corpus.append(words)
 return corpus
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
corpus=create_corpus(df )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']]
df_folds['bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds['source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds['stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 train_index, holdout_index = next(skf.split(X=df_folds.index, y=df_folds['stratify_group']))
df_holdout = df_folds.iloc[holdout_index].copy()
df_folds = df_folds.iloc[train_index].copy()
df_folds['fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number<create_dataframe>
embedding_dict={}
with open('.. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt','r')as f:
for line in f:
values=line.split()
word=values[0]
vectors=np.asarray(values[1:],'float32')
 embedding_dict[word]=vectors
f.close()
holdout_dataset = DatasetRetriever(
 image_ids=df_holdout.index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 holdout_loader = DataLoader(
 holdout_dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
MAX_LEN=50
tokenizer_obj=Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
 sequences=tokenizer_obj.texts_to_sequences(corpus)
 tweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post' )
def load_net(checkpoint_path, version):
config = get_efficientdet_config(f'tf_efficientdet_d{version}')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
models = [
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold0-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold1-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold2-epo50last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold3-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold4-epo50last.bin', 5),
]<categorify>
word_index=tokenizer_obj.word_index
print('Number of unique words:',len(word_index))
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes*2 ).clip(min=0, max=1023 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores
if USE_OPTIMIZE:
all_predictions = []
for images, targets, image_ids in tqdm(holdout_loader, total=len(holdout_loader)) :
with torch.no_grad() :
images = torch.stack(images)
 images = images.cuda().float()
fold_predictions = {}
for fold_number in range(2):
fold_predictions[fold_number] = models[fold_number](images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
image_predictions = {
'image_id': image_ids[i],
'gt_boxes':(targets[i]['boxes'].cpu().numpy() *2 ).clip(min=0, max=1023 ).astype(int),
}
for fold_number in range(2):
boxes, scores = process_det(i, fold_predictions[fold_number])
 image_predictions[f'pred_boxes_fold{fold_number}'] = boxes
image_predictions[f'scores_fold{fold_number}'] = scores
all_predictions.append(image_predictions )<normalization>
num_words=len(word_index)+1
embedding_matrix=np.zeros(( num_words,100))
for word,i in tqdm(word_index.items()):
if i > num_words:
continue
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'{TRAIN_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth")
 iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x )<define_variables>
model = Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SimpleRNN(100))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'])
 model.summary()
def calculate_final_score(
 all_predictions,
iou_thr,
skip_box_thr,
method,
sigma=0.5, 
):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
image_id = all_predictions[i]['image_id']
folds_boxes, folds_scores, folds_labels = [], [], []
for fold_number in range(2):
pred_boxes = all_predictions[i][f'pred_boxes_fold{fold_number}'].copy()
scores = all_predictions[i][f'scores_fold{fold_number}'].copy()
folds_boxes.append(pred_boxes)
 folds_scores.append(scores)
 folds_labels.append(np.ones(pred_boxes.shape[0]))
if method == 'weighted_boxes_fusion':
boxes, scores, labels = weighted_boxes_fusion(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 elif method == 'nms':
boxes, scores, labels = nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr)
 elif method == 'soft_nms':
boxes, scores, labels = soft_nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, thresh=skip_box_thr, sigma=sigma)
 elif method == 'non_maximum_weighted':
boxes, scores, labels = non_maximum_weighted(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 else:
raise
image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )<compute_train_metric>
train_df=tweet_pad[:tweet.shape[0]]
test_df=tweet_pad[tweet.shape[0]:]
if USE_OPTIMIZE:
print('[WBF]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='weighted_boxes_fusion', 
))
print('[NMS]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='nms', 
))
print('[SOFT NMS]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
sigma=0.1,
method='soft_nms', 
))
print('[NMW]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='non_maximum_weighted', 
))<find_best_params>
X_train,y_train = train_df[:train.shape[0]],tweet['target'][:train.shape[0]]
X_test,y_test= train_df[train.shape[0]:],tweet['target'][train.shape[0]:]
def log(text):
with open('opt.log', 'a+')as logger:
logger.write(f'{text}
')
 def optimize(space, all_predictions, method, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*5 + f'{method}' + '-'*5)
 log(params)
 final_score = calculate_final_score(all_predictions, method=method, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<find_best_params>
history=model.fit(X_train,y_train,batch_size=4,epochs=10,validation_data=(X_test,y_test),verbose=2 )
space = [
Real(0, 1, name='iou_thr'),
Real(0.25, 1, name='skip_box_thr'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='weighted_boxes_fusion',
n_calls=50,
)<find_best_params>
y_pre=model.predict(X_test)
 y_pre=np.round(y_pre ).astype(int ).reshape(1142 )
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
else:
best_final_score = 0.7197
best_iou_thr = 0.450
best_skip_box_thr = 0.450
print('-'*13 + 'WBF' + '-'*14)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )<create_dataframe>
print(roc_auc_score(y_pre,y_test))
if USE_OPTIMIZE:
all_predictions = []
for fold_number in range(5):
validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 validation_loader = DataLoader(
 validation_dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn 
)
 for images, targets, image_ids in tqdm(validation_loader, total=len(validation_loader)) :
with torch.no_grad() :
images = torch.stack(images)
 images = images.cuda().float()
det = models[fold_number](images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
all_predictions.append({
'pred_boxes':(boxes*2 ).clip(min=0, max=1023 ).astype(int),
'scores': scores,
'gt_boxes':(targets[i]['boxes'].cpu().numpy() *2 ).clip(min=0, max=1023 ).astype(int),
'image_id': image_ids[i],
} )<compute_train_metric>
scores_model = []
def calculate_final_score(all_predictions, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
pred_boxes = all_predictions[i]['pred_boxes'].copy()
scores = all_predictions[i]['scores'].copy()
image_id = all_predictions[i]['image_id']
indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )<find_best_params>
scores_model.append({'Model': 'SimpleRNN','AUC_Score': roc_auc_score(y_pre,y_test)} )
best_final_score, best_score_threshold = 0, 0
if USE_OPTIMIZE:
for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
else:
best_score_threshold = 0.40<compute_test_metric>
model=Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'])
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30 )<data_type_conversions>
history=model.fit(X_train,y_train,batch_size=4,epochs=10,validation_data=(X_test,y_test),verbose=2 )
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<categorify>
y_pre=model.predict(X_test)
 y_pre=np.round(y_pre ).astype(int ).reshape(1142 )
def get_valid_transforms512() :
return A.Compose([A.Resize(height=512, width=512, p=1.0), ToTensorV2(p=1.0)], p=1.0)
 def get_valid_transforms1024() :
return A.Compose([ToTensorV2(p=1.0)], p=1.0)
 dataset512 = TestDatasetRetriever(image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms512())
 data_loader512 = DataLoader(
 dataset512,
batch_size=2,
shuffle=False,
num_workers=0,
drop_last=False,
collate_fn=collate_fn 
)
 dataset1024 = TestDatasetRetriever(image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms1024())
 data_loader1024 = DataLoader(
 dataset1024,
batch_size=2,
shuffle=False,
num_workers=0,
drop_last=False,
collate_fn=collate_fn
)<predict_on_test>
print(roc_auc_score(y_pre,y_test))
def make_frcnn_predictions(images, score_threshold=0.50):
with torch.no_grad() :
images = torch.stack(images ).cuda().float()
predictions = []
for fold_number, model in enumerate(frcnn_models_trained):
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<train_model>
scores_model.append({'Model': 'LSTM','AUC_Score': roc_auc_score(y_pre,y_test)} )
for i, model in enumerate(models):
print(type(model))<categorify>
model=Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(SpatialDropout1D(0.2))
model.add(GRU(300))
model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'])
 model.summary()
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes*2 ).clip(min=0, max=1023 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores<predict_on_test>
history=model.fit(X_train,y_train,batch_size=8,epochs=10,validation_data=(X_test,y_test),verbose=2 )
def make_predictions(images, score_threshold=best_score_threshold):
with torch.no_grad() :
images = torch.stack(images ).cuda().float()
predictions = []
for fold_number, net in enumerate(models):
for tta_transform in tta_transforms:
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<categorify>
y_pre=model.predict(X_test)
 y_pre=np.round(y_pre ).astype(int ).reshape(1142)
def run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
model=Sequential()
embedding=Embedding(num_words,100,embeddings_initializer=Constant(embedding_matrix),
input_length=MAX_LEN,trainable=False)
 model.add(embedding)
 model.add(Bidirectional(LSTM(300, dropout=0.3, recurrent_dropout=0.3)))
 model.add(Dense(1, activation='sigmoid'))
optimzer=Adam(learning_rate=1e-5)
 model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'])
 model.summary()
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<predict_on_test>
history=model.fit(X_train,y_train,batch_size=4,epochs=5,validation_data=(X_test,y_test),verbose=2 )
results = []
for(images_effdet, image_ids_effdet),(images_frcnn, image_ids_frcnn)in zip(data_loader512, data_loader1024):
if image_ids_effdet == image_ids_frcnn:
frcnn_predictions = make_frcnn_predictions(images_frcnn)
 effdet_predictions = make_effdet_predictions(images_effdet)
 images = image_ids_effdet
predictions = frcnn_predictions + effdet_predictions
for i, image in enumerate(images):
boxes, scores, labels = run_wbf(effdet_predictions, image_index=i)
 boxes =(boxes * 2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 else:
print('Error')
 break<save_to_csv>
y_pre=model.predict(X_test)
 y_pre=np.round(y_pre ).astype(int ).reshape(1142)
 print(roc_auc_score(y_pre,y_test))
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )<choose_model_class>
scores_model.append({'Model': 'Bidirectional-LSTM','AUC_Score': roc_auc_score(y_pre,y_test)} )
def load_net(checkpoint_path, version):
config = get_efficientdet_config(f'tf_efficientdet_d{version}')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
models = [
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold0-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold1-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold2-epo50last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold3-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold4-epo50last.bin', 5),
]<define_variables>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
best_score_threshold = 0.40
best_iou_thr = 0.45
best_skip_box_thr = 0.45<categorify>
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
class BaseWheatTTA:
image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
transform = TTACompose([
TTARotate90() ,
TTAVerticalFlip() ,
] )<data_type_conversions>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)<predict_on_test>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
def make_predictions(images, score_threshold=best_score_threshold):
with torch.no_grad() :
images = torch.stack(images ).cuda().float()
predictions = []
for fold_number, net in enumerate(models):
for tta_transform in tta_transforms:
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<compute_train_metric>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
def run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<predict_on_test>
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
results = []
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<create_dataframe>
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
callbacks=[checkpoint],
batch_size=16
)
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df<define_variables>
model.load_weights('model.h5')
 test_pred = model.predict(test_input )
NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.25
best_iou_thr = 0.6
best_skip_box_thr = 0.43
best_final_score = 0
best_score_threshold = 0
SEED = 42
EPO = 15
WEIGHTS = '.. /input/yolov5-k-weights/full-best.pt'
CONFIG = '.. /input/configyolo5/yolov5x.yaml'
DATA = '.. /input/configyolo5/wheat0.yaml'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >11
is_AUG = True
is_ROT = True
VALIDATE = True
PSEUDO = True<set_options>
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )