Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<install_modules>
pd.options.mode.chained_assignment = None
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 1000 )
!python setup.py install<import_modules>
train_origin_df = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_origin_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 train_df = train_origin_df.copy()
test_df = test_origin_df.copy()
def callback(operation_future):
result = operation_future.result()
import pycocotools<install_modules>
dataset_origin_df = pd.concat([train_df, test_df], axis=0, sort=True)
 dataset_origin_df.reset_index(inplace=True, drop=True )
!pip install -v -e .<define_variables>
words_containing_alpha_df = pd.DataFrame(columns = ['word', 'real', 'fake'])
 for row in tqdm(dataset_origin_df.iterrows()):
row = row[1]
result = re.findall('@+\w*', row['text'])
 if(len(result)> 0):
for word in result:
real = 1 if row['target'] == 1 else 0
fake = 1 if row['target'] != 1 else 0
temp_df = pd.DataFrame([[word, real, fake]], columns = ['word', 'real', 'fake'])
 words_containing_alpha_df = words_containing_alpha_df.append(temp_df )
config_txt =
config_file = open("/kaggle/working/mmdetection/config.py", "w")
 n = config_file.write(config_txt)
 config_file.close()<define_variables>
words_containing_alpha_df = words_containing_alpha_df.groupby(['word'], as_index=False)['real', 'fake'].sum()
words_containing_alpha_df['total'] = words_containing_alpha_df['real'] + words_containing_alpha_df['fake']
words_containing_alpha_df = words_containing_alpha_df.sort_values(by = 'total', ascending = False)
 words_containing_alpha_df
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
<load_from_csv>
threshold_no_of_words = 5
threshold_ratio_of_real_or_fake = 0.75
sub_words_df = words_containing_alpha_df.loc[(words_containing_alpha_df['total'] >= threshold_no_of_words)&(words_containing_alpha_df[['real', 'fake']].max(axis=1)/words_containing_alpha_df['total'] >= threshold_ratio_of_real_or_fake), :]
sub_words_df
checkpoint_path = '.. /input/resnest3fcos1iouatseven/epoch_40.pth'
config_path = '/kaggle/working/mmdetection/config.py'
model = init_detector(config_path, checkpoint_path, device='cuda:0')
 val_df = pd.read_csv('.. /input/global-wheat-detection/sample_submission.csv')
 all_image_ids = set(val_df['image_id'].unique())
 pred_threshold = 0.25
pred_results = []
for image_id in all_image_ids:
img = '.. /input/global-wheat-detection/test/' + image_id + '.jpg'
result = inference_detector(model, img)
 boxes = result[0][:, 0:4]
scores = result[0][:, 4]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
boxes = boxes[scores >= pred_threshold].astype(np.int32)
 scores = scores[scores >= pred_threshold]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
pred_results.append(result)
 test_df = pd.DataFrame(pred_results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )<define_variables>
sub_words_df.drop(0, axis=0, inplace=True )
NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.25
best_iou_thr = 0.6
best_skip_box_thr = 0.43
best_final_score = 0
best_score_threshold = 0
SEED = 42
EPO = 15
WEIGHTS = '.. /input/yolov5test/yolo5x_panet_1.pt'
CONFIG = '.. /input/configtest/yolo5_PANET.yaml'
DATA = '.. /input/configyolo5/wheat0.yaml'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >11
is_AUG = True
is_ROT = True
VALIDATE = True
PSEUDO = True<load_from_csv>
def remove_words_containing_alpha_by_threshold(text):
containing_words = re.findall('@+\w*', text)
 if(len(containing_words)> 0):
for word in containing_words:
if(word not in np.array(sub_words_df['word'])) :
text = text.replace(word, " ID")
 return text
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
set_seed(SEED)
 marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True)
 marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name, mini in tqdm(df.groupby('image_id')) :
path2save = 'val2017/' if name in val_index else 'train2017/'
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save, exist_ok=True)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save), exist_ok=True)
 sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name))
def run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, NMS_CONF_THR, NMS_IOU_THR, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores)
 @jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def validate() :
source = 'convertor/fold0/images/val2017'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 records = marking[marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes = gtboxes.astype(np.int32 ).clip(min=0, max=1024)
 gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
result = {
'image_id': image_id,
'pred_enboxes': enboxes,
'pred_enscores': enscores,
'gt_boxes': gtboxes,
}
results.append(result)
 return results
def calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
enboxes = all_predictions[i]['pred_enboxes'].copy()
enscores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes = pred_boxes.astype(np.int32 ).clip(min=0, max=1024)
 indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
rank = np.argsort(scores)[::-1]
pred_boxes = pred_boxes[rank]
scores = scores[rank]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores)
 def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'.. /input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth")
 def log(text):
print(text)
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*10)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<load_pretrained>
def clean_1_2_1(tweet):
tweet = re.sub(r"%20", " ", tweet)
 tweet = re.sub(r"
", " ", tweet)
 tweet = re.sub(r"\x89Û_", "", tweet)
 tweet = re.sub(r"\x89ÛÒ", "", tweet)
 tweet = re.sub(r"\x89ÛÓ", "", tweet)
 tweet = re.sub(r"\x89ÛÏWhen", "When", tweet)
 tweet = re.sub(r"\x89ÛÏ", "", tweet)
 tweet = re.sub(r"China\x89Ûªs", "China's", tweet)
 tweet = re.sub(r"let\x89Ûªs", "let's", tweet)
 tweet = re.sub(r"\x89Û÷", "", tweet)
 tweet = re.sub(r"\x89Ûª", "", tweet)
 tweet = re.sub(r"\x89Û\x9d", "", tweet)
 tweet = re.sub(r"å_", "", tweet)
 tweet = re.sub(r"\x89Û¢", "", tweet)
 tweet = re.sub(r"\x89Û¢åÊ", "", tweet)
 tweet = re.sub(r"fromåÊwounds", "from wounds", tweet)
 tweet = re.sub(r"åÊ", "", tweet)
 tweet = re.sub(r"åÈ", "", tweet)
 tweet = re.sub(r"JapÌ_n", "Japan", tweet)
 tweet = re.sub(r"Ì©", "e", tweet)
 tweet = re.sub(r"å¨", "", tweet)
 tweet = re.sub(r"SuruÌ¤", "Suruc", tweet)
 tweet = re.sub(r"åÇ", "", tweet)
 tweet = re.sub(r"å£3million", "3 million", tweet)
 tweet = re.sub(r"åÀ", "", tweet)
 return tweet
def clean_1_2_2(tweet):
tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 return tweet
def clean_1_2_3(tweet):
tweet = re.sub(r"&gt;", ">", tweet)
 tweet = re.sub(r"&lt;", "<", tweet)
 tweet = re.sub(r"&amp;", "&", tweet)
 return tweet
def clean_1_2_4(tweet):
tweet = re.sub(r"w/e", "whatever", tweet)
 tweet = re.sub(r"w/", "with ", tweet)
 tweet = re.sub(r"USAgov", "USA government", tweet)
 tweet = re.sub(r"recentlu", "recently", tweet)
 tweet = re.sub(r"Ph0tos", "Photos", tweet)
 tweet = re.sub(r"amirite", "am I right", tweet)
 tweet = re.sub(r"exp0sed", "exposed", tweet)
 tweet = re.sub(r"<3", "love", tweet)
 tweet = re.sub(r"amageddon", "armageddon", tweet)
 tweet = re.sub(r"Trfc", "Traffic", tweet)
 tweet = re.sub(r"8/5/2015", "2015/08/05", tweet)
 tweet = re.sub(r"WindStorm", "Wind Storm", tweet)
 tweet = re.sub(r"8/6/2015", "2015/08/06", tweet)
 tweet = re.sub(r"10:38PM", "10:38 PM", tweet)
 tweet = re.sub(r"10:30pm", "10:30 PM", tweet)
 tweet = re.sub(r"16yr", "16 year", tweet)
 tweet = re.sub(r"lmao", "laughing my ass off", tweet)
 tweet = re.sub(r"TRAUMATISED", "traumatized", tweet)
 return tweet
def clean_1_2_5(tweet):
tweet = re.sub(r"IranDeal", "Iran Deal", tweet)
 tweet = re.sub(r"ArianaGrande", "Ariana Grande", tweet)
 tweet = re.sub(r"camilacabello97", "camila cabello", tweet)
 tweet = re.sub(r"RondaRousey", "Ronda Rousey", tweet)
 tweet = re.sub(r"MTVHottest", "MTV Hottest", tweet)
 tweet = re.sub(r"TrapMusic", "Trap Music", tweet)
 tweet = re.sub(r"ProphetMuhammad", "Prophet Muhammad", tweet)
 tweet = re.sub(r"PantherAttack", "Panther Attack", tweet)
 tweet = re.sub(r"StrategicPatience", "Strategic Patience", tweet)
 tweet = re.sub(r"socialnews", "social news", tweet)
 tweet = re.sub(r"NASAHurricane", "NASA Hurricane", tweet)
 tweet = re.sub(r"onlinecommunities", "online communities", tweet)
 tweet = re.sub(r"humanconsumption", "human consumption", tweet)
 tweet = re.sub(r"Typhoon-Devastated", "Typhoon Devastated", tweet)
 tweet = re.sub(r"Meat-Loving", "Meat Loving", tweet)
 tweet = re.sub(r"facialabuse", "facial abuse", tweet)
 tweet = re.sub(r"LakeCounty", "Lake County", tweet)
 tweet = re.sub(r"BeingAuthor", "Being Author", tweet)
 tweet = re.sub(r"withheavenly", "with heavenly", tweet)
 tweet = re.sub(r"thankU", "thank you", tweet)
 tweet = re.sub(r"iTunesMusic", "iTunes Music", tweet)
 tweet = re.sub(r"OffensiveContent", "Offensive Content", tweet)
 tweet = re.sub(r"WorstSummerJob", "Worst Summer Job", tweet)
 tweet = re.sub(r"HarryBeCareful", "Harry Be Careful", tweet)
 tweet = re.sub(r"NASASolarSystem", "NASA Solar System", tweet)
 tweet = re.sub(r"animalrescue", "animal rescue", tweet)
 tweet = re.sub(r"KurtSchlichter", "Kurt Schlichter", tweet)
 tweet = re.sub(r"aRmageddon", "armageddon", tweet)
 tweet = re.sub(r"Throwingknifes", "Throwing knives", tweet)
 tweet = re.sub(r"GodsLove", "God's Love", tweet)
 tweet = re.sub(r"bookboost", "book boost", tweet)
 tweet = re.sub(r"ibooklove", "I book love", tweet)
 tweet = re.sub(r"NestleIndia", "Nestle India", tweet)
 tweet = re.sub(r"realDonaldTrump", "Donald Trump", tweet)
 tweet = re.sub(r"DavidVonderhaar", "David Vonderhaar", tweet)
 tweet = re.sub(r"CecilTheLion", "Cecil The Lion", tweet)
 tweet = re.sub(r"weathernetwork", "weather network", tweet)
 tweet = re.sub(r"withBioterrorism&use", "with Bioterrorism & use", tweet)
 tweet = re.sub(r"Hostage&2", "Hostage & 2", tweet)
 tweet = re.sub(r"GOPDebate", "GOP Debate", tweet)
 tweet = re.sub(r"RickPerry", "Rick Perry", tweet)
 tweet = re.sub(r"frontpage", "front page", tweet)
 tweet = re.sub(r"NewsInTweets", "News In Tweets", tweet)
 tweet = re.sub(r"ViralSpell", "Viral Spell", tweet)
 tweet = re.sub(r"til_now", "until now", tweet)
 tweet = re.sub(r"volcanoinRussia", "volcano in Russia", tweet)
 tweet = re.sub(r"ZippedNews", "Zipped News", tweet)
 tweet = re.sub(r"MicheleBachman", "Michele Bachman", tweet)
 tweet = re.sub(r"53inch", "53 inch", tweet)
 tweet = re.sub(r"KerrickTrial", "Kerrick Trial", tweet)
 tweet = re.sub(r"abstorm", "Alberta Storm", tweet)
 tweet = re.sub(r"Beyhive", "Beyonce hive", tweet)
 tweet = re.sub(r"IDFire", "Idaho Fire", tweet)
 tweet = re.sub(r"DETECTADO", "Detected", tweet)
 tweet = re.sub(r"RockyFire", "Rocky Fire", tweet)
 tweet = re.sub(r"Listen/Buy", "Listen / Buy", tweet)
 tweet = re.sub(r"NickCannon", "Nick Cannon", tweet)
 tweet = re.sub(r"FaroeIslands", "Faroe Islands", tweet)
 tweet = re.sub(r"yycstorm", "Calgary Storm", tweet)
 tweet = re.sub(r"IDPs:", "Internally Displaced People :", tweet)
 tweet = re.sub(r"ArtistsUnited", "Artists United", tweet)
 tweet = re.sub(r"ClaytonBryant", "Clayton Bryant", tweet)
 tweet = re.sub(r"jimmyfallon", "jimmy fallon", tweet)
 tweet = re.sub(r"justinbieber", "justin bieber", tweet)
 tweet = re.sub(r"UTC2015", "UTC 2015", tweet)
 tweet = re.sub(r"Time2015", "Time 2015", tweet)
 tweet = re.sub(r"djicemoon", "dj icemoon", tweet)
 tweet = re.sub(r"LivingSafely", "Living Safely", tweet)
 tweet = re.sub(r"FIFA16", "Fifa 2016", tweet)
 tweet = re.sub(r"thisiswhywecanthavenicethings", "this is why we cannot have nice things", tweet)
 tweet = re.sub(r"bbcnews", "bbc news", tweet)
 tweet = re.sub(r"UndergroundRailraod", "Underground Railraod", tweet)
 tweet = re.sub(r"c4news", "c4 news", tweet)
 tweet = re.sub(r"OBLITERATION", "obliteration", tweet)
 tweet = re.sub(r"MUDSLIDE", "mudslide", tweet)
 tweet = re.sub(r"NoSurrender", "No Surrender", tweet)
 tweet = re.sub(r"NotExplained", "Not Explained", tweet)
 tweet = re.sub(r"greatbritishbakeoff", "great british bake off", tweet)
 tweet = re.sub(r"LondonFire", "London Fire", tweet)
 tweet = re.sub(r"KOTAWeather", "KOTA Weather", tweet)
 tweet = re.sub(r"LuchaUnderground", "Lucha Underground", tweet)
 tweet = re.sub(r"KOIN6News", "KOIN 6 News", tweet)
 tweet = re.sub(r"LiveOnK2", "Live On K2", tweet)
 tweet = re.sub(r"9NewsGoldCoast", "9 News Gold Coast", tweet)
 tweet = re.sub(r"nikeplus", "nike plus", tweet)
 tweet = re.sub(r"david_cameron", "David Cameron", tweet)
 tweet = re.sub(r"peterjukes", "Peter Jukes", tweet)
 tweet = re.sub(r"JamesMelville", "James Melville", tweet)
 tweet = re.sub(r"megynkelly", "Megyn Kelly", tweet)
 tweet = re.sub(r"cnewslive", "C News Live", tweet)
 tweet = re.sub(r"JamaicaObserver", "Jamaica Observer", tweet)
 tweet = re.sub(r"TweetLikeItsSeptember11th2001", "Tweet like it is september 11th 2001", tweet)
 tweet = re.sub(r"cbplawyers", "cbp lawyers", tweet)
 tweet = re.sub(r"fewmoretweets", "few more tweets", tweet)
 tweet = re.sub(r"BlackLivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"cjoyner", "Chris Joyner", tweet)
 tweet = re.sub(r"ENGvAUS", "England vs Australia", tweet)
 tweet = re.sub(r"ScottWalker", "Scott Walker", tweet)
 tweet = re.sub(r"MikeParrActor", "Michael Parr", tweet)
 tweet = re.sub(r"4PlayThursdays", "Foreplay Thursdays", tweet)
 tweet = re.sub(r"TGF2015", "Tontitown Grape Festival", tweet)
 tweet = re.sub(r"realmandyrain", "Mandy Rain", tweet)
 tweet = re.sub(r"GraysonDolan", "Grayson Dolan", tweet)
 tweet = re.sub(r"ApolloBrown", "Apollo Brown", tweet)
 tweet = re.sub(r"saddlebrooke", "Saddlebrooke", tweet)
 tweet = re.sub(r"TontitownGrape", "Tontitown Grape", tweet)
 tweet = re.sub(r"AbbsWinston", "Abbs Winston", tweet)
 tweet = re.sub(r"ShaunKing", "Shaun King", tweet)
 tweet = re.sub(r"MeekMill", "Meek Mill", tweet)
 tweet = re.sub(r"TornadoGiveaway", "Tornado Giveaway", tweet)
 tweet = re.sub(r"GRupdates", "GR updates", tweet)
 tweet = re.sub(r"SouthDowns", "South Downs", tweet)
 tweet = re.sub(r"braininjury", "brain injury", tweet)
 tweet = re.sub(r"auspol", "Australian politics", tweet)
 tweet = re.sub(r"PlannedParenthood", "Planned Parenthood", tweet)
 tweet = re.sub(r"calgaryweather", "Calgary Weather", tweet)
 tweet = re.sub(r"weallheartonedirection", "we all heart one direction", tweet)
 tweet = re.sub(r"edsheeran", "Ed Sheeran", tweet)
 tweet = re.sub(r"TrueHeroes", "True Heroes", tweet)
 tweet = re.sub(r"S3XLEAK", "sex leak", tweet)
 tweet = re.sub(r"ComplexMag", "Complex Magazine", tweet)
 tweet = re.sub(r"TheAdvocateMag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"CityofCalgary", "City of Calgary", tweet)
 tweet = re.sub(r"EbolaOutbreak", "Ebola Outbreak", tweet)
 tweet = re.sub(r"SummerFate", "Summer Fate", tweet)
 tweet = re.sub(r"RAmag", "Royal Academy Magazine", tweet)
 tweet = re.sub(r"offers2go", "offers to go", tweet)
 tweet = re.sub(r"foodscare", "food scare", tweet)
 tweet = re.sub(r"MNPDNashville", "Metropolitan Nashville Police Department", tweet)
 tweet = re.sub(r"TfLBusAlerts", "TfL Bus Alerts", tweet)
 tweet = re.sub(r"GamerGate", "Gamer Gate", tweet)
 tweet = re.sub(r"IHHen", "Humanitarian Relief", tweet)
 tweet = re.sub(r"spinningbot", "spinning bot", tweet)
 tweet = re.sub(r"ModiMinistry", "Modi Ministry", tweet)
 tweet = re.sub(r"TAXIWAYS", "taxi ways", tweet)
 tweet = re.sub(r"Calum5SOS", "Calum Hood", tweet)
 tweet = re.sub(r"po_st", "po.st", tweet)
 tweet = re.sub(r"scoopit", "scoop.it", tweet)
 tweet = re.sub(r"UltimaLucha", "Ultima Lucha", tweet)
 tweet = re.sub(r"JonathanFerrell", "Jonathan Ferrell", tweet)
 tweet = re.sub(r"aria_ahrary", "Aria Ahrary", tweet)
 tweet = re.sub(r"rapidcity", "Rapid City", tweet)
 tweet = re.sub(r"OutBid", "outbid", tweet)
 tweet = re.sub(r"lavenderpoetrycafe", "lavender poetry cafe", tweet)
 tweet = re.sub(r"EudryLantiqua", "Eudry Lantiqua", tweet)
 tweet = re.sub(r"15PM", "15 PM", tweet)
 tweet = re.sub(r"OriginalFunko", "Funko", tweet)
 tweet = re.sub(r"rightwaystan", "Richard Tan", tweet)
 tweet = re.sub(r"CindyNoonan", "Cindy Noonan", tweet)
 tweet = re.sub(r"RT_America", "RT America", tweet)
 tweet = re.sub(r"narendramodi", "Narendra Modi", tweet)
 tweet = re.sub(r"BakeOffFriends", "Bake Off Friends", tweet)
 tweet = re.sub(r"TeamHendrick", "Hendrick Motorsports", tweet)
 tweet = re.sub(r"alexbelloli", "Alex Belloli", tweet)
 tweet = re.sub(r"itsjustinstuart", "Justin Stuart", tweet)
 tweet = re.sub(r"gunsense", "gun sense", tweet)
 tweet = re.sub(r"DebateQuestionsWeWantToHear", "debate questions we want to hear", tweet)
 tweet = re.sub(r"RoyalCarribean", "Royal Carribean", tweet)
 tweet = re.sub(r"samanthaturne19", "Samantha Turner", tweet)
 tweet = re.sub(r"JonVoyage", "Jon Stewart", tweet)
 tweet = re.sub(r"renew911health", "renew 911 health", tweet)
 tweet = re.sub(r"SuryaRay", "Surya Ray", tweet)
 tweet = re.sub(r"pattonoswalt", "Patton Oswalt", tweet)
 tweet = re.sub(r"minhazmerchant", "Minhaz Merchant", tweet)
 tweet = re.sub(r"TLVFaces", "Israel Diaspora Coalition", tweet)
 tweet = re.sub(r"pmarca", "Marc Andreessen", tweet)
 tweet = re.sub(r"pdx911", "Portland Police", tweet)
 tweet = re.sub(r"jamaicaplain", "Jamaica Plain", tweet)
 tweet = re.sub(r"Japton", "Arkansas", tweet)
 tweet = re.sub(r"RouteComplex", "Route Complex", tweet)
 tweet = re.sub(r"INSubcontinent", "Indian Subcontinent", tweet)
 tweet = re.sub(r"NJTurnpike", "New Jersey Turnpike", tweet)
 tweet = re.sub(r"Politifiact", "PolitiFact", tweet)
 tweet = re.sub(r"Hiroshima70", "Hiroshima", tweet)
 tweet = re.sub(r"GMMBC", "Greater Mt Moriah Baptist Church", tweet)
 tweet = re.sub(r"versethe", "verse the", tweet)
 tweet = re.sub(r"TubeStrike", "Tube Strike", tweet)
 tweet = re.sub(r"MissionHills", "Mission Hills", tweet)
 tweet = re.sub(r"ProtectDenaliWolves", "Protect Denali Wolves", tweet)
 tweet = re.sub(r"NANKANA", "Nankana", tweet)
 tweet = re.sub(r"SAHIB", "Sahib", tweet)
 tweet = re.sub(r"PAKPATTAN", "Pakpattan", tweet)
 tweet = re.sub(r"Newz_Sacramento", "News Sacramento", tweet)
 tweet = re.sub(r"gofundme", "go fund me", tweet)
 tweet = re.sub(r"pmharper", "Stephen Harper", tweet)
 tweet = re.sub(r"IvanBerroa", "Ivan Berroa", tweet)
 tweet = re.sub(r"LosDelSonido", "Los Del Sonido", tweet)
 tweet = re.sub(r"bancodeseries", "banco de series", tweet)
 tweet = re.sub(r"timkaine", "Tim Kaine", tweet)
 tweet = re.sub(r"IdentityTheft", "Identity Theft", tweet)
 tweet = re.sub(r"AllLivesMatter", "All Lives Matter", tweet)
 tweet = re.sub(r"mishacollins", "Misha Collins", tweet)
 tweet = re.sub(r"BillNeelyNBC", "Bill Neely", tweet)
 tweet = re.sub(r"BeClearOnCancer", "be clear on cancer", tweet)
 tweet = re.sub(r"Kowing", "Knowing", tweet)
 tweet = re.sub(r"ScreamQueens", "Scream Queens", tweet)
 tweet = re.sub(r"AskCharley", "Ask Charley", tweet)
 tweet = re.sub(r"BlizzHeroes", "Heroes of the Storm", tweet)
 tweet = re.sub(r"BradleyBrad47", "Bradley Brad", tweet)
 tweet = re.sub(r"HannaPH", "Typhoon Hanna", tweet)
 tweet = re.sub(r"meinlcymbals", "MEINL Cymbals", tweet)
 tweet = re.sub(r"Ptbo", "Peterborough", tweet)
 tweet = re.sub(r"cnnbrk", "CNN Breaking News", tweet)
 tweet = re.sub(r"IndianNews", "Indian News", tweet)
 tweet = re.sub(r"savebees", "save bees", tweet)
 tweet = re.sub(r"GreenHarvard", "Green Harvard", tweet)
 tweet = re.sub(r"StandwithPP", "Stand with planned parenthood", tweet)
 tweet = re.sub(r"hermancranston", "Herman Cranston", tweet)
 tweet = re.sub(r"WMUR9", "WMUR-TV", tweet)
 tweet = re.sub(r"RockBottomRadFM", "Rock Bottom Radio", tweet)
 tweet = re.sub(r"ameenshaikh3", "Ameen Shaikh", tweet)
 tweet = re.sub(r"ProSyn", "Project Syndicate", tweet)
 tweet = re.sub(r"Daesh", "ISIS", tweet)
 tweet = re.sub(r"s2g", "swear to god", tweet)
 tweet = re.sub(r"listenlive", "listen live", tweet)
 tweet = re.sub(r"CDCgov", "Centers for Disease Control and Prevention", tweet)
 tweet = re.sub(r"FoxNew", "Fox News", tweet)
 tweet = re.sub(r"CBSBigBrother", "Big Brother", tweet)
 tweet = re.sub(r"JulieDiCaro", "Julie DiCaro", tweet)
 tweet = re.sub(r"theadvocatemag", "The Advocate Magazine", tweet)
 tweet = re.sub(r"RohnertParkDPS", "Rohnert Park Police Department", tweet)
 tweet = re.sub(r"THISIZBWRIGHT", "Bonnie Wright", tweet)
 tweet = re.sub(r"Popularmmos", "Popular MMOs", tweet)
 tweet = re.sub(r"WildHorses", "Wild Horses", tweet)
 tweet = re.sub(r"FantasticFour", "Fantastic Four", tweet)
 tweet = re.sub(r"HORNDALE", "Horndale", tweet)
 tweet = re.sub(r"PINER", "Piner", tweet)
 tweet = re.sub(r"BathAndNorthEastSomerset", "Bath and North East Somerset", tweet)
 tweet = re.sub(r"thatswhatfriendsarefor", "that is what friends are for", tweet)
 tweet = re.sub(r"residualincome", "residual income", tweet)
 tweet = re.sub(r"YahooNewsDigest", "Yahoo News Digest", tweet)
 tweet = re.sub(r"MalaysiaAirlines", "Malaysia Airlines", tweet)
 tweet = re.sub(r"AmazonDeals", "Amazon Deals", tweet)
 tweet = re.sub(r"MissCharleyWebb", "Charley Webb", tweet)
 tweet = re.sub(r"shoalstraffic", "shoals traffic", tweet)
 tweet = re.sub(r"GeorgeFoster72", "George Foster", tweet)
 tweet = re.sub(r"pop2015", "pop 2015", tweet)
 tweet = re.sub(r"_PokemonCards_", "Pokemon Cards", tweet)
 tweet = re.sub(r"DianneG", "Dianne Gallagher", tweet)
 tweet = re.sub(r"KashmirConflict", "Kashmir Conflict", tweet)
 tweet = re.sub(r"BritishBakeOff", "British Bake Off", tweet)
 tweet = re.sub(r"FreeKashmir", "Free Kashmir", tweet)
 tweet = re.sub(r"mattmosley", "Matt Mosley", tweet)
 tweet = re.sub(r"BishopFred", "Bishop Fred", tweet)
 tweet = re.sub(r"EndConflict", "End Conflict", tweet)
 tweet = re.sub(r"EndOccupation", "End Occupation", tweet)
 tweet = re.sub(r"UNHEALED", "unhealed", tweet)
 tweet = re.sub(r"CharlesDagnall", "Charles Dagnall", tweet)
 tweet = re.sub(r"Latestnews", "Latest news", tweet)
 tweet = re.sub(r"KindleCountdown", "Kindle Countdown", tweet)
 tweet = re.sub(r"NoMoreHandouts", "No More Handouts", tweet)
 tweet = re.sub(r"datingtips", "dating tips", tweet)
 tweet = re.sub(r"charlesadler", "Charles Adler", tweet)
 tweet = re.sub(r"twia", "Texas Windstorm Insurance Association", tweet)
 tweet = re.sub(r"txlege", "Texas Legislature", tweet)
 tweet = re.sub(r"WindstormInsurer", "Windstorm Insurer", tweet)
 tweet = re.sub(r"Newss", "News", tweet)
 tweet = re.sub(r"hempoil", "hemp oil", tweet)
 tweet = re.sub(r"CommoditiesAre", "Commodities are", tweet)
 tweet = re.sub(r"tubestrike", "tube strike", tweet)
 tweet = re.sub(r"JoeNBC", "Joe Scarborough", tweet)
 tweet = re.sub(r"LiteraryCakes", "Literary Cakes", tweet)
 tweet = re.sub(r"TI5", "The International 5", tweet)
 tweet = re.sub(r"thehill", "the hill", tweet)
 tweet = re.sub(r"3others", "3 others", tweet)
 tweet = re.sub(r"stighefootball", "Sam Tighe", tweet)
 tweet = re.sub(r"whatstheimportantvideo", "what is the important video", tweet)
 tweet = re.sub(r"ClaudioMeloni", "Claudio Meloni", tweet)
 tweet = re.sub(r"DukeSkywalker", "Duke Skywalker", tweet)
 tweet = re.sub(r"carsonmwr", "Fort Carson", tweet)
 tweet = re.sub(r"offdishduty", "off dish duty", tweet)
 tweet = re.sub(r"andword", "and word", tweet)
 tweet = re.sub(r"rhodeisland", "Rhode Island", tweet)
 tweet = re.sub(r"easternoregon", "Eastern Oregon", tweet)
 tweet = re.sub(r"WAwildfire", "Washington Wildfire", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"57am", "57 am", tweet)
 tweet = re.sub(r"fingerrockfire", "Finger Rock Fire", tweet)
 tweet = re.sub(r"JacobHoggard", "Jacob Hoggard", tweet)
 tweet = re.sub(r"newnewnew", "new new new", tweet)
 tweet = re.sub(r"under50", "under 50", tweet)
 tweet = re.sub(r"getitbeforeitsgone", "get it before it is gone", tweet)
 tweet = re.sub(r"freshoutofthebox", "fresh out of the box", tweet)
 tweet = re.sub(r"amwriting", "am writing", tweet)
 tweet = re.sub(r"Bokoharm", "Boko Haram", tweet)
 tweet = re.sub(r"Nowlike", "Now like", tweet)
 tweet = re.sub(r"seasonfrom", "season from", tweet)
 tweet = re.sub(r"epicente", "epicenter", tweet)
 tweet = re.sub(r"epicenterr", "epicenter", tweet)
 tweet = re.sub(r"sicklife", "sick life", tweet)
 tweet = re.sub(r"yycweather", "Calgary Weather", tweet)
 tweet = re.sub(r"calgarysun", "Calgary Sun", tweet)
 tweet = re.sub(r"approachng", "approaching", tweet)
 tweet = re.sub(r"evng", "evening", tweet)
 tweet = re.sub(r"Sumthng", "something", tweet)
 tweet = re.sub(r"EllenPompeo", "Ellen Pompeo", tweet)
 tweet = re.sub(r"shondarhimes", "Shonda Rhimes", tweet)
 tweet = re.sub(r"ABCNetwork", "ABC Network", tweet)
 tweet = re.sub(r"SushmaSwaraj", "Sushma Swaraj", tweet)
 tweet = re.sub(r"pray4japan", "Pray for Japan", tweet)
 tweet = re.sub(r"hope4japan", "Hope for Japan", tweet)
 tweet = re.sub(r"Illusionimagess", "Illusion images", tweet)
 tweet = re.sub(r"SummerUnderTheStars", "Summer Under The Stars", tweet)
 tweet = re.sub(r"ShallWeDance", "Shall We Dance", tweet)
 tweet = re.sub(r"TCMParty", "TCM Party", tweet)
 tweet = re.sub(r"marijuananews", "marijuana news", tweet)
 tweet = re.sub(r"onbeingwithKristaTippett", "on being with Krista Tippett", tweet)
 tweet = re.sub(r"Beingtweets", "Being tweets", tweet)
 tweet = re.sub(r"newauthors", "new authors", tweet)
 tweet = re.sub(r"remedyyyy", "remedy", tweet)
 tweet = re.sub(r"44PM", "44 PM", tweet)
 tweet = re.sub(r"HeadlinesApp", "Headlines App", tweet)
 tweet = re.sub(r"40PM", "40 PM", tweet)
 tweet = re.sub(r"myswc", "Severe Weather Center", tweet)
 tweet = re.sub(r"ithats", "that is", tweet)
 tweet = re.sub(r"icouldsitinthismomentforever", "I could sit in this moment forever", tweet)
 tweet = re.sub(r"FatLoss", "Fat Loss", tweet)
 tweet = re.sub(r"02PM", "02 PM", tweet)
 tweet = re.sub(r"MetroFmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"Bstrd", "bastard", tweet)
 tweet = re.sub(r"bldy", "bloody", tweet)
 tweet = re.sub(r"MetrofmTalk", "Metro Fm Talk", tweet)
 tweet = re.sub(r"terrorismturn", "terrorism turn", tweet)
 tweet = re.sub(r"BBCNewsAsia", "BBC News Asia", tweet)
 tweet = re.sub(r"BehindTheScenes", "Behind The Scenes", tweet)
 tweet = re.sub(r"GeorgeTakei", "George Takei", tweet)
 tweet = re.sub(r"WomensWeeklyMag", "Womens Weekly Magazine", tweet)
 tweet = re.sub(r"SurvivorsGuidetoEarth", "Survivors Guide to Earth", tweet)
 tweet = re.sub(r"incubusband", "incubus band", tweet)
 tweet = re.sub(r"Babypicturethis", "Baby picture this", tweet)
 tweet = re.sub(r"BombEffects", "Bomb Effects", tweet)
 tweet = re.sub(r"win10", "Windows 10", tweet)
 tweet = re.sub(r"idkidk", "I do not know I do not know", tweet)
 tweet = re.sub(r"TheWalkingDead", "The Walking Dead", tweet)
 tweet = re.sub(r"amyschumer", "Amy Schumer", tweet)
 tweet = re.sub(r"crewlist", "crew list", tweet)
 tweet = re.sub(r"Erdogans", "Erdogan", tweet)
 tweet = re.sub(r"BBCLive", "BBC Live", tweet)
 tweet = re.sub(r"TonyAbbottMHR", "Tony Abbott", tweet)
 tweet = re.sub(r"paulmyerscough", "Paul Myerscough", tweet)
 tweet = re.sub(r"georgegallagher", "George Gallagher", tweet)
 tweet = re.sub(r"JimmieJohnson", "Jimmie Johnson", tweet)
 tweet = re.sub(r"pctool", "pc tool", tweet)
 tweet = re.sub(r"DoingHashtagsRight", "Doing Hashtags Right", tweet)
 tweet = re.sub(r"ThrowbackThursday", "Throwback Thursday", tweet)
 tweet = re.sub(r"SnowBackSunday", "Snowback Sunday", tweet)
 tweet = re.sub(r"LakeEffect", "Lake Effect", tweet)
 tweet = re.sub(r"RTphotographyUK", "Richard Thomas Photography UK", tweet)
 tweet = re.sub(r"BigBang_CBS", "Big Bang CBS", tweet)
 tweet = re.sub(r"writerslife", "writers life", tweet)
 tweet = re.sub(r"NaturalBirth", "Natural Birth", tweet)
 tweet = re.sub(r"UnusualWords", "Unusual Words", tweet)
 tweet = re.sub(r"wizkhalifa", "Wiz Khalifa", tweet)
 tweet = re.sub(r"acreativedc", "a creative DC", tweet)
 tweet = re.sub(r"vscodc", "vsco DC", tweet)
 tweet = re.sub(r"VSCOcam", "vsco camera", tweet)
 tweet = re.sub(r"TheBEACHDC", "The beach DC", tweet)
 tweet = re.sub(r"buildingmuseum", "building museum", tweet)
 tweet = re.sub(r"WorldOil", "World Oil", tweet)
 tweet = re.sub(r"redwedding", "red wedding", tweet)
 tweet = re.sub(r"AmazingRaceCanada", "Amazing Race Canada", tweet)
 tweet = re.sub(r"WakeUpAmerica", "Wake Up America", tweet)
 tweet = re.sub(r"\\Allahuakbar\", "Allahu Akbar", tweet)
 tweet = re.sub(r"bleased", "blessed", tweet)
 tweet = re.sub(r"nigeriantribune", "Nigerian Tribune", tweet)
 tweet = re.sub(r"HIDEO_KOJIMA_EN", "Hideo Kojima", tweet)
 tweet = re.sub(r"FusionFestival", "Fusion Festival", tweet)
 tweet = re.sub(r"50Mixed", "50 Mixed", tweet)
 tweet = re.sub(r"NoAgenda", "No Agenda", tweet)
 tweet = re.sub(r"WhiteGenocide", "White Genocide", tweet)
 tweet = re.sub(r"dirtylying", "dirty lying", tweet)
 tweet = re.sub(r"SyrianRefugees", "Syrian Refugees", tweet)
 tweet = re.sub(r"changetheworld", "change the world", tweet)
 tweet = re.sub(r"Ebolacase", "Ebola case", tweet)
 tweet = re.sub(r"mcgtech", "mcg technologies", tweet)
 tweet = re.sub(r"withweapons", "with weapons", tweet)
 tweet = re.sub(r"advancedwarfare", "advanced warfare", tweet)
 tweet = re.sub(r"letsFootball", "let us Football", tweet)
 tweet = re.sub(r"LateNiteMix", "late night mix", tweet)
 tweet = re.sub(r"PhilCollinsFeed", "Phil Collins", tweet)
 tweet = re.sub(r"RudyHavenstein", "Rudy Havenstein", tweet)
 tweet = re.sub(r"22PM", "22 PM", tweet)
 tweet = re.sub(r"54am", "54 AM", tweet)
 tweet = re.sub(r"38am", "38 AM", tweet)
 tweet = re.sub(r"OldFolkExplainStuff", "Old Folk Explain Stuff", tweet)
 tweet = re.sub(r"BlacklivesMatter", "Black Lives Matter", tweet)
 tweet = re.sub(r"InsaneLimits", "Insane Limits", tweet)
 tweet = re.sub(r"youcantsitwithus", "you cannot sit with us", tweet)
 tweet = re.sub(r"2k15", "2015", tweet)
 tweet = re.sub(r"TheIran", "Iran", tweet)
 tweet = re.sub(r"JimmyFallon", "Jimmy Fallon", tweet)
 tweet = re.sub(r"AlbertBrooks", "Albert Brooks", tweet)
 tweet = re.sub(r"defense_news", "defense news", tweet)
 tweet = re.sub(r"nuclearrcSA", "Nuclear Risk Control Self Assessment", tweet)
 tweet = re.sub(r"Auspol", "Australia Politics", tweet)
 tweet = re.sub(r"NuclearPower", "Nuclear Power", tweet)
 tweet = re.sub(r"WhiteTerrorism", "White Terrorism", tweet)
 tweet = re.sub(r"truthfrequencyradio", "Truth Frequency Radio", tweet)
 tweet = re.sub(r"ErasureIsNotEquality", "Erasure is not equality", tweet)
 tweet = re.sub(r"ProBonoNews", "Pro Bono News", tweet)
 tweet = re.sub(r"JakartaPost", "Jakarta Post", tweet)
 tweet = re.sub(r"toopainful", "too painful", tweet)
 tweet = re.sub(r"melindahaunton", "Melinda Haunton", tweet)
 tweet = re.sub(r"NoNukes", "No Nukes", tweet)
 tweet = re.sub(r"curryspcworld", "Currys PC World", tweet)
 tweet = re.sub(r"ineedcake", "I need cake", tweet)
 tweet = re.sub(r"blackforestgateau", "black forest gateau", tweet)
 tweet = re.sub(r"BBCOne", "BBC One", tweet)
 tweet = re.sub(r"AlexxPage", "Alex Page", tweet)
 tweet = re.sub(r"jonathanserrie", "Jonathan Serrie", tweet)
 tweet = re.sub(r"SocialJerkBlog", "Social Jerk Blog", tweet)
 tweet = re.sub(r"ChelseaVPeretti", "Chelsea Peretti", tweet)
 tweet = re.sub(r"irongiant", "iron giant", tweet)
 tweet = re.sub(r"RonFunches", "Ron Funches", tweet)
 tweet = re.sub(r"TimCook", "Tim Cook", tweet)
 tweet = re.sub(r"sebastianstanisaliveandwell", "Sebastian Stan is alive and well", tweet)
 tweet = re.sub(r"Madsummer", "Mad summer", tweet)
 tweet = re.sub(r"NowYouKnow", "Now you know", tweet)
 tweet = re.sub(r"concertphotography", "concert photography", tweet)
 tweet = re.sub(r"TomLandry", "Tom Landry", tweet)
 tweet = re.sub(r"showgirldayoff", "show girl day off", tweet)
 tweet = re.sub(r"Yougslavia", "Yugoslavia", tweet)
 tweet = re.sub(r"QuantumDataInformatics", "Quantum Data Informatics", tweet)
 tweet = re.sub(r"FromTheDesk", "From The Desk", tweet)
 tweet = re.sub(r"TheaterTrial", "Theater Trial", tweet)
 tweet = re.sub(r"CatoInstitute", "Cato Institute", tweet)
 tweet = re.sub(r"EmekaGift", "Emeka Gift", tweet)
 tweet = re.sub(r"LetsBe_Rational", "Let us be rational", tweet)
 tweet = re.sub(r"Cynicalreality", "Cynical reality", tweet)
 tweet = re.sub(r"FredOlsenCruise", "Fred Olsen Cruise", tweet)
 tweet = re.sub(r"NotSorry", "not sorry", tweet)
 tweet = re.sub(r"UseYourWords", "use your words", tweet)
 tweet = re.sub(r"WordoftheDay", "word of the day", tweet)
 tweet = re.sub(r"Dictionarycom", "Dictionary.com", tweet)
 tweet = re.sub(r"TheBrooklynLife", "The Brooklyn Life", tweet)
 tweet = re.sub(r"jokethey", "joke they", tweet)
 tweet = re.sub(r"nflweek1picks", "NFL week 1 picks", tweet)
 tweet = re.sub(r"uiseful", "useful", tweet)
 tweet = re.sub(r"JusticeDotOrg", "The American Association for Justice", tweet)
 tweet = re.sub(r"autoaccidents", "auto accidents", tweet)
 tweet = re.sub(r"SteveGursten", "Steve Gursten", tweet)
 tweet = re.sub(r"MichiganAutoLaw", "Michigan Auto Law", tweet)
 tweet = re.sub(r"birdgang", "bird gang", tweet)
 tweet = re.sub(r"nflnetwork", "NFL Network", tweet)
 tweet = re.sub(r"NYDNSports", "NY Daily News Sports", tweet)
 tweet = re.sub(r"RVacchianoNYDN", "Ralph Vacchiano NY Daily News", tweet)
 tweet = re.sub(r"EdmontonEsks", "Edmonton Eskimos", tweet)
 tweet = re.sub(r"david_brelsford", "David Brelsford", tweet)
 tweet = re.sub(r"TOI_India", "The Times of India", tweet)
 tweet = re.sub(r"hegot", "he got", tweet)
 tweet = re.sub(r"SkinsOn9", "Skins on 9", tweet)
 tweet = re.sub(r"sothathappened", "so that happened", tweet)
 tweet = re.sub(r"LCOutOfDoors", "LC Out Of Doors", tweet)
 tweet = re.sub(r"NationFirst", "Nation First", tweet)
 tweet = re.sub(r"IndiaToday", "India Today", tweet)
 tweet = re.sub(r"HLPS", "helps", tweet)
 tweet = re.sub(r"HOSTAGESTHROSW", "hostages throw", tweet)
 tweet = re.sub(r"SNCTIONS", "sanctions", tweet)
 tweet = re.sub(r"BidTime", "Bid Time", tweet)
 tweet = re.sub(r"crunchysensible", "crunchy sensible", tweet)
 tweet = re.sub(r"RandomActsOfRomance", "Random acts of romance", tweet)
 tweet = re.sub(r"MomentsAtHill", "Moments at hill", tweet)
 tweet = re.sub(r"eatshit", "eat shit", tweet)
 tweet = re.sub(r"liveleakfun", "live leak fun", tweet)
 tweet = re.sub(r"SahelNews", "Sahel News", tweet)
 tweet = re.sub(r"abc7newsbayarea", "ABC 7 News Bay Area", tweet)
 tweet = re.sub(r"facilitiesmanagement", "facilities management", tweet)
 tweet = re.sub(r"facilitydude", "facility dude", tweet)
 tweet = re.sub(r"CampLogistics", "Camp logistics", tweet)
 tweet = re.sub(r"alaskapublic", "Alaska public", tweet)
 tweet = re.sub(r"MarketResearch", "Market Research", tweet)
 tweet = re.sub(r"AccuracyEsports", "Accuracy Esports", tweet)
 tweet = re.sub(r"TheBodyShopAust", "The Body Shop Australia", tweet)
 tweet = re.sub(r"yychail", "Calgary hail", tweet)
 tweet = re.sub(r"yyctraffic", "Calgary traffic", tweet)
 tweet = re.sub(r"eliotschool", "eliot school", tweet)
 tweet = re.sub(r"TheBrokenCity", "The Broken City", tweet)
 tweet = re.sub(r"OldsFireDept", "Olds Fire Department", tweet)
 tweet = re.sub(r"RiverComplex", "River Complex", tweet)
 tweet = re.sub(r"fieldworksmells", "field work smells", tweet)
 tweet = re.sub(r"IranElection", "Iran Election", tweet)
 tweet = re.sub(r"glowng", "glowing", tweet)
 tweet = re.sub(r"kindlng", "kindling", tweet)
 tweet = re.sub(r"riggd", "rigged", tweet)
 tweet = re.sub(r"slownewsday", "slow news day", tweet)
 tweet = re.sub(r"MyanmarFlood", "Myanmar Flood", tweet)
 tweet = re.sub(r"abc7chicago", "ABC 7 Chicago", tweet)
 tweet = re.sub(r"copolitics", "Colorado Politics", tweet)
 tweet = re.sub(r"AdilGhumro", "Adil Ghumro", tweet)
 tweet = re.sub(r"netbots", "net bots", tweet)
 tweet = re.sub(r"byebyeroad", "bye bye road", tweet)
 tweet = re.sub(r"massiveflooding", "massive flooding", tweet)
 tweet = re.sub(r"EndofUS", "End of United States", tweet)
 tweet = re.sub(r"35PM", "35 PM", tweet)
 tweet = re.sub(r"greektheatrela", "Greek Theatre Los Angeles", tweet)
 tweet = re.sub(r"76mins", "76 minutes", tweet)
 tweet = re.sub(r"publicsafetyfirst", "public safety first", tweet)
 tweet = re.sub(r"livesmatter", "lives matter", tweet)
 tweet = re.sub(r"myhometown", "my hometown", tweet)
 tweet = re.sub(r"tankerfire", "tanker fire", tweet)
 tweet = re.sub(r"MEMORIALDAY", "memorial day", tweet)
 tweet = re.sub(r"MEMORIAL_DAY", "memorial day", tweet)
 tweet = re.sub(r"instaxbooty", "instagram booty", tweet)
 tweet = re.sub(r"Jerusalem_Post", "Jerusalem Post", tweet)
 tweet = re.sub(r"WayneRooney_INA", "Wayne Rooney", tweet)
 tweet = re.sub(r"VirtualReality", "Virtual Reality", tweet)
 tweet = re.sub(r"OculusRift", "Oculus Rift", tweet)
 tweet = re.sub(r"OwenJones84", "Owen Jones", tweet)
 tweet = re.sub(r"jeremycorbyn", "Jeremy Corbyn", tweet)
 tweet = re.sub(r"paulrogers002", "Paul Rogers", tweet)
 tweet = re.sub(r"mortalkombatx", "Mortal Kombat X", tweet)
 tweet = re.sub(r"mortalkombat", "Mortal Kombat", tweet)
 tweet = re.sub(r"FilipeCoelho92", "Filipe Coelho", tweet)
 tweet = re.sub(r"OnlyQuakeNews", "Only Quake News", tweet)
 tweet = re.sub(r"kostumes", "costumes", tweet)
 tweet = re.sub(r"YEEESSSS", "yes", tweet)
 tweet = re.sub(r"ToshikazuKatayama", "Toshikazu Katayama", tweet)
 tweet = re.sub(r"IntlDevelopment", "Intl Development", tweet)
 tweet = re.sub(r"ExtremeWeather", "Extreme Weather", tweet)
 tweet = re.sub(r"WereNotGruberVoters", "We are not gruber voters", tweet)
 tweet = re.sub(r"NewsThousands", "News Thousands", tweet)
 tweet = re.sub(r"EdmundAdamus", "Edmund Adamus", tweet)
 tweet = re.sub(r"EyewitnessWV", "Eye witness WV", tweet)
 tweet = re.sub(r"PhiladelphiaMuseu", "Philadelphia Museum", tweet)
 tweet = re.sub(r"DublinComicCon", "Dublin Comic Con", tweet)
 tweet = re.sub(r"NicholasBrendon", "Nicholas Brendon", tweet)
 tweet = re.sub(r"Alltheway80s", "All the way 80s", tweet)
 tweet = re.sub(r"FromTheField", "From the field", tweet)
 tweet = re.sub(r"NorthIowa", "North Iowa", tweet)
 tweet = re.sub(r"WillowFire", "Willow Fire", tweet)
 tweet = re.sub(r"MadRiverComplex", "Mad River Complex", tweet)
 tweet = re.sub(r"feelingmanly", "feeling manly", tweet)
 tweet = re.sub(r"stillnotoverit", "still not over it", tweet)
 tweet = re.sub(r"FortitudeValley", "Fortitude Valley", tweet)
 tweet = re.sub(r"CoastpowerlineTramTr", "Coast powerline", tweet)
 tweet = re.sub(r"ServicesGold", "Services Gold", tweet)
 tweet = re.sub(r"NewsbrokenEmergency", "News broken emergency", tweet)
 tweet = re.sub(r"Evaucation", "evacuation", tweet)
 tweet = re.sub(r"leaveevacuateexitbe", "leave evacuate exit be", tweet)
 tweet = re.sub(r"P_EOPLE", "PEOPLE", tweet)
 tweet = re.sub(r"Tubestrike", "tube strike", tweet)
 tweet = re.sub(r"CLASS_SICK", "CLASS SICK", tweet)
 tweet = re.sub(r"localplumber", "local plumber", tweet)
 tweet = re.sub(r"awesomejobsiri", "awesome job siri", tweet)
 tweet = re.sub(r"PayForItHow", "Pay for it how", tweet)
 tweet = re.sub(r"ThisIsAfrica", "This is Africa", tweet)
 tweet = re.sub(r"crimeairnetwork", "crime air network", tweet)
 tweet = re.sub(r"KimAcheson", "Kim Acheson", tweet)
 tweet = re.sub(r"cityofcalgary", "City of Calgary", tweet)
 tweet = re.sub(r"prosyndicate", "pro syndicate", tweet)
 tweet = re.sub(r"660NEWS", "660 NEWS", tweet)
 tweet = re.sub(r"BusInsMagazine", "Business Insurance Magazine", tweet)
 tweet = re.sub(r"wfocus", "focus", tweet)
 tweet = re.sub(r"ShastaDam", "Shasta Dam", tweet)
 tweet = re.sub(r"go2MarkFranco", "Mark Franco", tweet)
 tweet = re.sub(r"StephGHinojosa", "Steph Hinojosa", tweet)
 tweet = re.sub(r"Nashgrier", "Nash Grier", tweet)
 tweet = re.sub(r"NashNewVideo", "Nash new video", tweet)
 tweet = re.sub(r"IWouldntGetElectedBecause", "I would not get elected because", tweet)
 tweet = re.sub(r"SHGames", "Sledgehammer Games", tweet)
 tweet = re.sub(r"bedhair", "bed hair", tweet)
 tweet = re.sub(r"JoelHeyman", "Joel Heyman", tweet)
 tweet = re.sub(r"viaYouTube", "via YouTube", tweet)
 return tweet
def clean_1_2_6(tweet):
tweet = re.sub(r"\.{3,}", '...', tweet)
 if '...' not in tweet:
tweet = tweet.replace('.. ', '...')
 return tweet
def clean_1_2_7(tweet):
tweet = re.sub(r"MH370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"mÌ¼sica", "music", tweet)
 tweet = re.sub(r"okwx", "Oklahoma City Weather", tweet)
 tweet = re.sub(r"arwx", "Arkansas Weather", tweet)
 tweet = re.sub(r"gawx", "Georgia Weather", tweet)
 tweet = re.sub(r"scwx", "South Carolina Weather", tweet)
 tweet = re.sub(r"cawx", "California Weather", tweet)
 tweet = re.sub(r"tnwx", "Tennessee Weather", tweet)
 tweet = re.sub(r"azwx", "Arizona Weather", tweet)
 tweet = re.sub(r"alwx", "Alabama Weather", tweet)
 tweet = re.sub(r"wordpressdotcom", "wordpress", tweet)
 tweet = re.sub(r"usNWSgov", "United States National Weather Service", tweet)
 tweet = re.sub(r"Suruc", "Sanliurfa", tweet)
 tweet = re.sub(r"Bestnaijamade", "bestnaijamade", tweet)
 tweet = re.sub(r"SOUDELOR", "Soudelor", tweet)
 tweet = re.sub(r"mh370", "Malaysia Airlines Flight 370", tweet)
 tweet = re.sub(r"legionnaires", "pneumonia", tweet)
 tweet = re.sub(r"derailment", "railway accident", tweet)
 tweet = re.sub(r"inundated", "flood", tweet)
 tweet = re.sub(r"deluged", "flood", tweet)
 tweet = re.sub(r"curfew", "stay at home", tweet)
 tweet = re.sub(r"obliteration", "destruction", tweet)
 tweet = re.sub(r"quarantine", "prevent the spread of disease", tweet)
 tweet = re.sub(r"lol", "laugh", tweet)
 tweet = re.sub(r"obliterate", "destroy", tweet)
 tweet = re.sub(r"hijacking", "seize", tweet)
 tweet = re.sub(r"detonation", "explosion", tweet)
 tweet = re.sub(r"electrocuted", "killed", tweet)
 tweet = re.sub(r"destroyd", "destroyed", tweet)
 return tweet
def clean_1_2_8(tweet):
tweet = re.sub(r"'cause", "because", tweet)
 tweet = re.sub(r"could've", "could have", tweet)
 tweet = re.sub(r"hadn't", "had not", tweet)
 tweet = re.sub(r"he'd", "he would", tweet)
 tweet = re.sub(r"how'd", "how did", tweet)
 tweet = re.sub(r"how'd'y", "how do you", tweet)
 tweet = re.sub(r"how'll", "how will", tweet)
 tweet = re.sub(r"how's", "how is", tweet)
 tweet = re.sub(r"I'd've", "I would have", tweet)
 tweet = re.sub(r"I'll've", "I will have", tweet)
 tweet = re.sub(r"i'd've", "i would have", tweet)
 tweet = re.sub(r"i'll've", "i will have", tweet)
 tweet = re.sub(r"it'd", "it would", tweet)
 tweet = re.sub(r"it'd've", "it would have", tweet)
 tweet = re.sub(r"it'll've", "it will have", tweet)
 tweet = re.sub(r"ma'am", "madam", tweet)
 tweet = re.sub(r"mayn't", "may not", tweet)
 tweet = re.sub(r"might've", "might have", tweet)
 tweet = re.sub(r"mightn't", "might not", tweet)
 tweet = re.sub(r"mightn't've", "might not have", tweet)
 tweet = re.sub(r"must've", "must have", tweet)
 tweet = re.sub(r"mustn't", "must not", tweet)
 tweet = re.sub(r"mustn't've", "must not have", tweet)
 tweet = re.sub(r"needn't", "need not", tweet)
 tweet = re.sub(r"needn't've", "need not have", tweet)
 tweet = re.sub(r"o'clock", "of the clock", tweet)
 tweet = re.sub(r"oughtn't", "ought not", tweet)
 tweet = re.sub(r"oughtn't've", "ought not have", tweet)
 tweet = re.sub(r"shan't", "shall not", tweet)
 tweet = re.sub(r"sha'n't", "shall not", tweet)
 tweet = re.sub(r"shan't've", "shall not have", tweet)
 tweet = re.sub(r"she'd", "she would", tweet)
 tweet = re.sub(r"she'd've", "she would have", tweet)
 tweet = re.sub(r"she'll", "she will", tweet)
 tweet = re.sub(r"she'll've", "she will have", tweet)
 tweet = re.sub(r"she's", "she is", tweet)
 tweet = re.sub(r"shouldn't've", "should not have", tweet)
 tweet = re.sub(r"so've", "so have", tweet)
 tweet = re.sub(r"so's", "so as", tweet)
 tweet = re.sub(r"this's", "this is", tweet)
 tweet = re.sub(r"that'd", "that would", tweet)
 tweet = re.sub(r"that'd've", "that would have", tweet)
 tweet = re.sub(r"there'd", "there would", tweet)
 tweet = re.sub(r"there'd've", "there would have", tweet)
 tweet = re.sub(r"they'd've", "they would have", tweet)
 tweet = re.sub(r"they'll've", "they will have", tweet)
 tweet = re.sub(r"to've", "to have", tweet)
 tweet = re.sub(r"we'd've", "we would have", tweet)
 tweet = re.sub(r"we'll've", "we will have", tweet)
 tweet = re.sub(r"what'll", "what will", tweet)
 tweet = re.sub(r"what'll've", "what will have", tweet)
 tweet = re.sub(r"what're", "what are", tweet)
 tweet = re.sub(r"what've", "what have", tweet)
 tweet = re.sub(r"when's", "when is", tweet)
 tweet = re.sub(r"when've", "when have", tweet)
 tweet = re.sub(r"where'd", "where did", tweet)
 tweet = re.sub(r"where've", "where have", tweet)
 tweet = re.sub(r"who'll", "who will", tweet)
 tweet = re.sub(r"who'll've", "who will have", tweet)
 tweet = re.sub(r"who've", "who have", tweet)
 tweet = re.sub(r"why's", "why is", tweet)
 tweet = re.sub(r"why've", "why have", tweet)
 tweet = re.sub(r"will've", "will have", tweet)
 tweet = re.sub(r"won't've", "will not have", tweet)
 tweet = re.sub(r"wouldn't've", "would not have", tweet)
 tweet = re.sub(r"y'all'd", "you all would", tweet)
 tweet = re.sub(r"y'all'd've", "you all would have", tweet)
 tweet = re.sub(r"y'all're", "you all are", tweet)
 tweet = re.sub(r"y'all've", "you all have", tweet)
 tweet = re.sub(r"you'd've", "you would have", tweet)
 tweet = re.sub(r"you'll've", "you will have", tweet)
 return tweet
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<categorify>
def remove_url(text):
url_pattern = re.compile(r'https?://\S*|www\.\S*')
 return url_pattern.sub(r'URL', text)
 def remove_url_for_labelling(text):
url_pattern = re.compile(r'https?://\S*|www\.\S*')
 return url_pattern.sub(r'', text )
if PSEUDO or VALIDATE:
convertTrainLabel()<find_best_params>
EMOTICONS = {
u":‑\)":"Happy face smiley",
u":\)":"Happy face smiley",
u":-\]":"Happy face smiley",
u":\]":"Happy face smiley",
u":-3":"Happy face smiley",
u":->":"Happy face smiley",
u":>":"Happy face smiley",
u"8-\)":"Happy face smiley",
u":o\)":"Happy face smiley",
u":-\}":"Happy face smiley",
u":\}":"Happy face smiley",
u":-\)":"Happy face smiley",
u":c\)":"Happy face smiley",
u":\^\)":"Happy face smiley",
u"=\]":"Happy face smiley",
u"=\)":"Happy face smiley",
u":‑D":"Laughing, big grin laugh with glasses",
u"8‑D":"Laughing, big grin laugh with glasses",
u"X‑D":"Laughing, big grin laugh with glasses",
u"B\^D":"Laughing, big grin laugh with glasses",
u":-\)\)":"Very happy",
u":‑\(":"Frown, sad, andry pouting",
u":-\(":"Frown, sad, andry pouting",
u":\(":"Frown, sad, andry pouting",
u":‑c":"Frown, sad, andry pouting",
u":‑<":"Frown, sad, andry pouting",
u":<":"Frown, sad, andry pouting",
u":‑\[":"Frown, sad, andry pouting",
u":\[":"Frown, sad, andry pouting",
u":-\|\|":"Frown, sad, andry pouting",
u">:\[":"Frown, sad, andry pouting",
u":\{":"Frown, sad, andry pouting",
u":@":"Frown, sad, andry pouting",
u">:\(":"Frown, sad, andry pouting",
u":'‑\(":"Crying",
u":'\(":"Crying",
u":'‑\)":"Tears of happiness",
u":'\)":"Tears of happiness",
u"D:<":"Disgust",
u":‑O":"Surprise",
u":‑o":"Surprise",
u":-0":"Shock",
u"8‑0":"Yawn",
u">:O":"Yawn",
u":-\*":"Kiss",
u":\*":"Kiss",
u";‑\)":"Wink smirk",
u";\)":"Wink smirk",
u"\*-\)":"Wink smirk",
u"\*\)":"Wink smirk",
u";‑\]":"Wink smirk",
u";\]":"Wink smirk",
u";\^\)":"Wink smirk",
u":‑,":"Wink smirk",
u":‑P":"Tongue sticking out, cheeky, playful blowing a raspberry",
u"X‑P":"Tongue sticking out, cheeky, playful blowing a raspberry",
u":‑Þ":"Tongue sticking out, cheeky, playful blowing a raspberry",
u">:P":"Tongue sticking out, cheeky, playful blowing a raspberry",
u":‑/":"Skeptical, annoyed, undecided, uneasy hesitant",
u":/":"Skeptical, annoyed, undecided, uneasy hesitant",
u":-[.]":"Skeptical, annoyed, undecided, uneasy hesitant",
u">:[(\\\)]":"Skeptical, annoyed, undecided, uneasy hesitant",
u">:/":"Skeptical, annoyed, undecided, uneasy hesitant",
u":[(\\\)]":"Skeptical, annoyed, undecided, uneasy hesitant",
u"=/":"Skeptical, annoyed, undecided, uneasy hesitant",
u"=[(\\\)]":"Skeptical, annoyed, undecided, uneasy hesitant",
u":‑\|":"Straight face",
u":\|":"Straight face",
u":\$":"Embarrassed blushing",
u":‑x":"Sealed lips wearing braces tongue-tied",
u":‑
u":‑&":"Sealed lips wearing braces tongue-tied",
u":&":"Sealed lips wearing braces tongue-tied",
u"O:‑\)":"Angel, saint innocent",
u"O:\)":"Angel, saint innocent",
u"0:‑3":"Angel, saint innocent",
u"0:‑\)":"Angel, saint innocent",
u"0:\)":"Angel, saint innocent",
u":‑b":"Tongue sticking out, cheeky, playful blowing a raspberry",
u"0;\^\)":"Angel, saint innocent",
u">:‑\)":"Evil devilish",
u">:\)":"Evil devilish",
u"\}:‑\)":"Evil devilish",
u"\}:\)":"Evil devilish",
u"3:‑\)":"Evil devilish",
u"3:\)":"Evil devilish",
u">;\)":"Evil devilish",
u"\|;‑\)":"Cool",
u"\|‑O":"Bored",
u":‑J":"Tongue-in-cheek",
u"
u"%‑\)":"Drunk confused",
u":-
u":
u"<:‑\|":"Dump",
u"\(>_<\)":"Troubled",
u"\(>_<\)>":"Troubled",
u"\(';'\)":"Baby",
u"\(\^\^>``":"Nervous Embarrassed Troubled Shy Sweat drop",
u"\(\^_\^;\)":"Nervous Embarrassed Troubled Shy Sweat drop",
u"\(-_-;\)":"Nervous Embarrassed Troubled Shy Sweat drop",
u"\(~_~;\)\(・\.・;\)":"Nervous Embarrassed Troubled Shy Sweat drop",
u"\(-_-\)zzz":"Sleeping",
u"\(\^_-\)":"Wink",
u"\(\(\+_\+\)\)":"Confused",
u"\(\+o\+\)":"Confused",
u"\(o\|o\)":"Ultraman",
u"\^_\^":"Joyful",
u"\(\^_\^\)/":"Joyful",
u"\(\^O\^\)／":"Joyful",
u"\(\^o\^\)／":"Joyful",
u"\(__\)":"Kowtow as a sign of respect, dogeza for apology",
u"_\(\._\.\)_":"Kowtow as a sign of respect, dogeza for apology",
u"<\(_ _\)>":"Kowtow as a sign of respect, dogeza for apology",
u"<m\(__\)m>":"Kowtow as a sign of respect, dogeza for apology",
u"m\(__\)m":"Kowtow as a sign of respect, dogeza for apology",
u"m\(_ _\)m":"Kowtow as a sign of respect, dogeza for apology",
u"\('_'\)":"Sad Crying",
u"\(/_;\)":"Sad Crying",
u"\(T_T\)\(;_;\)":"Sad Crying",
u"\(;_;":"Sad of Crying",
u"\(;_:\)":"Sad Crying",
u"\(;O;\)":"Sad Crying",
u"\(:_;\)":"Sad Crying",
u"\(ToT\)":"Sad Crying",
u";_;":"Sad Crying",
u";-;":"Sad Crying",
u";n;":"Sad Crying",
u"Q\.Q":"Sad Crying",
u"T\.T":"Sad Crying",
u"Q_Q":"Sad Crying",
u"\(-\.-\)":"Shame",
u"\(-_-\)":"Shame",
u"\(一一\)":"Shame",
u"\(；一_一\)":"Shame",
u"\(=_=\)":"Tired",
u"\(=\^\·\^=\)":"cat",
u"\(=\^\·\·\^=\)":"cat",
u"=_\^=	":"cat",
u"\(\.\.\)":"Looking down",
u"\(\._\.\)":"Looking down",
u"\^m\^":"Giggling with hand covering mouth",
u">\^_\^<":"Normal Laugh",
u"<\^!\^>":"Normal Laugh",
u"\^/\^":"Normal Laugh",
u"\（\*\^_\^\*）" :"Normal Laugh",
u"\(\^<\^\)\(\^\.\^\)":"Normal Laugh",
u"\(^\^\)":"Normal Laugh",
u"\(\^\.\^\)":"Normal Laugh",
u"\(\^_\^\.\)":"Normal Laugh",
u"\(\^_\^\)":"Normal Laugh",
u"\(\^\^\)":"Normal Laugh",
u"\(\^J\^\)":"Normal Laugh",
u"\(\*\^\.\^\*\)":"Normal Laugh",
u"\(\^—\^\）":"Normal Laugh",
u"\(
 u"\（\^—\^\）":"Waving",
u"\(;_;\)/~~~":"Waving",
u"\(\^\.\^\)/~~~":"Waving",
u"\(-_-\)/~~~ \($\·\·\)/~~~":"Waving",
u"\(T_T\)/~~~":"Waving",
u"\(ToT\)/~~~":"Waving",
u"\(\*\^0\^\*\)":"Excited",
u"\(\*_\*\)":"Amazed",
u"\(\*_\*;":"Amazed",
u"\(\+_\+\)\(@_@\)":"Amazed",
u"\(\*\^\^\)v":"Laughing,Cheerful",
u"\(\^_\^\)v":"Laughing,Cheerful",
u"\(\(d[-_-]b\)\)":"Headphones,Listening to music",
u'\(-"-\)':"Worried",
u"\(ーー;\)":"Worried",
u"\(\^0_0\^\)":"Eyeglasses",
u"\(\＾ｖ\＾\)":"Happy",
u"\(\＾ｕ\＾\)":"Happy",
u"\(\^\)o\(\^\)":"Happy",
u"\(\^O\^\)":"Happy",
u"\(\^o\^\)":"Happy",
u"\)\^o\^\(":"Happy",
u":O o_O":"Surprised",
u"o_0":"Surprised",
u"o\.O":"Surpised",
u"\(o\.o\)":"Surprised",
u"\(\*￣m￣\)":"Dissatisfied",
u"\(‘A`\)":"Snubbed Deflated",
u':)':"Standard smile",
u':-)':"With nose",
u':-E':"Buck-tooth Vampire",
u'>-)':"Evil grin",
u':(':"Sad frown smile",
u':-(':"Sad with nose",
u':-<':"Super sad",
u':-O':"Surprised",
u':-*':"Kissing",
u':-@':"Angry",
u':-$':"Confused",
u':-'':"Shifty",
u':-
u'(((H)))':"Hugs",
u':-X':"Kiss on the lips",
u'`:-)':"One eyebrow raised",
u':^)':"A broken nose",
u':-&':"tongue tied",
u'E-:-)':"a Ham radio operator",
u'<:-)':"Uni-brow",
u':->':"Big grin happy",
u'(-}{-)':"Couple kissing",
u':-Q':"Smoking",
u'$_$':"Greedy",
u':-!': ,
u':-D':"Laughter",
u':*)':"Drunk smile",
u':@':"Exclamation ""What???
if VALIDATE and is_TEST:
all_predictions = validate()

 for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
<load_pretrained>
EMOTICONS_fix = {
u":‑\)":"happy",
u":\)":"happy",
u":-\]":"happy",
u":\]":"happy",
u":-3":"happy",
u":->":"happy",
u":>":"happy",
u"8-\)":"happy",
u":o\)":"happy",
u":-\}":"happy",
u":\}":"happy",
u":-\)":"happy",
u":c\)":"happy",
u":\^\)":"happy",
u"=\]":"happy",
u"=\)":"happy",
u":‑D":"happy",
u"8‑D":"happy",
u"X‑D":"happy",
u"B\^D":"happy",
u":-\)\)":"happy",
u":‑\(":"sad",
u":-\(":"sad",
u":\(":"sad",
u":‑c":"sad",
u":‑<":"sad",
u":<":"sad",
u":‑\[":"sad",
u":\[":"sad",
u":-\|\|":"sad",
u">:\[":"sad",
u":\{":"sad",
u":@":"sad",
u">:\(":"sad",
u":'‑\(":"sad",
u":'\(":"sad",
u":'‑\)":"happy",
u":'\)":"happy",
u"D:<":"Disgust",
u":‑O":"surprise",
u":‑o":"surprise",
u":-0":"shocked",
u"8‑0":"Yawn",
u">:O":"Yawn",
u":-\*":"Kiss you",
u":\*":"Kiss you",
u":‑P":"giggle",
u"X‑P":"giggle",
u":‑Þ":"giggle",
u">:P":"giggle",
u":‑/":"sad",
u":/":"sad",
u":-[.]":"sad",
u">:[(\\\)]":"sad",
u">:/":"sad",
u":[(\\\)]":"sad",
u"=/":"sad",
u"=[(\\\)]":"sad",
u":\$":"Embarrassed",
u":‑x":"silent",
u":‑
u":‑&":"silent",
u":&":"silent",
u":‑b":"giggle",
u"\|;‑\)":"Cool",
u"\|‑O":"Bored",
u":-
u":
u"<:‑\|":"Dump",
u"\(>_<\)":"Troubled",
u"\(>_<\)>":"Troubled",
u"\(';'\)":"Baby",
u"\(\^\^>``":"Nervous",
u"\(\^_\^;\)":"Nervous",
u"\(-_-;\)":"Nervous",
u"\(~_~;\)\(・\.・;\)":"Nervous",
u"\(-_-\)zzz":"Sleeping",
u"\(\^_-\)":"Wink",
u"\(\(\+_\+\)\)":"Confused",
u"\(\+o\+\)":"Confused",
u"\(o\|o\)":"Ultraman",
u"\^_\^":"Joyful",
u"\(\^_\^\)/":"Joyful",
u"\(\^O\^\)／":"Joyful",
u"\(\^o\^\)／":"Joyful",
u"\(__\)":"apology",
u"_\(\._\.\)_":"apology",
u"<\(_ _\)>":"apology",
u"<m\(__\)m>":"apology",
u"m\(__\)m":"apology",
u"m\(_ _\)m":"apology",
u"\('_'\)":"sad",
u"\(/_;\)":"sad",
u"\(T_T\)\(;_;\)":"sad",
u"\(;_;":"sad",
u"\(;_:\)":"sad",
u"\(;O;\)":"sad",
u"\(:_;\)":"sad",
u"\(ToT\)":"sad",
u";_;":"sad",
u";-;":"sad",
u";n;":"sad",
u"Q\.Q":"sad",
u"T\.T":"sad",
u"Q_Q":"sad",
u"\(-\.-\)":"Shame",
u"\(-_-\)":"Shame",
u"\(一一\)":"Shame",
u"\(；一_一\)":"Shame",
u"\(=_=\)":"Tired",
u"\(\.\.\)":"Looking down",
u"\(\._\.\)":"Looking down",
u"\^m\^":"Giggle",
u">\^_\^<":"Laugh",
u"<\^!\^>":"Laugh",
u"\^/\^":"Laugh",
u"\（\*\^_\^\*）" :"Laugh",
u"\(\^<\^\)\(\^\.\^\)":"Laugh",
u"\(^\^\)":"Laugh",
u"\(\^\.\^\)":"Laugh",
u"\(\^_\^\.\)":"Laugh",
u"\(\^_\^\)":"Laugh",
u"\(\^\^\)":"Laugh",
u"\(\^J\^\)":"Laugh",
u"\(\*\^\.\^\*\)":"Laugh",
u"\(\^—\^\）":"Laugh",
u"\(
 u"\（\^—\^\）":"Waving",
u"\(;_;\)/~~~":"Waving",
u"\(\^\.\^\)/~~~":"Waving",
u"\(-_-\)/~~~ \($\·\·\)/~~~":"Waving",
u"\(T_T\)/~~~":"Waving",
u"\(ToT\)/~~~":"Waving",
u"\(\*\^0\^\*\)":"Excited",
u"\(\*_\*\)":"Amazed",
u"\(\*_\*;":"Amazed",
u"\(\+_\+\)\(@_@\)":"Amazed",
u"\(\*\^\^\)v":"Laugh",
u"\(\^_\^\)v":"Laugh",
u"\(\(d[-_-]b\)\)":"Listening to music",
u'\(-"-\)':"Worried",
u"\(ーー;\)":"Worried",
u"\(\^0_0\^\)":"Eyeglasses",
u"\(\＾ｖ\＾\)":"happy",
u"\(\＾ｕ\＾\)":"happy",
u"\(\^\)o\(\^\)":"happy",
u"\(\^O\^\)":"happy",
u"\(\^o\^\)":"happy",
u"\)\^o\^\(":"happy",
u":O o_O":"Surprised",
u"o_0":"Surprised",
u"o\.O":"Surpised",
u"\(o\.o\)":"Surprised",
u"\(\*￣m￣\)":"Dissatisfied",
u"\(‘A`\)":"Snubbed Deflated",
u':)':"Laugh",
u':-)':"Laugh",
u':(':"sad",
u':-(':"sad",
u':-<':"sad",
u':-O':"Surprised",
u':-*':"Kissing",
u':-@':"Angry",
u':-$':"Confused",
u':-'':"Shifty",
u":-
}
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
if count<10:
img_ = cv2.imread(path)
 for box, score in zip(boxes,scores):
cv2.rectangle(img_,(box[0], box[1]),(box[2]+box[0], box[3]+box[1]),(220, 0, 0), 2)
 cv2.putText(img_, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(img_)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
def convert_emoticons(text):
for emot in EMOTICONS_fix.items() :
if emot[0] in text:
text = text.replace(emot[0],".I feel "+emot[1]+".")
 return text
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<define_variables>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r' ', text)
 remove_emoji("Omg another Earthquake 😔😔" )
NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.5
best_iou_thr = 0.6
best_skip_box_thr = 0.43
best_final_score = 0
best_score_threshold = 0
EPO = 15
WEIGHTS = '.. /input/best-yolov5x-fold0pt/best_yolov5x_fold0.pt'
CONFIG = '.. /input/best-yolov5x-fold0pt/yolov5x.yaml'
DATA = '.. /input/best-yolov5x-fold0pt/wheat0.yaml'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >11
is_AUG = True
is_ROT = True
VALIDATE = True
PSEUDO = True
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
Abbreviation = {
'?':"I have a question",
'?4U':"I have a question for you",
';S':"Hmm? What did you say?",
'^^':"read line",
'<3':"sideways heart",
'<3':"broken heart",
'<33':"heart or love",
'@TEOTD':"At the end of the day",
'0.02':"My(or your)two cents worth",
"'1TG, 2TG'":" number of items needed for win ",
'1UP':"extra life",
'121':"One-to-one",
'1337':"Leet, 'elite'",
'143':"I love you",
'1432':"I love you too",
'14AA41':"One for all, and all for one",
'182':"I hate you",
'19':"Zero hand",
'10M':"Ten man",
'10X':"Thanks",
'10Q':"Thank you",
'1CE':"Once",
'1DR':"I wonder",
'1NAM':"One in a million",
'2': "to"" in SMS",
'20': "location",
'2B':"To be",
'2EZ':"Too easy",
'2G2BT':"Too good to be true",
'2M2H':"Too much too handle",
'2MI':"Too much information",
'2MOR':"Tomorrow",
'2MORO':"Tomorrow",
'2M2H':"Too much to handle",
'2N8':"Tonight",
'2NTE':"Tonight",
'4':'for',
'411': "information",
'404':"I don't know",
'411':'information',
'420':"Lets get high",
'420': "Marijuana",
'459':"I love you",
'4AO':"For adults only",
'4COL':"For crying out loud",
'4EAE':"Forever and ever",
'4EVA':"Forever",
'4NR':"Foreigner",
'4SALE':"For sale",
'^5':"High-five",
'511':"Too much information",
'555':"Sobbing, crying.",
'55555':"Crying your eyes out",
'55555':"Laughing",
'6Y':"Sexy",
'7K':"Sick",
'81':"Hells Angels",
'831':"I love you",
'86':"Over",
'88':"Bye-bye",
'88':"Hugs and kisses",
'9':"Parent is watching ",
'*s*': "smile",
'*w*': "wink",
'A3':"Anytime, anywhere, anyplace",
'AA':"Alcoholics Anonymous",
'AA':"As above",
'AA':"Ask about",
'AAF':"As a matter of fact",
'AAF':"As a friend",
'AAK':"Asleep at keyboard",
'AAK':"Alive and kicking",
'AAMOF':"As a matter of fact",
'AAMOI':"As a matter of interest",
'AAP':"Always a pleasure",
'AAR':"At any rate",
'AAS':"Alive and smiling",
'AASHTA':"As always, Sheldon has the answer",
'AATK':"Always at the keyboard",
'AAYF':"As always, your friend",
'ABBR':"Meaning abbreviation",
'ABC':"Already been chewed",
'ABD':"Already been done",
'ABT':"About",
'ABT2':"About to",
'ABTA':"Good-bye",
'ABU':"All bugged up",
'AC':"Acceptable content",
'ACC':"Anyone can come",
'ACD':"ALT / CONTROL / DELETE",
'ACDNT':"Accident",
'ACE':"marijuana cigarette",
'ACK':"Acknowledge",
'ACPT':"Accept",
'ACQSTN':"Acquisition",
'ADAD':"Another day, another dollar",
'ADBB':"All done, bye-bye",
'ADD':"Address",
'ADDY':"Address",
'ADIH':"Another day in hell",
'ADIP':"Another day in paradise",
'ADMIN':"Administrator",
'ADMINR':"Administrator(Government)",
'ADN':"Any day now",
'ADR':"Address",
'AE':"Area effect ",
'AEAP':"As early as possible",
'AF':"April Fools",
'AF':"As *Freak*",
'AF':"Aggression factor",
'AFC':"Away from computer",
'AFAIAA':"As far as I am aware",
'AFAIC':"As far as I am concerned",
'AFAIK':"As far as I know",
'AFAIUI':"As far as I understand it",
'AFAP':"As far as possible",
'AFFA':"Angels Forever, Forever Angels",
'AFJ':"April Fool's joke",
'AFK':"Away from keyboard",
'AFZ':"Acronym Free Zone",
'AFPOE':"A fresh pair of eyes",
'AGI': "agility"" ",
'AH':"At home",
'AIAMU':"And I am a money's uncle",
'AIGHT':"Alright",
'AIR':"As I remember",
'AISB':"As it should be",
'AISB':"As I said before",
'AISI':"As I see it",
'AITR':"Adult in the room",
'AKA':"Also known as",
'ALCON':"All concerned",
'ALOL':"Actually laughing out loud",
'AMA':"Ask me anything(Reddit)",
'AMAP':"As much as possible",
'AMBW':"All my best wishes",
'AML':"All my love",
'AMOF':"As a matter of fact",
'A/N':"Author's note",
'AO':"Anarchy Online ",
'AOC':"Available on cell",
'AOE':"Area of effect(online game)",
'AOM':"Age of majority",
'AOM':"Age of Mythology ",
'AOTA':"All of the above",
'AOYP':"Angel on your pillow",
'APAC':"All praise and credit",
'APP':"Application",
'APP':"Appreciate",
'AQAP':"As quick(or quiet)as possible",
'ARC':"Archive(compressed files)",
'ARE':"Acronym rich environment",
'ARG':"Argument",
'ASIG':"And so it goes",
'ASAP':"As soon as possible",
'A/S/L':"Age/sex/location",
'ASL':"Age/sex/location",
'ASLA':"Age/sex/location/availability",
'AT':"At your terminal",
'ATB':"All the best",
'ATEOTD':"At the end of the day",
'ATM':"At the moment",
'ATSITS':"All the stars in the sky",
'ATSL':"Along the same line(or lines)",
'AWC':"After awhile crocodile",
'AWESO':"Awesome",
'AWOL':"Away without leaving",
'AWOL':"Absent without leave",
'AYDY':"Are you done yet?",
'AYBABTU':"All your base are belong to us ",
'AYEC':"At your earliest convenience",
'AYOR':"At your own risk",
'AYSOS':"Are you stupid or something?",
'AYS':"Are you serious?",
'AYT':"Are you there?",
'AYTMTB':"And you're telling me this because",
'AYV':"Are you vertical?",
'AYW':"As you were",
'AYW':"As you want / As you wish",
'AZN':"Asian",
'B':"Back",
'B':"Be",
'B&':"Banned",
'B2B':"Business-to-business",
'B2C':"Business-to-consumer",
'B2W':"Back to work",
'B8':"Bait(person teased or joked with, or under-aged person/teen)",
'B9':"Boss is watching",
'B/F':"Boyfriend",
'B/G':"Background(personal information request)",
'B4':"Before",
'B4N':"Bye for now",
'BAG':"Busting a gut",
'BA':"Bad *a*",
'BAE':"Before anyone else",
'BAE':"Meaning Babe or baby",
'BAFO':"Best and final offer",
'BAK':"Back at keyboard",
'BAM':"Below average mentality",
'BAMF':"Bad *a* mother *f*",
'BAO':"Be aware of",
'BAS':"Big 'butt' smile",
'BASIC':"Meaning anything mainstream",
'BASOR':"Breathing a sigh of relief",
'BAU':"Business as usual",
'BAY':"Back at ya",
'BB':"Be back",
'BB':"Big brother",
'BB':"Bebi / Baby(Spanish SMS)",
'BBC':"Big bad challenge",
'BBIAB':"Be back in a bit",
'BBIAF':"Be back in a few",
'BBIAM':"Be back in a minute",
'BBIAS':"Be back in a sec",
'BBL':"Be back later",
'BBN':"Bye, bye now",
'BBQ':"Barbeque, ""Ownage"", shooting score/frag ",
'BBS':"Be back soon",
'BBT':"Be back tomorrow",
'BC':"Because",
'B/C':"Because",
'BC':"Be cool",
'BCNU':"Be seeing you",
'BCOS':"Because",
'BCO':"Big crush on",
'BCOY':"Big crush on you",
'BD':"Big deal",
'BDAY':"Birthday",
'B-DAY':"Birthday",
'BDN':"Big darn number",
'BEG':"Big evil grin",
'BELF': "Blood Elf"" ",
'BF':"Boyfriend",
'BF':"Brain fart",
'BFAW':"Best friend at work",
'BF2':"Battlefield 2 ",
'BF':"Best friend",
'BFF':"Best friends forever",
'BFFL':"Best friends for life",
'BFFLNMW':"Best friends for life, no matter what",
'BFD':"Big freaking deal",
'BFG':"Big freaking grin",
'BFFN':"Best friend for now",
'BFN':"Bye for now",
'BG':"Big grin",
'BGWM':"Be gentle with me",
'BHL8':"Be home late",
'BIB':"Boss is back",
'BIBO':"Beer in, beer out",
'BIC':"Butt in chair",
'BIF':"Before I forget",
'BIH':"Burn in hell",
'BIL':"Brother in law",
'BIO': "I'm going to the bathroom""(or)""Bathroom break",
'BION':"Believe it or not",
'BIOYA':"Blow it out your *a*",
'BIOYN':"Blow it out your nose",
'BIS':"Best in slot ",
'BISFLATM':"Boy, I sure feel like a turquoise monkey!(unverified)",
'BITMT':"But in the meantime",
'BL':"Belly laugh",
'BLNT':"Better luck next time",
'Bloke':"Meaning Man(Discord)",
'BM':"Bite me",
'BME':"Based on my experience",
'BM&Y':"Between me and you",
'BOB':"Back off *buddy*",
'BN':"Bad news",
'BOE': "bind on equip"" ",
'BOHICA':"Bend over here it comes again",
'BOL':"Best of luck",
'BOM':"*b* of mine",
'BOLO':"Be on the look out",
'BOOMS':"Bored out of my skull",
'BOP': "bind on pickup"" ",
'BOSMKL':"Bending over smacking my knee laughing",
'BOT':"Back on topic",
'BOT':"Be on that",
'BMS':"Broke my scale, used when rating someone",
'BOYF':"Boyfriend",
'BPLM':"Big person little mind",
'BRB':"Be right back",
'BR':"Best regards",
'BRBB':"Be right back *b*",
'BRNC':"Be right back, nature calls",
'BRD':"Bored",
'BRH':"Be right here",
'BRT':"Be right there",
'BSF':"But seriously folks",
'BSOD':"Blue screen of death",
'BSTS':"Better safe than sorry",
'BT':"Bite this",
'BT':"Between technologies",
'BTA':"But then again",
'BTDT':"Been there, done that",
'BTW':"By the way ",
'BTYCL':"Bootycall",
'BUBU':"Slang term for the most beautiful of women",
'BURN':"Used to reference an insult",
'Buff': "changed and is now stronger",
'BWL':"Bursting with laughter ",
'BYOB':"Bring your own beer ",
'BYOC':"Bring your own computer",
'BYOD':"Bring your own device",
'BYOH':"Bat you on head",
'BYOP':"Bring your own paint",
'BYTM':"Better you than me",
'C&G':"Chuckle & grin",
'C4N':"Ciao for now",
'CAD':"Control + Alt + Delete",
'CAD':"Short for Canada/Canadian",
'Cakeday':"Birthday",
'CAM':"Camera(SMS)",
'CB':"Coffee break",
'CB':"Chat break",
'CB':"Crazy *b*",
'CD9':"parents are around",
'CFS':"Care for secret?",
'CFY':"Calling for you",
'CHK':"Check",
'CIAO':"Good-bye",
'CICO':"Coffee in, coffee out",
'CID':"Crying in disgrace",
'CID':"Consider it done",
'CLAB':"Crying like a baby",
'CLD':"Could",
'CLK':"Click",
'CM':"Call me",
'CMAP':"Cover my *a* partner ",
'CMB':"Call me back",
'CMGR': "Community manager",
'CMIIW':"Correct me if I'm wrong",
'CMON':"Come on",
'CNP':"Continued(in)next post",
'COB':"Close of business",
'COH':"City of Heroes ",
'COS':"Because",
'C/P':"Cross post",
'CP':"Chat post",
'CR8':"Create",
'Cray':"Meaning crazy",
'CRE8':"Create",
'CRA CRA':"crazy",
'CRAFT':"Can't remember a *freaking* thing",
'CRB':"Come right back",
'CRBT ':"Crying really big tears ",
'CRIT ': "critical hit",
'CRZ ':"Crazy",
'CRS ':"Can't remember *stuff*",
'CSG':"Chuckle, snicker, grin",
'CSL':"Can't stop laughing",
'CSS':"Counter-Strike Source ",
'CT':"Can't talk",
'CTC':"Care to chat?",
'CTHU':"Cracking the *heck* up",
'CTN':"Can't talk now",
'CTO':"Check this out",
'CU':"See you too",
'CU':"See you",
'CU2':"See you",
'CUA':"See you around",
'CUL':"See you later",
'CULA':"See you later alligator",
'CUL8R':"See you later",
'CUMID':"See you in my dreams",
'CURLO':"See you around like a donut",
'CWD':"Comment when done",
'CWOT':"Complete waste of time",
'CWYL':"Chat with you later",
'CX': "Correction",
'CYA':"See you",
'CYAL8R':"See you later",
'CYE':"Check your e-mail",
'CYEP':"Close your eyes partner ",
'CYO':"See you online",
'D2':"Dedos",
'D46?':"Down for sex?",
'DA': "The",
'DAE':"Does anyone else?",
'DAFUQ':"(What)the *Freak*?",
'DAM':"Don't annoy me",
'DAoC':"Dark Age of Camelot ",
'DBAU':"Doing business as usual",
'DBEYR':"Don't believe everything you read",
'DC':"Disconnect",
'DD':"Dear daughter",
'DD':"Due diligence",
'DDG':"Drop dead gorgeous",
'DEEZ NUTZ':"A phrase used in online chat to annoy or frustrate those involved in the conversation.",
'DEGT':"Dear daughter",
'DERP':"stupid",
'DF':"Don't even go there",
'DFL':"Dead *freaking* last ",
'DGA':"Don't go anywhere",
'DGAF':"Don't give a *freak*",
'DGT':"Don't go there",
'DGTG':"Don't go there, girlfriend",
'DGYF':"Dang, girl you fine",
'DH':"Dear husband",
'DHU':"Dinosaur hugs(used to show support)",
'DIIK':"Darned if I know",
'DIKU':"Do I know you?",
'DILLIGAF':"Do I look like I give a *freak*?",
'DILLIGAS':"Do I look like I give a sugar?",
'DIS':"Did I say?",
'DITYID':"Did I tell you I'm distressed?",
'DIY':"Do it yourself",
'DKDC':"Don't know, don't care",
'DKP':"Dragon kill points ",
'D/L':"Download",
'DL':"Download",
'DL':"Down low",
'DL':"Dead link",
'DLBBB':"Don't let(the)bed bugs bite",
'DLTBBB':"Don't let the bed bugs bite",
'DM':"Doesn't matter",
'DM':"Direct message",
'DM':"Do me",
'DM':"Dungeon Master ",
'DMNO':"Dude Man No Offense",
'DMY':"Don't mess yourself",
'DN':"Down",
'DNC':"I do not understand",
'DNR':"Dinner",
'DNT':"Don't",
'd00d':"Dude",
'DOE':"Daughter of Eve",
'DORBS': "Adorable",
'DOT':"Damage over time ",
'Downvote':"Voting negatively on a thread using Reddit’s voting system",
'DPS':"Damage per second ",
'DQMOT':"Don't quote me on this",
'DR':"Didn't read",
'DS':"Dear son",
'DTR':"Define the relationship",
'DTRT':"Do the right thing",
'DTS':"Don't think so",
'DTTD':"Don't touch that dial",
'DUPE':"Duplicate",
'DUR':"Do you remember?",
'DV8':"Deviate",
'DW':"Dear wife",
'DWF':"Divorced white female",
'DWM':"Divorced white male",
'DXNRY':"Dictionary",
'DYNWUTB':"Do you know what you are talking about?",
'DYFI':"Did you find it?",
'DYFM':"Dude, you fascinate me",
'DYJHIW':"Don't you just hate it when...?",
'DYOR':"Do your own research(common stock market chat slang)",
'E':"Ecstasy",
'E':"Enemy ",
'E1':"Everyone",
'E123':"Easy as one, two, three",
'E2EG':"Ear to ear grin",
'EAK':"Eating at keyboard",
'EBKAC':"Error between keyboard and chair",
'ED':"Erase display",
'EF4T':"Effort",
'EG':"Evil grin",
'EI':"Eat it",
'EIP':"Editing in progress",
'ELI5':"Explain like I'm 5",
'EM':"E-mail",
'EMA':"E-mail address",
'EMBAR': "Embarassing",
'EMFBI':"Excuse me for butting in",
'EMFBI':"Excuse me for jumping in",
'EMSG':"E-mail message",
'ENUF':"Enough",
'EOD':"End of day",
'EOD':"End of discussion",
'EOL':"End of lecture",
'EOL':"End of life",
'EOM':"End of message",
'EOS':"End of show",
'EOT':"End of transmission",
'EQ':"EverQuest ",
'ERP': "Erotic Role-Play"" ",
'ERS2':"Eres tz / are you(Spanish SMS)",
'ES':"Erase screen",
'ESAD':"Eat *S* and die!",
'ETA':"Estimated time(of)arrival",
'ETA':"Edited to add",
'EVA':"Ever",
'EVO':"Evolution",
'EWG':"Evil wicked grin",
'EWI':"Emailing while intoxicated",
'EXTRA':"over the top",
'EYC':"Excitable, yet calm",
'EZ':"Easy",
'EZY':"Easy",
'F':"female",
'F2F':"Face to face",
'F2P':"Free to play ",
'F4F':"Follow for follow",
'FAAK':"Falling asleep at keyboard",
'FAB':"Fabulous",
'Facepalm':"smacking your forehead with your palm",
'FAF':"Funny as *freak*",
'FAM':"friends",
'FAQ':"Frequently asked questions",
'FAY':"*Freak* all you",
'FB':"Facebook",
'FBB': "Facebook buddy",
'FBC':"Facebook chat",
'FBF':"Flashback Friday",
'FBF': "Facebook friend",
'FBF':"Fat boy food",
'FBFR':"FaceBook friend",
'FBM':"Fine by me",
'FBO':"Facebook official",
'FBOW':"For better or worse",
'FC':"Fingers crossed",
'FC':"Full card ",
'FC'INGO':"For crying out loud",
'FCOL':"For crying out loud",
'Feelsbadman':"feel negative.",
'Feelsbatman':"A social meme taking ""feelsbadman"" to the extreme",
'Feelsgoodman':"A social meme that means to feel positive.",
'FEITCTAJ':"*Freak* 'em if they can't take a joke",
'FF':"Follow Friday ",
'FFA':"Free for all ",
'FFS':"For *freak'*sakes",
'FICCL':"Frankly I couldn't care a less",
'FIF':"*Freak* I'm funny",
'FIIK':"*Freaked* if I know",
'FIIOOH':"Forget it, I'm out of here",
'FIL':"Father in law",
'FIMH':"Forever in my heart",
'Finna':"Means ""Going to",
'Finsta':"A second Instagram account where someone can post things that they're too afraid to post on their main account.",
'FISH':"First in, still here",
'FITB':"Fill in the blank",
'FML':"*Freak* My Life",
'FOMC':"Falling off my chair",
'FOMO':"Fear of missing out(definition)",
'FOAD':"*Freak* off and die",
'FOAF':"Friend of a friend",
'FOMCL':"Falling off my chair laughing",
'FRT':"For real though",
'FTBOMH':"From the bottom of my heart",
'FTFY':"Fixed that for you",
'FTL':"For the loss",
'FTW':"For the win",
'FU':"*Freak* you",
'FUBAR':"Fouled up beyond all recognition",
'FUBB':"Fouled up beyond belief",
'FUD':"Face up deal ",
'FUTAB':"Feet up, take a break",
'FW':"Forward",
'FWB':"Friend with benefits",
'FWIW':"For what it's worth",
'FWM':"Fine with me",
'FWP':"First world problems",
'FYE':"Fire, something that is cool",
'FYEO':"For your eyes only",
'FYA':"For your amusement",
'FYI':"For your information",
'G':"Grin",
'G':"Giggle",
'G+':"Google+",
'G/F':"Girlfriend",
'G2CU':"Good to see you",
'G2G':"Got to go",
'G2GICYAL8ER':"Got to go I'll see you later",
'G2R':"Got to run",
'G2TU':"Got to tell you",
'G4C':"Going for coffee",
'G9':"Genius",
'GA':"Go ahead",
'GAC':"Get a clue",
'GAFC':"Get a *freaking* clue",
'GAL':"Get a life",
'GANK':"Meaning a player ambush or unfair player kill ",
'GAS':"Got a second?",
'GAS':"Greetings and salutations",
'GB':"Goodbye",
'GBTW':"Get back to work",
'GBU':"God bless you",
'GD':"Good",
'GDR':"Grinning, ducking, and running",
'GD/R':"Grinning, ducking, and running",
'GFI':"Go for it",
'GF':"Girl friend",
'GFN':"Gone for now",
'GG':"Gotta Go",
'GG':"Good Game ",
'GG':"Brother",
'GGA':"Good game, all ",
'GGE1':"Good game, everyone ",
'GGU2':"Good game, you too",
'GGMSOT':"Gotta get me some of that",
'GGOH':"Gotta Get Outa Here",
'GGP':"Got to go pee",
'GH':"Good hand ",
'GIAR':"Give it a rest",
'GIC':"Gift in crib ",
'GIGO':"Garbage in, garbage out",
'GIRL':"Guy in real life",
'GJ':"Good job",
'GL':"Good luck",
'GL2U':"Good luck to you ",
'GLA':"Good luck all ",
'GL/HF':"Good luck, have fun ",
'GLE':"Good luck everyone ",
'GLE1':"Good luck everyone ",
'GLNG':"Good luck next game ",
'GMBA':"Giggling my butt off",
'GMTA':"Great minds think alike",
'GMV':"Got my vote",
'GN':"Good night",
'GNA':"Good night all",
'GNE1':"Good night everyone",
'GNIGHT':"Good night",
'GNITE':"Good night",
'GNSD':"Good night, sweet dreams",
'GOAT':"Greatest of all Time(s)",
'GOI':"Get over it",
'GOL':"Giggling out loud",
'GOMB':"Get off my back",
'GPOY':"Gratuitous picture of yourself",
'GR8':"Great",
'GRATZ':"Congratulations",
'GRL':"Girl",
'GRWG':"Get right with God",
'GR&D':"Grinning, running and ducking",
'GS':"Good shot",
'GS':"Good split ",
'GT':"Good try",
'GTFO':"Get the *freak* out",
'GTFOH':"Get the *freak* outta here",
'GTG':"Got to go",
'GTM':"Giggling to myself",
'GTRM':"Going to read mail ",
'GTSY':"Great(or good)to see you ",
'GUCCI':"Good",
'GUD':"Good ",
'GWHTLC':"Glad we had this little chat",
'H':"Hug",
'H8':"Hate",
'H8TTU':"Hate to be you",
'HAG1':"Have a good one",
'HAK':"Hug and kiss",
'HALP':"Help(Discord)",
'HAU':"How about you?",
'H&K':"Hugs & kisses",
'H2CUS':"Hope to see you soon",
'HAGN':"Have a good night",
'HAGO':"Have a good one",
'HAND':"Have a nice day",
'HAWT':"Have a wonderful day(out-dated, see next in list)",
'HAWT': "sexy"" or ""attractive",
'HB':"Hurry back",
'HB':"Hug back",
'HBD':"Happy birthday",
'H-BDAY':"Happy Birthday",
'HBU':"How about you?",
'HF':"Have fun",
'HFAC':"Holy flipping animal crackers",
'H-FDAY':"Happy Father's Day",
'HHIS':"Head hanging in shame",
'HIFW':"How I felt when...",
'HL':"Half Life ",
'HLA':"Hola / hello",
'H-MDAY':"Happy Mother's Day",
'HMU':"Hit me up",
'HNL':"(w)Hole 'nother level",
'HOAS':"Hold on a second",
'HP':"Hit points",
'HRU':"How are you?",
'HTH':"Hope this helps",
'HUB':"Head up butt",
'HUYA':"Head up your *butt*",
'HV':"Have ",
'HVH':"Heroic Violet Hold ",
'HW':"Homework",
'HYFR':"Hell yeah, *Freaking* right!",
'I2':"me too",
'IA8':"I already ate",
'IAAA':"I am an accountant",
'IAAD':"I am a doctor",
'IAAL':"I am a lawyer",
'IAC':"In any case",
'IAE':"In any event",
'IANAC':"I am not a crook",
'IANAL':"I am not a lawyer",
'IAO':"I am out(of here)",
'IB':"I'm back",
'IB':"I'm back",
'IC':"I see",
'ICAM':"I couldn't agree more",
'ICBW':"It could be worse",
'ICEDI':"I can't even discuss it",
'ICFILWU':"I could fall in love with you",
'ICYMI':"In case you missed it ",
'IDBI':"I don't believe it",
'IDC':"I don't care",
'IDGAF':" I don't give a *freak*",
'IDK':"I don't know",
'IDTS':"I don't think so",
'IDUNNO':"I don't know",
'IFYP':"I feel your pain",
'IG':"Instagram",
'IG2R':"I got to run",
'IGHT':"I got high tonight",
'IGN':"I got nothing",
'IGP':"I got to(go)pee",
'IHNI':"I have no idea",
'IIRC':"If I remember correctly",
'IIIO':"Intel inside, idiot outside",
'IK':"I know",
'IKR':"I know, right?",
'ILBL8':"I'll be late",
'ILU':"I love you",
'ILUM':"I love you man",
'ILYSM':"I love you so much",
'ILY':"I love you",
'IM':"Instant message",
'IMAO':"In my arrogant opinion",
'IMHO':"In my humble opinion",
'ImL':"Means I love you",
'IMNSHO':"In my not so humble opinion",
'IMO':"In my opinion",
'IMS':"I am sorry",
'IMSB':"I am so bored",
'IMTM':"I am the man",
'IMU':"I miss you",
'INAL':"I'm not a lawyer",
'INC': "incoming"" ",
'Inspoo':"Inspiration",
'INV': "Invite",
'IOMH':"In over my head",
'IOW':"In other words",
'IRL':"In real life",
'IRMC':"I rest my case",
'ISLY':"I still love you",
'ISO':"In search of",
'ITAM':"It's The Accounting, Man",
'ITT':"In This Thread",
'ITYK':"I thought you knew",
'IUSS':"If you say so",
'IWALU':"I will always love you",
'IWAWO':"I want a way out",
'IWIAM':"Idiot wrapped in a moron",
'IWSN':"I want sex now",
'IYKWIM':"If you know what I mean",
'IYO':"In your opinion",
'IYQ': "I like you",
'IYSS':"If you say so",
'j00':"You",
'j00r':"Your",
'JAC':"Just a second",
'JAM':"Just a minute",
'JAS':"Just a second",
'JC(J/C)':"Just checking",
'JDI':"Just do it",
'JELLY': "jealous",
'JFF':"Just for fun",
'JFGI':"Just *freaking* Google it",
'JIC':"Just in case",
'JJ(J/J)':"Just joking",
'JJA':"Just joking around",
'JK(J/K)':"Just kidding",
'JLMK':"Just let me know",
'JMO':"Just my opinion",
'JP':"Just playing ",
'JP':"Jackpot",
'JT(J/T)':"Just teasing",
'JTLYK':"Just to let you know",
'JV':"Joint venture",
'JW':"Just wondering",
'K':"Okay",
'KK':"Knock, knock",
'KK':"Okay, Okay!",
'K8T':"Katie",
'k/b':"Keyboard",
'KB':"Keyboard",
'KB':"Kick butt ",
'KDFU':"Cracking(K)the(D as in Da)*freak* up",
'KEWL':"Cool",
'KEYA':"I will key you later",
'KEYME':"Key me when you get in",
'KFY':"Kiss for you",
'KIA':"Know it all",
'KIR':"Keep it real",
'KISS':"Keep it simple, stupid",
'KIT':"Keep in touch",
'KMA':"Kiss my *a*",
'KMK':"Kiss my keister",
'KMS':"Killing myself",
'KMT':"Kiss my tushie",
'KOC':"Kiss on cheek",
'KOL':"Key opinion leader",
'Koreaboo':"Someone obsessed with Korean culture",
'KOS':"Kid over shoulder",
'KOS':"Kill on sight ",
'KOW':"Knock on wood",
'KOTC':"Kiss on the cheek",
'KOTD':"Kicks of the day(Instagram)",
'KOTL':"Kiss on the lips",
'KNIM':"Know what I mean?",
'KNOW': "knowledge",
'KPC':"Keeping parents clueless",
'KS':"Kill then steal ",
'KSC':"Kind(of)sort(of)chuckle",
'KT':"Katie",
'KUTGW':"Keep up the good work",
'KYS':"Kill yourself",
'L2G':"Like to go?",
'L2G':"Love to go",
'L2K':"Like to come",
'L2P':"Learn to play",
'l33t':"Leet, meaning 'elite'",
'L4L':"Like for like",
'L8R':"Later",
'L8RG8R':"Later, gator",
'LAB':"Life's a *b*",
'LBAY':"Laughing back at you",
'LBS':"Laughing, but serious",
'LBVS':"Laughing, but very serious",
'LD':"Later, dude",
'LD':"Long distance",
'LDO':"Like, duh obviously",
'LEMENO':"Let me know",
'LERK':"Leaving easy reach of keyboard",
'LFD':"Left for day",
'LFG':"Looking for group ",
'LFG':"Looking for guard ",
'LFM':"Looking for more ",
'LGH':"Lets get high",
'LH6':"Lets have sex",
'LHSX':"Lets have sex",
'LHM':"Lord help me",
'LHO':"Laughing head off",
'LI':"LinkedIn",
'LIC':"Like I care",
'LIK':"Meaning liquor",
'LIMT':"Laugh in my tummy",
'LIT':"Meaning really good or something fun and exciting",
'LIT':"Extremely intoxicated",
'LLGB':"Love, later, God bless",
'LLS':"Laughing like *silly*",
'LMAO':"Laughing my *a* off",
'LMBO':"Laughing my butt off",
'LMFAO':"Laughing my freaking *a* off",
'LMIRL':"Lets meet in real life",
'LMK':"Let me know",
'LMMFAO':"Laughing my mother freaking a** off",
'LMNK':"Leave my name out",
'LMS':"Like my status(Facebook)",
'LNT':"lost in translation",
'LOA':"List of acronyms",
'LOL':"Laughing out loud",
'LOL':"Laugh out loud",
'LOL':"Lots of love",
'LOLH':"Laughing out loud hysterically",
'LOLO':"Lots of love",
'LOLWTF':"Laughing out loud ""What the *freak*?",
'LOTI':"Laughing on the inside",
'LOTR':"Lord of The Rings ",
'LQTM':"Laughing quietly to myself",
'LSHMBH':"Laugh so hard my belly hurts",
'LSV':"Language, sex and violence",
'LTD':"Living the dream",
'LTLWDLS':"Let's twist like we did last summer",
'LTNS':"Long time no see",
'LTOD':"Laptop of death",
'LTS':"Laughing to self",
'LULT':"Love you long time",
'LULZ':"Meaning joke, or for laughs",
'LVM':"Left voice mail",
'LWOS':"Laughing without smiling",
'LY':"Love ya ",
'LYLAS':"Love you like a sis",
'LYLC':"Love you like crazy",
'LYSM':"Love you so much",
'M$':"Microsoft",
'M8':"Mate",
'MB':"Mamma's boy",
'MBS':"Mom behind shoulder",
'MC':"Merry Christmas",
'MDIAC':"My Dad is a cop",
'MEGO':"My eyes glaze over",
'MEH':"shrugging shoulders",
'MEH':"just okay",
'MEHH':"sigh",
'MEZ': "mesmerize"" ",
'MFI':"Mad for it",
'MFW':"My face when...",
'MGB':"May God bless",
'MGMT':"Management",
'MHOTY':"My hat off to you",
'MIRL':"Me in real life",
'MIRL':"Meet in real life",
'MISS.( number)': "child and her age",
'MKAY': "Mmm, okay",
'MLM':"Meaning give the middle finger",
'MM':"Sister(Mandarin Chinese txt msg)",
'MMK':"okay?",
'MNC':"Mother nature calls",
'MNSG':"Mensaje",
'MOD':"Moderator",
'MOD':"Modification ",
'MORF':"Male or female?",
'MOMBOY':"Mamma's boy",
'MOO':"My own opinion",
'MOOS':"Member of the opposite sex",
'MOS':"Mother over shoulder",
'MOSS':"Member of same sex",
'MP':"Mana points ",
'MR.': "child and his age"".Mr.3 would be a 3-year old son",
'MRT':"Modified ReTweet",
'MRW':"My reaction when...",
'MSG':"Message",
'MTF':"More to follow",
'MTFBWU':"May the force be with you",
'MU':"Miss U(you)",
'MUAH':"Multiple unsuccessful attempts to humor",
'MUSM':"Miss you so much",
'MWAH': "kiss",
'MYO':"Mind your own business",
'MYOB':"Mind your own business",
'n00b':"Newbie",
'N1':"Nice one",
'N2M':"Nothing too much",
'NADT':"Not a darn thing",
'NALOPKT':"Not a lot of people know that",
'NANA':"Not now, no need",
'NBD':"No big deal",
'NBFAB':"No bad for a beginner ",
'NC':"Nice crib ",
'ND':"Nice double ",
'NE':"Any",
'NE1':"Anyone",
'NERF': "Changed and is now weaker",
'NFM':"None for me / Not for me",
'NFM':"Not for me",
'NGL':"Not gonna(going to)lie",
'NFS':"Need for Speed ",
'NFS':"Not for sale",
'NFW':"No *freaking* way",
'NFW':"Not for work",
'NFWS':"Not for work safe",
'NH':"Nice hand ",
'NIFOC':"Naked in front of computer",
'NIGI':"Now I get it",
'NIMBY':"Not in my back yard",
'NIROK':"Not in reach of keyboard",
'NLT':"No later than",
'NM':"Nothing much",
'NM':"Never mind",
'NM':"Nice meld ",
'NMH':"Not much here",
'NMJC':"Nothing much, just chilling",
'NMU':"Not much, you?",
'NO1':"No one",
'NOOB':"someone who is bad at games",
'NOWL': "knowledge",
'NOYB':"None of your business",
'NP':"No problem",
'NPC':"Non-playing character ",
'NQT':"Newly qualified teacher",
'NR':"Nice roll ",
'NRN':"No response/reply necessary",
'NS':"Nice score ",
'NS':"Nice split ",
'NSA':"No strings attached",
'NSFL':"Not safe for life",
'NSFW':"Not safe for work",
'NSISR':"Not sure if spelled right",
'NT':"Nice try",
'NTHING':"Nothing(SMS)",
'NTS':"Note to self",
'NVM':"Never mind",
'NVR':"Never",
'NW':"No way",
'NWO':"No way out",
'O4U':"Only for you",
'O':"Opponent ",
'O': "hugs",
'O':"Over",
'OA':"Online auctions",
'OATUS':"On a totally unrelated subject",
'OB':"Oh baby",
'OB':"Oh brother",
'OBV':"Obviously",
'OFC':"Of course",
'OG':"Original gangster",
'OGIM':"Oh God, it's Monday",
'OH':"Overheard",
'OHHEMMGEE': "Oh My God",
'OI':"Operator indisposed",
'OIB':"Oh, I'm back",
'OIC':"Oh, I see",
'OJ':"Only joking",
'OL':"Old lady",
'OLL':"Online love",
'OM':"Old man",
'OM':"Oh, my",
'OMAA':"Oh, my aching *A*(butt)",
'OMDB':"Over my dead body",
'OMFG':"Oh my *freaking* God",
'omfglmaobbqrofl':"oh my *freaking* god, laugh my *a* off, owned, roll on floor spinning around I'm so sad",
'copteriss':"nan",
'OMG':"Oh my God",
'OMG':"Oh my gosh",
'OMGYG2BK':"Oh my God, you got to be kidding",
'OMGYS':"Oh my gosh you suck ",
'OMS':"On my soul(meaning ""promise"")",
'OMW':"On my way",
'ONL':"Online",
'OO':"Over and out",
'OOC':"Out of character",
'OOH':"Out of here",
'OOMF':"One of my followers",
'OOTD':"One of these days",
'OOTD':"Outfit of the day",
'OOTO':"Out of the office",
'OP':"On phone",
'ORLY':"Oh really?",
'OS':"Operating system",
'OT':"Off topic",
'OTB':"Off to bed",
'OTFL':"On the floor laughing",
'OTL':"Out to lunch",
'OTOH':"On the other hand",
'OTP':"On the phone",
'OTP':"One true pairing; two people you would love to see as a couple",
'OTT':"Over the top",
'OTTOMH':"Off the top of my head",
'OTW':"Off to work",
'OVA':"Over",
'OYO':"On your own",
'P':"Partner ",
'P2P':"Parent to parent",
'P2P':"Peer to peer",
'P2P':"Pay to play ",
'P911':"Parents coming into room alert",
'PAP':"Post a picture",
'PAT': "patrol"" ",
'PAW':"Parents are watching",
'PBOOK':"Phonebook(e-mail)",
'PC':"Player character ",
'PCM':"Please call me",
'PDA':"Personal display(of)affection",
'PDH':"Pretty darn happy",
'PDS':"Please don't shoot",
'PDQ':"Pretty darn quick",
'PEEPS':"People",
'PFT':"Pretty *freaking* tight",
'PIC':"Picture",
'PIP':"Peeing in pants",
'PIR':"Parents in room",
'PISS':"Put in some sugar",
'PITA':"Pain in the *butt*",
'PKMN':"Pokemon ",
'PL8':"Plate",
'PLD':"Played",
'PLMK':"Please let me know",
'PLS':"Please",
'PLU':"People like us",
'PLZ':"Please",
'PLZTLME':"Please tell me",
'PM':"Private Message",
'PMFI':"Pardon me for interrupting",
'PMFJI':"Pardon me for jumping in",
'PMSL':"Pee myself laughing",
'POAHF':"Put on a happy face",
'POIDH':"Picture, or it didn't happen",
'POS':"Parent over shoulder",
'POT': "potion"" ",
'POTD':"Photo of the day",
'POV':"Point of view",
'POV':"Privately owned vehicle",
'PPL':"People",
'PPU':"Pending pick-up",
'PRESH':"Precious",
'PROBS':"Probably",
'PROLLY':"Probably",
'PROGGY':"Meaning computer program",
'PRON':"Meaning pornography",
'PRT':"Party",
'PRT':"Please Retweet ",
'PRW':"People/parents are watching",
'PSA':"Public Service Announcement",
'PSOS':"Parent standing over shoulder",
'PSP':"Playstation Portable",
'PST':"Please send tell ",
'PTFO':"Pass the *freak* out",
'PTIYPASI':"Put that in your pipe and smoke it",
'PTL':"Praise the Lord",
'PTMM':"Please tell me more",
'PTO':"Paid time off",
'PTO':"Personal time off",
'PTO':"Parent Teacher Organization",
'PUG':"Pick up group ",
'PVE':"Player vs enemy, Player versus environment ",
'PVP':"Player versus player ",
'PWN': "own",
'PXT':"Please explain that",
'PU':"That stinks!",
'PUKS':"Pick up kids(SMS)",
'PYT':"Pretty young thing",
'PZ':"Peace ",
'PZA':"Pizza",
'Q':"Queue",
'Q4U':"(I have a)question for you",
'QC':"Quality control",
'QFE':"Question for everyone",
'QFI':"Quoted for idiocy",
'QFI':"Quoted for irony",
'QFT':"Quoted for truth",
'QIK':"Quick",
'QL':"Quit laughing",
'QOTD':"Quote of the day",
'QQ(qq )(Q_Q)': "crying eyes",
'QQ':"Quick question",
'QSL':"Reply ",
'QSO':"Conversation",
'QT':"Cutie",
'QTPI':"Cutie pie",
'R': "are",
'R8':"Rate(SMS)",
'RBAY':"Right back at you",
'RFN':"Right *freaking* now",
'RGR':"I agree",
'RHIP':"Rank has its privileges",
'RIP':"Rest in peace",
'RL':"Real life",
'RLY':"Really",
'RME':"Rolling my eyes",
'RMLB':"Read my lips baby",
'RMMM':"Read my mail man",
'ROFL':"Rolling on floor laughing",
'ROFLCOPTER':"Rolling on floor laughing and spinning around",
'ROFLMAO':"Rolling on the floor, laughing my *butt* off",
'ROTFL':"Rolling on the floor laughing",
'ROTFLUTS':"Rolling on the floor laughing unable to speak",
'RS':"Runescape ",
'RSN':"Real soon now",
'RT':"Roger that",
'RT':"Retweet ",
'RTBS':"Reason to be single",
'RTFM':"Read the *freaking* manual",
'RTFQ':"Read the *freaking* question",
'RTHX': "Thanks for the RT(Retweet)"" ",
'RTMS':"Read the manual, stupid",
'RTNTN':"Retention",
'RTRCTV':"Retroactive",
'RTRMT':"Retirement",
'RTSM':"Read the stupid manual",
'RTWFQ':"Read the whole *freaking* question",
'RU':"Are you?",
'RUMOF':"Are you male or female?",
'RUT':"Are u(you)there?",
'RUOK':"Are you okay?",
'RX':"Regards",
'RW':"Real world",
'RX':"Meaning drugs or prescriptions",
'RYB':"Read your Bible ",
'RYO':"Roll your own",
'RYS':"Read your screen ",
'RYS':"Are you single?",
'S2R':"Send to receive(meaning send me your picture to get mine)",
'S2S':"Sorry to say",
'S4L':"Spam for life",
'SAL':"Such a laugh",
'SAT':"Sorry about that",
'SAVAGE':"Slang for a shockingly careless expression or response to an event or message",
'SB':"Snap back",
'SB':"Should be",
'SB':"Smiling back",
'SBIA':"Meaning Standing back in amazement",
'SBT':"Sorry 'bout that",
'SC':"Stay cool",
'SD':"Sweet dreams",
'SDMB':"Sweet dreams, my baby",
'SENPAI':"Meaning someone older than you, someone you look up to",
'SEO':"Search engine optimization",
'SETE':"Smiling Ear-to-Ear",
'SELFIE':"A photo that is taken of oneself for social media sharing(definition)",
'SFAIK':"So far as I know",
'SH':"Same here",
'SH^':"Shut up",
'SHID':"Slapping head in disgust",
'SHIP':"Slang for ""wishing two people were in a relationship",
'SICNR':"Sorry, I could not resist",
'SIG2R':"Sorry, I got to run",
'SIHTH':"Stupidity is hard to take",
'SIMYC':"Sorry I missed your call",
'SIR':"Strike it rich",
'SIS':"Snickering in silence",
'SIS':"Meaning sister",
'SIT':"Stay in touch",
'SITD':"Still in the dark",
'SJW':"Social justice warrior",
'SK8':"Skate",
'SK8NG':"Skating",
'SK8R':"Skater",
'SK8RBOI':"Skater Boy",
'SLAP':"Sounds like a plan",
'Slay':"To succeed at something",
'SM':"Social media",
'SMAZED':"Smoky haze",
'SMEXI':"Combination of sexy and Mexican, used to describe attractive people",
'SMH':"Shaking my head",
'SMHID':"Scratching my head in disbelief",
'SNAFU':"Situation normal all fouled up",
'SNERT':"Snot nosed egotistical rude teenager",
'SNR':"Streaks and Recents(Snapchat)",
'SO':"Significant other",
'SOAB':"Son of a *B*",
'S'OK':"Meaning It'(s)okay ",
'SOL':"Sooner or later",
'SOMY':"Sick of me yet?",
'SorG':"Straight or Gay?",
'SOS':"Meaning help",
'SOS':"Son of Sam",
'SOT':"Short of time",
'SOTMG':"Short of time, must go",
'SOWM':"Someone with me",
'SPK':"Speak(SMS)",
'SRSLY':"Seriously",
'SPST':"Same place, same time",
'SPTO':"Spoke to",
'SQ':"Square",
'SRY':"Sorry",
'SS':"So sorry",
'SSDD':"Same stuff, different day",
'SSIF':"So stupid it's funny",
'SSINF':"So stupid it's not funny",
'ST&D':"Stop texting and drive",
'Stan':"Meaning a die-hard fan of someone",
'STFU':"Shut the *freak* up",
'STR8':"Straight",
'STW':"Search the Web",
'SU':"Shut up",
'SUITM':"See you in the morning",
'SUL':"See you later",
'SUP':"What's up?",
'SUTH':"So use to haters",
'SUX':"Meanings sucks or ""it sucks",
'SUYF':"Shut up you fool",
'SWAG':"Meaning free stuff and giveaways from tech tradeshows",
'SWAG':"Scientific wild *a* guess",
'SWAK':"Sent with a kiss",
'SWALK':"Sealed with a loving kiss",
'SWAT':"Scientific wild *butt* guess",
'SWL':"Screaming with laughter",
'SWMBO':"She who must be obeyed.Meaning wife or partner",
'SYL':"See you later ",
'SYS':"See you soon ",
'SYY':"Shut your yapper",
'T+':"Think positive",
'T4BU':"Thanks for being you",
'T:)T':"Think happy thoughts",
'TA':"Thanks a lot",
'TAFN':"That's all for now",
'TAM':"Tomorro a.m.",
'TANK':"really strong",
'TANKED': "owned",
'TANKING': "owning",
'TARFU':"Things Are Really *fouled* Up.",
'TAU':"Thinking about you",
'TAUMUALU':"Thinking about you miss you always love you",
'TBAG':"Process of disgracing a corpse, taunting a fragged/killed player ",
'TBBH':"To be brutally honest",
'TBC':"To be continued",
'TBD':"To be determined",
'TBH':"To be honest",
'TBL':"Text back later",
'TBT':"Throwback Thursday ",
'TC':"Take care",
'TCB':"Take care of business",
'TCOY':"Take care of yourself",
'TD':"Tower defense ",
'TD2M':"Talk dirty to me",
'TDTM':"Talk dirty to me",
'TEA':"Meaning gossip",
'TFF':"Too *freaking* funny",
'TFS':"Thanks for sharing",
'TFTF':"Thanks for the follow ",
'TFTI':"Thanks for the invitation",
'TFTT':"Thanks for this tweet ",
'TG':"Thank goodness",
'TGIF':"Thank God it's Friday",
'THNQ':"Thank-you(SMS)",
'THNX':"Thanks",
'THOT':"That wh*re over there",
'THT':"Think happy thoughts",
'THX':"Thanks",
'TIA':"Thanks in advance",
'TIAD':"Tomorrow is another day",
'TIC':"Tongue-in-cheek",
'TIL':"Today I learned",
'TILIS':"Tell it like it is",
'TIR':"Teacher in room",
'TLK2UL8R':"Talk to you later",
'TL':"Too long",
'TL;DR':"Too long; didn't read",
'TM':"Trust me",
'TMA':"Take my advice",
'TMB':"Text me back",
'TMB':"Tweet me back ",
'TMI':"Too much information",
'TMOT':"Trust me on this",
'TMTH':"Too much to handle",
'TMYL':"Tell me your location",
'TMWFI':"Take my word for it",
'TNSTAAFL':"There's no such thing as a free lunch",
'TNT':"Til next time",
'TOJ':"Tears of joy",
'TOS':"Terms of service",
'TOTES':"Totally",
'TOU':"Thinking of you",
'TOY':"Thinking of you",
'TPM':"Tomorrow p.m.",
'TPTB':"The powers that be",
'TQ':"Te quiero / I love you(Spanish SMS)",
'TSH':"Tripping so hard",
'TSNF':"That's so not fair",
'TSTB':"The sooner, the better",
'TT':"Trending topic ",
'TTFN':"Ta ta for now",
'TTLY':"Totally",
'TTTT':"These things take time",
'TTUL':"Talk to you later",
'TU':"Thank you",
'TUI':"Turning you in",
'TURNT':"Meaning excitement, turned up",
'TWSS':"That's what she said",
'TTG':"Time to go",
'TTYAFN':"Talk to you awhile from now",
'TTYL':"Talk to you later",
'TTYS':"Talk to you soon",
'TY':"Thank you",
'TYFC':"Thank you for charity ",
'TYFYC':"Thank you for your comment",
'TYS':"Told you so",
'TYT':"Take your time",
'TYSO':"Thank you so much",
'TYAFY':"Thank you and *freak* you",
'TYVM':"Thank you very much",
'TX':"Thanks",
'^URS':"Up yours",
'UCMU':"You crack me up",
'UDI':"Unidentified drinking injury(meaning bruise, scratch, ache and so on)",
'UDM':"You the man",
'UDS':"Ugly domestic scene",
'UFB':"Un *freaking* believable",
'UFN':"Until further notice",
'UFWM':"You *freaking* with me?",
'UGTBK':"You've got to be kidding",
'UHGTBSM':"You have got to be s
'UKTR':"You know that's right",
'UL':"Upload",
'U-L': "You will",
'UNA':"Use no acronyms",
'UN4TUN8':"Unfortunate",
'UNBLEFBLE':"Unbelievable",
'UNCRTN':"Uncertain",
'UNPC':"Un-(not)politically correct",
'UOK':"Are You ok?",
'UR':"You are / You're",
'UR2YS4ME':"You are too wise for me",
'URA*':"You are a star",
'URH':"You are hot(U R Hot)",
'URSKTM':"You are so kind to me",
'URTM':"You are the man",
'URW':"You are welcome",
'USBCA':"Until something better comes along",
'USU':"Usually",
'UT':"Unreal Tournament ",
'UT2L':"You take too long",
'UTM':"You tell me",
'UV':"Unpleasant visual",
'UW':"You're welcome",
'UX':"User experience",
'V':"Very",
'VAT':"Value added tax",
'VBL':"Visible bra line",
'VBS':"Very big smile",
'VC':"Voice chat",
'VEG':"Very evil grin",
'VFF':"Very freaking funny",
'VFM':"Value for money",
'VGC':"Very good condition",
'VGG':"Very good game ",
'VGH':"Very good hand ",
'VIP':"Very important person",
'VM':"Voice mail",
'VN':"Very nice",
'VNH':"Very nice hand ",
'VoIP':"Voice over Internet Protocol(definition)",
'VRY':"Very",
'VSC':"Very soft chuckle",
'VSF':"Very sad face",
'VWD':"Very well done ",
'VWP':"Very well played ",
'W@':"What?",
'W/':"With",
'W/B':"Welcome back",
'W3':"WWW(Web address)",
'W8':"Wait",
'WAH':"Working at home",
'WAJ':"What a jerk",
'WAM':"Wait a minute",
'WAN2':"Want to?(SMS)",
'WAN2TLK':"Want to talk",
'WAREZ':"Meaning pirated(illegally gained)software",
'WAS':"Wait a second",
'WAS':"Wild *a* guess",
'WAT':"What",
'WAWA':"Where are we at?",
'WAYF':"Where are you from?",
'W/B':"Write back",
'WB':"Welcome back",
'WBS':"Write back soon",
'WBU':"What about you?",
'WC':"Welcome",
'WC':"Who cares",
'WCA':"Who cares anyway",
'W/E':"Whatever",
'W/END':"Weekend",
'WE':"Whatever",
'WEBO':"Webopedia",
'WEEBO':"Describes a person obsessed with of Japanese culture",
'WEP':"Weapon ",
'WH5':"Who, what, when, where, why",
'WHATEVES':"Whatever",
'WIBNI':"Wouldn't it be nice if",
'WDALYIC':"Who died and left you in charge",
'WDYK':"What do you know?",
'WDYT':"What do you think?",
'WGACA':"What do you think?",
'WIIFM':"What's in it for me?",
'WISP':"Winning is so pleasurable",
'WITP':"What is the point?",
'WITW':"What in the world",
'WIU':"Wrap it up",
'WK':"Week",
'WKD':"Weekend",
'WRT':"With regard to",
'WL':"Whatta loser",
'W/O':"Without",
'WOA':"Work of Art",
'WOKE':"Slang for people who are aware of current social issues, and politics",
'WOMBAT':"Waste of money, brains and time",
'WOW':"World of Warcraft ",
'WRK':"Work",
'WRU':"Where are you?",
'WRU@':"Where are you at?",
'WRUD':"What are you doing?",
'WTB':"Want to buy ",
'WTF':"What the *freak* ?",
'WTFE':"What the *freak* ever",
'WTFO':"What the *freak* ?, over.",
'WTG':"Way to go",
'WTGP':"Want to go private(talk out of public chat area)",
'WTH':"What the heck?",
'WTM':"Who's the man?",
'WTS':"Want to sell? ",
'WTT':"Want to trade? ",
'WU':"What's up?",
'WUCIWUG':"What you see is what you get",
'WUF':"Where are you from?",
'WUP':"What's up?",
'WUT': "what",
'WUW':"What u(you)want?",
'WUU2':"What are you up to?",
'WUZ': "was",
'WWJD':"What would Jesus do?",
'WWJD':"What Would Judd Do?(Artist Donald Judd, who was known to do things in his own compulsive way.) ",
'WWNC':"Will wonders never cease",
'WWYC':"Write when you can",
'WYCM':"Will you call me?",
'WYD':"What(are)you doing?",
'WYGAM':"When you get a minute",
'WYHAM':"When you have a minute",
'WYLEI':"When you least expect it",
'WYSIWYG':"What you see is what you get ",
'WYWH':"Wish you were here",
'X-1-10': "Exciting",
'X':"Kiss",
'X!': "a typical woman",
'XD': "really hard laugh""(where D is a smiley mouth)",
'XD':"Meaning a ""devilish smile",
'XME':"Excuse Me",
'XOXOXO':"Hugs & Kisses",
'XLNT':"Excellent ",
'XLR8': "faster"" or ""going faster",
'XPOST':"Meaning Cross-post.A link posted to a subreddit that was already shared on a different subreddit(Reddit)",
'XYL':"Ex-young lady, meaning wife.( amateur radio)",
'XYZ':"Examine your zipper",
'Y?':"Why?",
'Y':"Meaning Yawn",
'Y2K':"You're too kind",
'YA':"Your",
'YAA':"Yet another acronym",
'YABA':"Yet another bloody acronym",
'YARLY':"Ya, really?",
'YAS': "praise",
'YBIC':"Your brother in Christ",
'YBS':"You'll be sorry",
'YCDBWYCID':"You can't do business when your computer is down",
'YCHT':"You can have them",
'YCLIU':"You can look it up",
'YCMU':"You crack me up",
'YCT':"Meaning Your comment to?",
'YD':"Yesterday",
'YEET':"Meaning excitement, approval or display of energy(i.e.throwing something)",
'YF':"Wife",
'YG':"Young gentleman",
'YGG':"You go girl",
'YGTBKM':"You've got to be kidding me",
'YGTR':"You got that right",
'YHBT':"You have been trolled",
'YHBW':"You have been warned",
'YHL':"You have lost",
'YIU':"Yes, I understand",
'YKW':"You know what",
'YKWYCD':"You know what you can do",
'YL':"Young lady",
'YMMV':"Your mileage may vary",
'YNK':"You never know",
'YOLO':"You only live once",
'YR':"Your",
'YR':"Yeah right",
'YRYOCC':"You're running your own cuckoo clock",
'YSIC':"Your sister in Christ",
'YSYD':"Yeah sure you do",
'YT':"YouTube",
'YT':"You there?",
'YTB':"You're the best",
'YTB':"Youth talk back",
'YTTL':"You take too long",
'YTG':"You're the greatest",
'YW':"You're welcome",
'YWHNB':"Yes, we have no bananas",
'YWHOL':"Yelling ""woohoo"" out loud",
'YWSYLS':"You win some, you lose some",
'YYSSW':"Yeah, yeah, sure, sure, whatever",
'Z':"Zero",
'Z':"Z's are calling(meaning going to bed/sleep)",
'Z': "Said",
'Z%':"Zoo",
'ZH':"Sleeping Hour",
'ZOMG':"Used in World of Warcraft to mean OMG(Oh My God)",
'ZOT':"Zero tolerance",
'ZUP': "What's up?",
'ZZZZ':"Sleeping(or bored)"
}
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name, mini in tqdm(df.groupby('image_id')) :
path2save = 'val2017/' if name in val_index else 'train2017/'
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save, exist_ok=True)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save), exist_ok=True)
 sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name)) 
<concatenate>
def convert_Abbreviation(text):
for abb in Abbreviation.items() :
if(not abb[0].isdigit())and(" "+abb[0]+" " in text)and(len(abb[0])>3):
text = text.replace(abb[0],abb[1])
 return text
def run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, NMS_CONF_THR, NMS_IOU_THR, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<normalization>
bracket_pattern = re.compile('\[|\]')
 time_pattern = re.compile('\d+(( \\|\/)\d+)+')
 number_pattern = re.compile('\d+(,|\.|\d+)*')
 including_number_pattern = re.compile('\s\d+(\W*\d)*\s')
 hashtag_pattern = re.compile('
alpha_pattern = re.compile('@')
 remove_except_chracter_pattern = re.compile('[ ](?=[ ])|[^A-Za-z\s]+')
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def validate() :
source = 'convertor/fold0/images/val2017'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 records = marking[marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes = gtboxes.astype(np.int32 ).clip(min=0, max=1024)
 gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
result = {
'image_id': image_id,
'pred_enboxes': enboxes,
'pred_enscores': enscores,
'gt_boxes': gtboxes,
}
results.append(result)
 return results
def calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
enboxes = all_predictions[i]['pred_enboxes'].copy()
enscores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes = pred_boxes.astype(np.int32 ).clip(min=0, max=1024)
 indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores)
 def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'.. /input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth" )<train_model>
def remove_reg(text, reg):
if reg == time_pattern:
return reg.sub(r' TIME ', text)
 elif reg == number_pattern:
return reg.sub(r'00', text)
 else:
return reg.sub(r' ', text )
def log(text):
print(text)
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*10)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<load_pretrained>
remove_reg("sd1:2f",time_pattern )
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<categorify>
repeated_pattern = re.compile(r'(\!|\? )(( \!|\?){1,})')
 unnecessary_pattern = re.compile(r"'|"")
 def remove_punctuation(text):

 text = unnecessary_pattern.sub(r' ', text)
 return repeated_pattern.sub(r'\1', text )
if PSEUDO or VALIDATE:
convertTrainLabel()<find_best_params>
remove_punctuation("1!! !!! ???? ?
if VALIDATE and is_TEST:
all_predictions = validate()

 for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
<load_pretrained>
redundant_white_spaces_pattern = re.compile(r'\s+')
 redundant_url = re.compile(r'url(\s*url)+')
 redundant_id = re.compile(r'id(\s*id)+')
 def remove_redundant_white_spaces(text):
text = redundant_url.sub(r'\1', text)
 text = redundant_id.sub(r'\1', text)
 text = text.strip()
text = redundant_white_spaces_pattern.sub(r' ', text)
 return text
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
if count<10:
img_ = cv2.imread(path)
 for box, score in zip(boxes,scores):
cv2.rectangle(img_,(box[0], box[1]),(box[2]+box[0], box[3]+box[1]),(220, 0, 0), 2)
 cv2.putText(img_, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(img_)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
reg_list = [bracket_pattern, number_pattern, time_pattern, including_number_pattern, hashtag_pattern, alpha_pattern, remove_except_chracter_pattern]
def preprocessor(dataset, function_list, columns):
for column in columns:
cleaned_colname = column+"_cleaned"
dataset[cleaned_colname] = dataset[column]
for i, function in enumerate(function_list):
if function in reg_list:
dataset[cleaned_colname] = dataset[cleaned_colname].apply(lambda text: remove_reg(text, function)if text is not np.nan else "")
 print(i,")",function, "is done.")
 else:
dataset[cleaned_colname] = dataset[cleaned_colname].apply(lambda text: function(text)if text is not np.nan else "")
 print(i,")",function.__name__, "is done.")
 dataset[cleaned_colname].replace('', np.nan, inplace=True)
 print("For", column, "column, preprocessing is done.")
 return None
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<define_variables>
function_list = [
remove_words_containing_alpha_by_threshold,
clean_1_2_1,clean_1_2_2,clean_1_2_3,clean_1_2_4, clean_1_2_6,clean_1_2_7,clean_1_2_8,
remove_url,
convert_emoticons,
mylower,
bracket_pattern,
time_pattern,
including_number_pattern,
hashtag_pattern,
alpha_pattern,
remove_punctuation,
remove_redundant_white_spaces
]
preprocess_target_cols = ['text']
sys.path.insert(0, ".. /input/weightedboxesfusion")
 sys.path.append(".. /input/yolov5train")
 NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.5
best_iou_thr = 0.6
best_skip_box_thr = 0.43
best_final_score = 0
best_score_threshold = 0
EPO = 15
WEIGHTS = '.. /input/yolov5weight/last0712.pt'
CONFIG = '.. /input/wheatyolov5/yolov5x.yaml'
DATA = '.. /input/configyolo5/wheat0.yaml'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >11
is_AUG = True
is_ROT = True
VALIDATE = True
PSEUDO = True
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<load_pretrained>
def compare_2col(df, target_colname, base_colname):
result_colname = target_colname+"_from_"+base_colname
df[result_colname] = df.apply(lambda row :
row[base_colname] != row[target_colname], axis=1)
 total = df.shape[0]
changed = len(df[df[result_colname]==True])
 print(f'{base_colname}에서 {target_colname}가 되면서
전체 {total}개 중 {changed}개({(changed/total)*100 :0.1f}%)가 변함')
 return None
def check_cleaning(df, func, target_colname, base_colname,print_num=10):
result_colname = target_colname+"_from_"+base_colname
if func != None:
if func in reg_list:
df[target_colname] = df[base_colname].apply(lambda text: remove_reg(text, func))
print(func, "is done.")
 else:
df[target_colname] = df[base_colname].apply(lambda text: func(text))
print(func.__name__, "is done.")
 compare_2col(df, target_colname, base_colname)
 display(df.loc[df[result_colname]==True, ['target',base_colname,target_colname]].head(print_num))
df.drop(columns=[result_colname], axis=1, inplace=True)
 return None
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id))<feature_engineering>
df_temp = dataset_origin_df.copy()
func_num = 14
preprocessor(df_temp, function_list[:func_num], preprocess_target_cols)
 df_temp['text'] = df_temp['text_cleaned']
def convertTrainLabel() :
df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name, mini in tqdm(df.groupby('image_id')) :
path2save = 'val2017/' if name in val_index else 'train2017/'
os.makedirs('convertor/fold{}/labels/'.format(fold)+path2save, exist_ok=True)
 with open('convertor/fold{}/labels/'.format(fold)+path2save+name+".txt", 'w+')as f:
row = mini[['classes','x_center','y_center','w','h']].astype(float ).values
row = row/1024
row = row.astype(str)
 for j in range(len(row)) :
text = ' '.join(row[j])
 f.write(text)
 f.write("
")
 os.makedirs('convertor/fold{}/images/{}'.format(fold,path2save), exist_ok=True)
 sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/fold{}/images/{}/{}.jpg'.format(fold,path2save,name))<concatenate>
check_cleaning(df_temp, function_list[func_num],'text_cleaned', 'text',100)
def run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, NMS_CONF_THR, NMS_IOU_THR, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<normalization>
df_temp[df_temp['text'].str.contains('camilacabello97', na=False, case=False)]
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def validate() :
source = 'convertor/fold0/images/val2017'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 records = marking[marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes = gtboxes.astype(np.int32 ).clip(min=0, max=1024)
 gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
result = {
'image_id': image_id,
'pred_enboxes': enboxes,
'pred_enscores': enscores,
'gt_boxes': gtboxes,
}
results.append(result)
 return results
def calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
enboxes = all_predictions[i]['pred_enboxes'].copy()
enscores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes = pred_boxes.astype(np.int32 ).clip(min=0, max=1024)
 indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores)
 def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'.. /input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth" )<train_model>
preprocessor(train_df, function_list, preprocess_target_cols)
 preprocessor(test_df, function_list, preprocess_target_cols )
def log(text):
print(text)
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*10)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<load_pretrained>
def remove_cleaned_col(df):
for col in preprocess_target_cols:
cleaned_colname = col+"_cleaned"
df[col] = df[cleaned_colname]
df.drop(columns=[cleaned_colname], axis=1, inplace=True)
 remove_cleaned_col(train_df)
 remove_cleaned_col(test_df )
def makePseudolabel() :
source = '.. /input/global-wheat-detection/test/'
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 path2save = 'train2017/'
if not os.path.exists('convertor/fold0/labels/'+path2save):
os.makedirs('convertor/fold0/labels/'+path2save)
 if not os.path.exists('convertor/fold0/images/{}'.format(path2save)) :
os.makedirs('convertor/fold0/images/{}'.format(path2save))
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
lineo = ''
for box in boxes:
x1, y1, w, h = box
xc, yc, w, h =(x1+w/2)/1024,(y1+h/2)/1024, w/1024, h/1024
lineo += '0 %f %f %f %f
'%(xc, yc, w, h)
 fileo = open('convertor/fold0/labels/'+path2save+image_id+".txt", 'w+')
 fileo.write(lineo)
 fileo.close()
sh.copy(".. /input/global-wheat-detection/test/{}.jpg".format(image_id),'convertor/fold0/images/{}/{}.jpg'.format(path2save,image_id)) 
<categorify>
mislabeled_corrected_df = pd.read_csv('/kaggle/input/nlp-wdt-hapjeong/mislabeled_corrected_V19.csv')
if PSEUDO or VALIDATE:
convertTrainLabel()<find_best_params>
i=0
for index, row in mislabeled_corrected_df[mislabeled_corrected_df['target'].isin([0.0,1.0])].iterrows() :
i=i+1
train_df.loc[train_df['id'] == row['id'], 'target'] = int(row['target'])
 print(i )
if VALIDATE and is_TEST:
all_predictions = validate()

 for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, best_iou_thr, best_skip_box_thr, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30 )<load_pretrained>
i=0
for index, row in mislabeled_corrected_df[~mislabeled_corrected_df['target'].isin([0.0,1.0])].iterrows() :
i=i+1
train_df = train_df[train_df['id'] != row['id']]
print(i )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def detect() :
source = '.. /input/global-wheat-detection/test/'
weights = 'weights/best.pt'
if not os.path.exists(weights):
weights = WEIGHTS
imagenames = os.listdir(source)
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = torch.load(weights, map_location=device)['model'].float()
model.to(device ).eval()
dataset = LoadImages(source, img_size=1024)
 results = []
fig, ax = plt.subplots(5, 2, figsize=(30, 70))
count = 0
for path, img, img0, vid_cap in dataset:
image_id = os.path.basename(path ).split('.')[0]
img = img.transpose(1,2,0)
 enboxes = []
enscores = []
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=is_AUG)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
indices = scores >= best_score_threshold
boxes = boxes[indices]
scores = scores[indices]
if count<10:
img_ = cv2.imread(path)
 for box, score in zip(boxes,scores):
cv2.rectangle(img_,(box[0], box[1]),(box[2]+box[0], box[3]+box[1]),(220, 0, 0), 2)
 cv2.putText(img_, '%.2f'%(score),(box[0], box[1]), cv2.FONT_HERSHEY_SIMPLEX, 0.5,(255,255,255), 2, cv2.LINE_AA)
 ax[count%5][count//5].imshow(img_)
 count+=1
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 return results<save_to_csv>
train_df.to_csv('train_cleaned.csv', index=False)
 test_df.to_csv('test_cleaned.csv', index=False)
results = detect()
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
import numpy as np
import pandas as pd
import time
from datetime import datetime
!pip uninstall -y tensorflow
!pip install chainer-chemistry==0.5.0<import_modules>
train = pd.read_csv('train_cleaned.csv')
 test = pd.read_csv('test_cleaned.csv' )
import random
import numpy as np
import pandas as pd
import chainer
import chainer_chemistry
from IPython.display import display<load_from_csv>
from google.cloud import storage, automl_v1beta1 as automl
from google.api_core.gapic_v1.client_info import ClientInfo
from automlwrapper import AutoMLWrapper
def load_dataset() :
train = pd.merge(pd.read_csv('.. /input/champs-scalar-coupling/train.csv'),
pd.read_csv('.. /input/champs-scalar-coupling/scalar_coupling_contributions.csv'))
test = pd.read_csv('.. /input/champs-scalar-coupling/test.csv')
 counts = train['molecule_name'].value_counts()
moles = list(counts.index)
 random.shuffle(moles)
 num_train = int(len(moles)* 0.9)
 train_moles = sorted(moles[:num_train])
 valid_moles = sorted(moles[num_train:])
 test_moles = sorted(list(set(test['molecule_name'])))
 valid = train.query('molecule_name not in @train_moles')
 train = train.query('molecule_name in @train_moles')
 train.sort_values('molecule_name', inplace=True)
 valid.sort_values('molecule_name', inplace=True)
 test.sort_values('molecule_name', inplace=True)
 return train, valid, test, train_moles, valid_moles, test_moles
train, valid, test, train_moles, valid_moles, test_moles = load_dataset()
train_gp = train.groupby('molecule_name')
 valid_gp = valid.groupby('molecule_name')
 test_gp = test.groupby('molecule_name')
 structures = pd.read_csv('.. /input/champs-scalar-coupling/structures.csv')
 structures_groups = structures.groupby('molecule_name' )<categorify>
PROJECT_ID = 'kaggle-nlp-wdt'
BUCKET_NAME = 'kaggle-nlp-wdt-lcm'
region = 'us-central1'
storage_client = storage.Client(project=PROJECT_ID)
 client = automl.AutoMlClient(client_info=ClientInfo())
 print(f'Starting AutoML notebook at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}' )
class Graph:
def __init__(self, points_df, list_atoms):
self.points = points_df[['x', 'y', 'z']].values
self._dists = distance.cdist(self.points, self.points)
 self.adj = self._dists < 1.5
self.num_nodes = len(points_df)
 self.atoms = points_df['atom']
dict_atoms = {at: i for i, at in enumerate(list_atoms)}
atom_index = [dict_atoms[atom] for atom in self.atoms]
one_hot = np.identity(len(dict_atoms)) [atom_index]
bond = np.sum(self.adj, 1)- 1
bonds = np.identity(len(dict_atoms)) [bond - 1]
self._array = np.concatenate([one_hot, bonds], axis=1 ).astype(np.float32)
 @property
def input_array(self):
return self._array
@property
def dists(self):
return self._dists.astype(np.float32 )<create_dataframe>
VERSION = 'V19'
BUCKET_PATH = 'preprocessing/'+VERSION+'/'
FILE_NAME = 'train_cleaned'+'_'+VERSION
training_gcs_path = BUCKET_PATH+FILE_NAME+'.csv'
dataset_display_name = FILE_NAME
model_display_name = 'model_'+FILE_NAME
train_dataset = DictDataset(graphs=train_graphs, targets=train_targets)
 valid_dataset = DictDataset(graphs=valid_graphs, targets=valid_targets)
 test_dataset = DictDataset(graphs=test_graphs, targets=test_targets )<feature_engineering>
train.loc[:,['text','target']].to_csv('train.csv', index=False, header=False )
class SchNetUpdateBN(SchNetUpdate):
def __init__(self, *args, **kwargs):
super(SchNetUpdateBN, self ).__init__(*args, **kwargs)
 with self.init_scope() :
self.bn = GraphBatchNormalization(args[0])
 def __call__(self, h, adj, **kwargs):
v = self.linear[0](h)
 v = self.cfconv(v, adj)
 v = self.linear[1](v)
 v = F.softplus(v)
 v = self.linear[2](v)
 return h + self.bn(v)
 class SchNet(chainer.Chain):
def __init__(self, num_layer=3):
super(SchNet, self ).__init__()
self.num_layer = num_layer
with self.init_scope() :
self.gn = GraphLinear(512)
 for l in range(self.num_layer):
self.add_link('sch{}'.format(l), SchNetUpdateBN(512))
self.interaction1 = L.Linear(128)
 self.interaction2 = L.Linear(128)
 self.interaction3 = L.Linear(4)
 def __call__(self, input_array, dists, pairs_index, targets):
out = self.predict(input_array, dists, pairs_index)
 loss = F.mean_absolute_error(out, targets)
 reporter.report({'loss': loss}, self)
 return loss
def predict(self, input_array, dists, pairs_index, **kwargs):
h = self.gn(input_array)
 for l in range(self.num_layer):
h = self['sch{}'.format(l)](h, dists)
 h = F.concat(( h, input_array), axis=2)
 concat = F.concat([
h[pairs_index[:, 0], pairs_index[:, 1], :],
h[pairs_index[:, 0], pairs_index[:, 2], :],
F.expand_dims(dists[pairs_index[:, 0],
pairs_index[:, 1],
pairs_index[:, 2]], 1)
 ], axis=1)
 h1 = F.leaky_relu(self.interaction1(concat))
h2 = F.leaky_relu(self.interaction2(h1))
out = self.interaction3(h2)
 return out
model = SchNet(num_layer=3)
 model.to_gpu(device=0 )<define_variables>
bucket = storage.Bucket(storage_client, name=BUCKET_NAME)
 if not bucket.exists() :
bucket.create(location=BUCKET_REGION )
class SameSizeSampler(OrderSampler):
def __init__(self, structures_groups, moles, batch_size,
random_state=None, use_remainder=False):
self.structures_groups = structures_groups
self.moles = moles
self.batch_size = batch_size
if random_state is None:
random_state = np.random.random.__self__
self._random = random_state
self.use_remainder = use_remainder
def __call__(self, current_order, current_position):
batches = list()
atom_counts = pd.DataFrame()
atom_counts['mol_index'] = np.arange(len(self.moles))
atom_counts['molecular_name'] = self.moles
atom_counts['num_atom'] = [len(self.structures_groups.get_group(mol))
for mol in self.moles]
num_atom_counts = atom_counts['num_atom'].value_counts()
for count, num_mol in num_atom_counts.to_dict().items() :
if self.use_remainder:
num_batch_for_this = -(-num_mol // self.batch_size)
 else:
num_batch_for_this = num_mol // self.batch_size
target_mols = atom_counts.query('num_atom==@count')['mol_index'].values
random.shuffle(target_mols)
 devider = np.arange(0, len(target_mols), self.batch_size)
 devider = np.append(devider, 99999)
 if self.use_remainder:
target_mols = np.append(
 target_mols,
np.repeat(target_mols[-1], -len(target_mols)% self.batch_size))
for b in range(num_batch_for_this):
batches.append(target_mols[devider[b]:devider[b + 1]])
 random.shuffle(batches)
 batches = np.concatenate(batches ).astype(np.int32)
 return batches
batch_size = 8
train_sampler = SameSizeSampler(structures_groups, train_moles, batch_size)
 valid_sampler = SameSizeSampler(structures_groups, valid_moles, batch_size,
use_remainder=True)
 test_sampler = SameSizeSampler(structures_groups, test_moles, batch_size,
use_remainder=True )<choose_model_class>
def upload_blob(bucket_name, source_file_name, destination_blob_name):

 bucket = storage_client.get_bucket(bucket_name)
 blob = bucket.blob(destination_blob_name)
 blob.upload_from_filename(source_file_name)
 print('File {} uploaded to {}'.format(
 source_file_name,
'gs://' + bucket_name + '/' + destination_blob_name))
def download_to_kaggle(bucket_name,destination_directory,file_name,prefix=None):

 os.makedirs(destination_directory, exist_ok = True)
 full_file_path = os.path.join(destination_directory, file_name)
 blobs = storage_client.list_blobs(bucket_name,prefix=prefix)
 for blob in blobs:
blob.download_to_filename(full_file_path )
optimizer = optimizers.Adam(alpha=1e-3)
 optimizer.setup(model )<categorify>
upload_blob(BUCKET_NAME, 'train.csv', training_gcs_path)
def coupling_converter(batch, device):
list_array = list()
list_dists = list()
list_targets = list()
list_pairs_index = list()
with_target = 'fc' in batch[0]['targets'].columns
for i, d in enumerate(batch):
list_array.append(d['graphs'].input_array)
 list_dists.append(d['graphs'].dists)
 if with_target:
list_targets.append(
 d['targets'][['fc', 'sd', 'pso', 'dso']].values.astype(np.float32))
sample_index = np.full(( len(d['targets']), 1), i)
 atom_index = d['targets'][['atom_index_0', 'atom_index_1']].values
list_pairs_index.append(np.concatenate([sample_index, atom_index], axis=1))
input_array = to_device(device, np.stack(list_array))
dists = to_device(device, np.stack(list_dists))
pairs_index = np.concatenate(list_pairs_index)
 array = {'input_array': input_array, 'dists': dists, 'pairs_index': pairs_index}
if with_target:
array['targets'] = to_device(device, np.concatenate(list_targets))
return array
updater = training.StandardUpdater(train_iter, optimizer,
converter=coupling_converter, device=0)
 trainer = training.Trainer(updater,(41, 'epoch'), out="result" )<import_modules>
amw = AutoMLWrapper(client=client,
project_id=PROJECT_ID,
bucket_name=BUCKET_NAME,
region='us-central1',
dataset_display_name=dataset_display_name,
model_display_name=model_display_name)
class TypeWiseEvaluator(Evaluator):
def __init__(self, iterator, target, converter, device, name,
is_validate=False, is_submit=False):
super(TypeWiseEvaluator, self ).__init__(
 iterator, target, converter=converter, device=device)
 self.is_validate = is_validate
self.is_submit = is_submit
self.name = name
def calc_score(self, df_truth, pred):
target_types = list(set(df_truth['type']))
diff = df_truth['scalar_coupling_constant'] - pred
scores = 0
metrics = {}
for target_type in target_types:
target_pair = df_truth['type'] == target_type
score_exp = np.mean(np.abs(diff[target_pair]))
scores += np.log(score_exp)
 metrics[target_type] = scores
metrics['ALL_LogMAE'] = scores / len(target_types)
 observation = {}
with reporter.report_scope(observation):
reporter.report(metrics, self._targets['main'])
 return observation
def evaluate(self):
iterator = self._iterators['main']
eval_func = self._targets['main']
iterator.reset()
it = iterator
y_total = []
t_total = []
for batch in it:
in_arrays = self.converter(batch, self.device)
 with chainer.no_backprop_mode() , chainer.using_config('train', False):
y = eval_func.predict(**in_arrays)
 y_data = cuda.to_cpu(y.data)
 y_total.append(y_data)
 t_total.extend([d['targets'] for d in batch])
 df_truth = pd.concat(t_total, axis=0)
 y_pred = np.sum(np.concatenate(y_total), axis=1)
 if self.is_submit:
submit = pd.DataFrame()
submit['id'] = df_truth['id']
submit['scalar_coupling_constant'] = y_pred
submit.drop_duplicates(subset='id', inplace=True)
 submit.sort_values('id', inplace=True)
 submit.to_csv('kernel_schnet.csv', index=False)
 if self.is_validate:
return self.calc_score(df_truth, y_pred)
 return {}
trainer.extend(
 TypeWiseEvaluator(iterator=valid_iter, target=model, converter=coupling_converter,
name='valid', device=0, is_validate=True))
trainer.extend(
 TypeWiseEvaluator(iterator=test_iter, target=model, converter=coupling_converter,
name='test', device=0, is_submit=True))<train_model>
print(f'Getting dataset ready at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 if not amw.get_dataset_by_display_name(dataset_display_name):
print('dataset not found')
 amw.create_dataset()
amw.import_gcs_data(training_gcs_path)
 amw.dataset
print(f'Dataset ready at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
chainer.config.train = True
trainer.run()<install_modules>
print(f'Getting model trained at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 if not amw.get_model_by_display_name(model_display_name):
print(f'Training model at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 amw.train_model()
print(f'Model trained.Ensuring model is deployed at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 amw.deploy_model()
amw.model
print(f'Model trained and deployed at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
!pip install tensorflow-gpu==2.0a0<import_modules>
amw.model_full_path
print(tf.__version__ )<set_options>
print(f'Begin getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}')
 prediction_client = automl.PredictionServiceClient()
amw.set_prediction_client(prediction_client)
 predictions_df = amw.get_predictions(test,
input_col_name='text',
limit=None,
threshold=0.5,
verbose=False)
 print(f'Finished getting predictions at {datetime.fromtimestamp(time.time() ).strftime("%Y-%m-%d, %H:%M:%S UTC")}' )
tf.test.is_gpu_available(
 cuda_only=False,
min_cuda_compute_capability=None 
)
<define_variables>
submission_df = pd.concat([test['id'], predictions_df['class']], axis=1)
 submission_df.head()
tf.random.set_seed(42)
 datadir = ".. /input/"<choose_model_class>
submission_df = submission_df.rename(columns={'class':'target'})
 submission_df.head()
<normalization><EOS> 
submission_df.to_csv("submission.csv", index=False, header=True )