Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<init_hyperparams>
warnings.simplefilter(action='ignore', category=FutureWarning)
TIME_LIMIT = 6.9*60*60
Origin = False
p_HF = True
p_ROT = False
eimg_size = 1024
pesudo_thres = 0.4
pesudo_iou = 0.35
pesudo_score_threshold = 0
img_size = 1024
NMS_IOU_THR = 0.6
NMS_CONF_THR = 0.35
best_iou_thr = 0.35
best_skip_box_thr = 0.35
best_final_score = 0
best_score_threshold = 0.1
EPO = 8
WEIGHTS = ".. /input/yolov4-pytorch/last_trainF5_onlyfix.weights"
PESUDO_WEIGHTS = '.. /input/effdet-cutmix-augmix-result/best-checkpoint_d7.bin'
CONFIG = '.. /input/yolov4-pytorch/cd53s_self_1class.cfg'
OPTIMIZER = '.. /input/yolov4-pytorch/last_optimizer.pth'
is_TEST = len(os.listdir('.. /input/global-wheat-detection/test/')) >11
is_AUG = False
is_ROT = True
VALIDATE = False
PSEUDO = True
Yolov4 = True
Effdet = True
GET_PESUDO = True
csv_file = '.. /input/yolov4-pytorch/newtrain.csv'
marking = pd.read_csv(csv_file)
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True)
 max_label_numbers = 0
pesudo_result = []<install_modules>
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
sys.path.insert(0, ".. /input/timm-efficientdet-pytorch")
 sys.path.insert(0, ".. /input/omegaconf")
<feature_engineering>
train_dir = ".. /input/nlp-getting-started/train.csv"
df = pd.read_csv(train_dir)
 x =(df['text'] 
.str.lower() 
.str.replace('\x89Ûª|Ûª', "'")
.str.replace('
|\x89.|\x9d *', ' ')
.str.replace('&gt;', ">")
.str.replace('&lt;', "<")
.str.replace('&amp;', " and ")
.str.replace('won't', 'will not')
.str.replace('can't', 'cannot')
.str.replace('i'm', 'i am')
.str.replace('ain't', 'is not')
.str.replace('hwy.', 'highway')
.str.replace('(\w+)'ll', '\g<1> will')
.str.replace('(\w+)n't', '\g<1> not')
.str.replace('(\w+)'ve', '\g<1> have')
.str.replace('(\w+)'s', '\g<1> is')
.str.replace('(\w+)'re', '\g<1> are')
.str.replace('(\w+)'d', '\g<1> would')
.str.replace('(\w+)'m', '\g<1> am')
.str.replace('<3', 'love')
.str.replace('w/e', 'whatever')
.str.replace('w/', 'with')
.str.replace('\b', ' ')
.str.replace('-', ' ')
.str.replace(' *', ' ')
.str.strip() 
.to_numpy() 
)
 y = df['target'].to_numpy()
x_train, x_val, y_train, y_val = train_test_split(x, y, train_size = 0.7, random_state = 1000 )
def convertTrainLabel(max_label_numbers):
df = pd.read_csv(csv_file)
 bboxs = np.stack(df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
df[column] = bboxs[:,i]
df.drop(columns=['bbox'], inplace=True)
 df['x1'] = df['x'] + df['w']
df['y1'] = df['y'] + df['h']
df['x_center'] = df['x'] + df['w']/2
df['y_center'] = df['y'] + df['h']/2
df['classes'] = 0
df = df[['image_id','x', 'y', 'w', 'h','x1','y1','x_center','y_center','classes']]
index = list(set(df.image_id))
source = 'train'
if True:
for fold in [0]:
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
for name,mini in tqdm(df.groupby('image_id')) :
os.makedirs('convertor', exist_ok=True)
 sh.copy(".. /input/global-wheat-detection/{}/{}.jpg".format(source,name),'convertor/{}.jpg'.format(name))
if name in val_index:
path2save = 'convertor/val.txt'
else:
path2save = 'convertor/train.txt'
with open(path2save, 'a')as f:
f.write(f'{name}.jpg')
 row = mini[['x','y','x1','y1','classes']].astype(int ).values
if row.shape[0]>max_label_numbers:
max_label_numbers = row.shape[0]
row = row.astype(str)
 for j in range(len(row)) :
text = ' '+','.join(row[j])
 f.write(text)
 f.write('
')
 return max_label_numbers<concatenate>
tokenizer = AutoTokenizer.from_pretrained('vinai/bertweet-base',
normalization=True,
use_fast = False,
add_special_tokens=True,
pad_to_max_length=True,
return_attention_mask=True)
 train_token = tokenizer(x_train.tolist() ,
padding="max_length",
truncation=True,
return_tensors = 'tf' ).data
val_token = tokenizer(x_val.tolist() ,
padding="max_length",
truncation=True,
return_tensors = 'tf' ).data
train_features = {x: train_token[x] for x in tokenizer.model_input_names}
train_tf_dataset = tf.data.Dataset.from_tensor_slices(( train_features, y_train))
train_tf_dataset = train_tf_dataset.shuffle(len(x_train)).batch(32 ).prefetch(tf.data.AUTOTUNE)
 val_features = {x: val_token[x] for x in tokenizer.model_input_names}
val_tf_dataset = tf.data.Dataset.from_tensor_slices(( val_features, y_val))
val_tf_dataset = val_tf_dataset.shuffle(len(x_val)).batch(32 ).prefetch(tf.data.AUTOTUNE )
def run_wbf(boxes, scores, image_size=1024, iou_thr=0.5, skip_box_thr=0.7, weights=None):
labels = [np.zeros(score.shape[0])for score in scores]
boxes = [box/(image_size)for box in boxes]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size)
 return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes )<normalization>
with strategy.scope() :
bert_model = TFRobertaModel.from_pretrained("vinai/bertweet-base")
 def build_model(hidden_n, drop = 0.3, lr = 1e-5, weight_decay = 1e-6):
with strategy.scope() :
input_ids = tf.keras.Input(shape=(128,),dtype='int32', name = 'input_ids')
 attention_masks = tf.keras.Input(shape=(128,),dtype='int32', name = 'attention_mask')
 output = bert_model([input_ids,attention_masks])[1]
output = tf.keras.layers.Dense(hidden_n,activation='relu' )(output)
 output = tf.keras.layers.Dropout(drop )(output)
 output = tf.keras.layers.Dense(1,activation='sigmoid' )(output)
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks], outputs = output)
 f1_score = tfa.metrics.F1Score(1, threshold = 0.5)
 model.compile(tfa.optimizers.AdamW(weight_decay = weight_decay, lr=lr),
loss='binary_crossentropy',
metrics=['accuracy', f1_score])
 model.layers[2].trainable = False
return model
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x)
 def validate() :

 use_cuda = True
truth = {}
DATA_ROOT_PATH = '.. /input/global-wheat-detection/train'
with open('convertor/val.txt', 'r', encoding='utf-8')as f:
for line in f.readlines() :
data = line.split(" ")
 truth[data[0]] = []
for i in data[1:]:
truth[data[0]].append([int(j)for j in i.split(',')])
 image_ids = [image_id.split('.')[0] for image_id in list(truth.keys())]
weights = 'checkpoints/last.pt'
Effdet = False
if not os.path.exists(weights):
Effdet = True
model = Darknet(CONFIG)
 model.load_weights(WEIGHTS)
 print('OLD WEIGHT')
 else:
model = torch.load(weights)
 if use_cuda:
model.cuda()
if Effdet:
net = get_valnet_file(PESUDO_WEIGHTS)
 model.eval()
num_classes = model.num_classes
class_names = 'wheat'
results = []
for image_id in image_ids:
enboxes = []
enscores = []
img = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg')
 img = cv2.resize(img,(model.width, model.height))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 if Yolov4:
if is_ROT:
for i in range(4):
img1 = TTAImage(img, i)
 img1 = img1.transpose(2, 0, 1)
 img1 = torch.from_numpy(img1 ).float().div(255.0 ).unsqueeze(0)
 img1 = img1.cuda()
img1 = torch.autograd.Variable(img1)
 with torch.no_grad() :
output = model(img1)
 boxes = utils.post_processing(img1, NMS_CONF_THR, NMS_CONF_THR, output)
 det = np.array(boxes)
 boxes, scores = process_det(0, det, NMS_CONF_THR)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 with torch.no_grad() :
img1 = img.transpose(2, 0, 1)
 img1 = torch.from_numpy(img1 ).float().div(255.0 ).unsqueeze(0)
 img1 = img1.cuda()
img1 = torch.autograd.Variable(img1)
 output = model(img1)
 boxes = utils.post_processing(img1, NMS_CONF_THR, NMS_CONF_THR, output)
 det = np.array(boxes)
 boxes, scores = process_det(0, det, NMS_CONF_THR)
 enboxes.append(boxes)
 enscores.append(scores)
 if Effdet:
img1 = img.transpose(2, 0, 1)
 img1 = torch.from_numpy(img1 ).cuda()
img1 = img1.float()
img1 /= 255.0
if p_HF:
with torch.no_grad() :
imgs = torch.stack(( img1,)).float().cuda()
for tta_transform in tta_transforms:
det = net(tta_transform.batch_augment(imgs.clone()), torch.tensor([1]*imgs.shape[0] ).float().cuda())
 for i in range(imgs.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > pesudo_thres)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 enboxes.append(boxes)
 enscores.append(scores)
 records = marking[marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes = gtboxes.astype(np.int32 ).clip(min=0, max=1024)
 gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
result = {
'image_id': image_id,
'pred_enboxes': enboxes,
'pred_enscores': enscores,
'gt_boxes': gtboxes,
}
results.append(result)
 return results
def calculate_final_score(all_predictions, iou_thr, skip_box_thr, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
enboxes = all_predictions[i]['pred_enboxes'].copy()
enscores = all_predictions[i]['pred_enscores'].copy()
image_id = all_predictions[i]['image_id']
pred_boxes, scores, labels = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 pred_boxes = pred_boxes.astype(np.int32 ).clip(min=0, max=1024)
 indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores)
 def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'.. /input/global-wheat-detection/train/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth" )<compute_train_metric>
grid = [{'hidden_n': 16, 'drop': 0.3, 'lr': 1e-5, 'weight_decay': 1e-6},
{'hidden_n': 32, 'drop': 0.3, 'lr': 1e-5, 'weight_decay': 1e-6},
{'hidden_n': 32, 'drop': 0.3, 'lr': 5e-6, 'weight_decay': 1e-6},
{'hidden_n': 32, 'drop': 0.25, 'lr': 1e-5, 'weight_decay': 5e-6},
{'hidden_n': 32, 'drop': 0.35, 'lr': 1e-5, 'weight_decay': 5e-7},
{'hidden_n': 64, 'drop': 0.3, 'lr': 1e-5, 'weight_decay': 1e-6},
{'hidden_n': 64, 'drop': 0.3, 'lr': 1e-5, 'weight_decay': 5e-6},
]
def run_loop(**kwargs):
model = build_model(**kwargs)
 history = model.fit(train_tf_dataset,
validation_data = val_tf_dataset,
epochs = 2)
 return {**kwargs, 'model': model, 'f1': history.history['val_f1_score'][-1][0]}
results = [run_loop(**params)for params in grid]
def log(text):
print(text)
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*10)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<normalization>
TEST_PATH = ".. /input/nlp-getting-started/test.csv"
df = pd.read_csv(TEST_PATH)
 x =(df['text'] 
.str.lower() 
.str.replace('\x89Ûª|Ûª', "'")
.str.replace('
|\x89.|\x9d *', ' ')
.str.replace('&gt;', ">")
.str.replace('&lt;', "<")
.str.replace('&amp;', " and ")
.str.replace('won't', 'will not')
.str.replace('can't', 'cannot')
.str.replace('i'm', 'i am')
.str.replace('ain't', 'is not')
.str.replace('hwy.', 'highway')
.str.replace('(\w+)'ll', '\g<1> will')
.str.replace('(\w+)n't', '\g<1> not')
.str.replace('(\w+)'ve', '\g<1> have')
.str.replace('(\w+)'s', '\g<1> is')
.str.replace('(\w+)'re', '\g<1> are')
.str.replace('(\w+)'d', '\g<1> would')
.str.replace('(\w+)'m', '\g<1> am')
.str.replace('<3', 'love')
.str.replace('w/e', 'whatever')
.str.replace('w/', 'with')
.str.replace('\b', ' ')
.str.replace('-', ' ')
.str.replace(' *', ' ')
.str.strip() 
.to_numpy() 
)
 token = tokenizer(x.tolist() ,
padding="max_length",
truncation=True,
return_tensors = 'tf' ).data
df['p'] = model.predict(token)
 df['target'] =(df['p'] > 0.5 ).astype(int )
<categorify><EOS> 
df[['id', 'target']].to_csv("submissions.csv", index = False )