Kaggle competition: Natural Language Processing with Disaster Tweets
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<set_options>
! python -m pip install tf-models-nightly --no-deps -q
! python -m pip install tf-models-official==2.4.0 -q
! python -m pip install tensorflow-gpu==2.4.1 -q
! python -m pip install tensorflow-text==2.4.1 -q
! python -m spacy download en_core_web_sm -q
! python -m spacy validate
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss<init_hyperparams>
print(f'TensorFlow Version: {tf.__version__}')
 print(f'Python Version: {python_version() }' )
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = RAdam(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/retrain.bin')
 t = time.time()
summary_loss = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-retrain.bin')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
self.optimizer.zero_grad()
loss, _, _ = self.model(images, boxes, labels)
 loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'epoch': self.epoch,
}, path)
 def load(self, path, device):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.model.to(device)
 self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<define_variables>
RANDOM_SEED = 123
nlp = spacy.load('en_core_web_sm')
 pd.set_option('display.max_colwidth', None)
 rcParams['figure.figsize'] =(10, 6)
 sns.set_theme(palette='muted', style='whitegrid' )
N_FOLD = 2
USE_TTA = True
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
TEST_ROOT_PATH = '.. /input/global-wheat-detection/test'
cfg.MODEL.PRETRAIN = False<load_pretrained>
path = '.. /input/nlp-getting-started/train.csv'
df = pd.read_csv(path)
 print(df.shape)
 df.head()
MODEL = {
"effdet": [
load_net_eval('.. /input/effdetd5sourcelee/best-retrain-epoch51.bin', 5),
load_net_eval('.. /input/effdetd7/retrains/best-retrain-epoch42.bin', 7)
 ],
"resnest":[
load_res_net(".. /input/resnest-source-weights-andreshuang/checkpoint-60arvalis1.bin", cfg),
load_res_net(".. /input/rssnest-source-weights-cjc/best-checkpoint_PL_ethz_1.bin", cfg)
 ]
}<data_type_conversions>
path_test = '.. /input/nlp-getting-started/test.csv'
df_test = pd.read_csv(path_test)
 print(df_test.shape)
 df_test.head()
def to_tensor(images):
tmp = []
for img in images:
img = img.astype(np.float32)
 img /= 255.0
img = torch.tensor(img, dtype=torch.float32)
 tmp.append(img.permute(2,0,1))
return torch.stack(tmp)
 def get_handout_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 class HandoutDatasetRetriever(Dataset):
def __init__(self, marking, image_ids, path):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.path = path
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{self.path}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 records = self.marking[self.marking['image_id'] == image_id]
gtboxes = records[['x', 'y', 'w', 'h']].values
gtboxes[:, 2] = gtboxes[:, 0] + gtboxes[:, 2]
gtboxes[:, 3] = gtboxes[:, 1] + gtboxes[:, 3]
return image, gtboxes, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<predict_on_test>
duplicates = df[df.duplicated(['text', 'target'], keep=False)]
print(f'Train Duplicate Entries(text, target): {len(duplicates)}')
 duplicates.head()
def make_predictions(
 models,
images,
score_threshold=0.25, 
):
predictions = []
for fold_number, net in enumerate(models):
with torch.no_grad() :
net.eval()
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.5, skip_box_thr=0.4, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*1023
return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def run_wbf_tta(predictions, image_size=512, iou_thr=0.5, skip_box_thr=0.4, weights=None):
boxes_1 = []
scores_1 = []
labels_1 = []
for i in range(2):
bb =(predictions['boxes'][i]/(image_size-1)).tolist()
ss = predictions['scores'][i].tolist()
ls = predictions['labels'][i].tolist()
boxes_1.append(bb)
 scores_1.append(ss)
 labels_1.append(ls)
 boxes_1, scores_1, labels_1 = weighted_boxes_fusion(boxes_1, scores_1, labels_1,
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes_2 = []
scores_2 = []
labels_2 = []
for i in range(2,4):
bb =(predictions['boxes'][i]/(image_size-1)).tolist()
ss = predictions['scores'][i].tolist()
ls = predictions['labels'][i].tolist()
boxes_2.append(bb)
 scores_2.append(ss)
 labels_2.append(ls)
 boxes_2, scores_2, labels_2 = weighted_boxes_fusion(boxes_2, scores_2, labels_2,
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = [boxes_1.tolist() , boxes_2.tolist() ]
scores = [scores_1.tolist() , scores_2.tolist() ]
labels = [labels_1.tolist() , labels_2.tolist() ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels,
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<define_variables>
df.drop_duplicates(['text', 'target'], inplace=True, ignore_index=True)
 print(df.shape, df_test.shape )
def calculate_final_score(
 all_predictions,
iou_thr,
skip_box_thr,
resweight,
edetweight,
sigma=0.5, 
):
final_scores = []
for predictions in all_predictions:
gt_boxes = predictions['gtboxes'].copy()
image_id = predictions['image_id']
img_boxes = []
img_scores = []
img_labels = []
for index in range(4):
effdet_p = predictions["effdet"][index]
boxes, scores, labels = run_wbf(effdet_p, image_size=512, image_index=0,
iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 resnest_p = predictions["resnest"][index]
boxes = [resnest_p["boxes"], boxes]
scores = [resnest_p["scores"], scores]
labels = [resnest_p["labels"], labels]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels,
weights=[resweight, edetweight], iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 for _ in range(3-index):
boxes = rotBoxes90(boxes, 1024, 1024)
 if index == 3:
boxes = boxes.astype(np.int32)
 if index == 0:
img_boxes =(boxes / 1024 ).tolist()
img_scores = scores.tolist()
img_labels = labels.tolist()
else:
img_boxes = img_boxes +(boxes / 1024 ).tolist()
img_scores = img_scores + scores.tolist()
img_labels = img_labels + labels.tolist()
boxes, scores, labels = weighted_boxes_fusion([img_boxes],
[img_scores],
[img_labels],
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes =(boxes*1024 ).astype(np.int32 ).clip(min=0, max=1023)
 image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')
 final_scores.append(image_precision)
 score = np.mean(final_scores)
 print(f"score:{score} iou_thr:{iou_thr} skip_box_thr:{skip_box_thr} weights:[{resweight, edetweight}]")
 return score<load_from_csv>
new_duplicates = df[df.duplicated(['keyword', 'text'], keep=False)]
print(f'Train Duplicate Entries(keyword, text): {len(new_duplicates)}')
 new_duplicates[['text', 'target']].sort_values(by='text' )
USE_OPTIMIZE = False
if USE_OPTIMIZE:
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True)
 marking['area'] = marking['w'] * marking['h']
marking = marking[marking['area'] < 154200.0]
error_boxes = [100648.0, 145360.0, 149744.0,119790.0, 106743.0]
marking = marking[~marking['area'].isin(error_boxes)]
all_train_images = marking.copy()
all_train_images['count'] = all_train_images.apply(lambda row: 1 if np.isfinite(row.width)else 0, axis=1)
 train_images_count = all_train_images.groupby('image_id' ).sum().reset_index()
indexes = train_images_count[train_images_count['count'] < 10].image_id
marking = marking[~marking['image_id'].isin(indexes)]
skf = StratifiedKFold(n_splits=4, shuffle=True, random_state=42)
 df_folds = marking[['image_id']]
df_folds['bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds['source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds['stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 train_index, holdout_index = next(skf.split(X=df_folds.index, y=df_folds['stratify_group']))
df_holdout = df_folds.iloc[holdout_index].copy()
df_folds = df_folds.iloc[train_index].copy()
df_folds['fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
holdout_dataset = HandoutDatasetRetriever(
 image_ids=df_holdout.index.values,
marking=marking,
path=TRAIN_ROOT_PATH, 
)
 holdout_loader = DataLoader(
 holdout_dataset,
batch_size=1,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn 
)
 all_predictions = []
for images, gtboxes, image_ids in tqdm(holdout_loader, total=len(holdout_loader)) :
image = images[0]
image = cv2.resize(image,(512, 512))
image_res = images[0]
predictions_tta = {"effdet":[],
"resnest":[],
"gtboxes":gtboxes[0],
"image_id":image_ids[0]}
for index in range(4):
roated = TTAImage(image, index)
 roated = to_tensor([roated] ).cuda()
predictions = make_predictions(MODEL["effdet"], roated)
 del roated
predictions_tta["effdet"].append(predictions)
 roated = TTAImage(image_res, index)
 roated = to_tensor([roated] ).cuda()
predictions = {}
predictions_1 = MODEL["resnest"][0](roated)
 predictions_2 = MODEL["resnest"][1](roated)
 predictions["boxes"] = predictions_1[0]["boxes"].tolist() + predictions_2[0]["boxes"].tolist()
predictions["scores"] = predictions_1[0]["scores"].tolist() + predictions_2[0]["scores"].tolist()
predictions["labels"] = predictions_1[0]["labels"].tolist() + predictions_2[0]["labels"].tolist()
predictions_tta["resnest"].append(predictions)
 del roated
all_predictions.append(predictions_tta )<compute_train_metric>
df.drop([4253, 4193, 2802, 4554, 4182, 3212, 4249, 4259, 6535, 4319, 4239, 606, 3936, 6018, 5573], inplace=True )
def log(text):
with open('opt.log', 'a+')as logger:
logger.write(f'{text}
')
 def optimize(space, all_predictions, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*5 + 'WBF' + '-'*5)
 log(params)
 final_score = calculate_final_score(all_predictions, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<find_best_params>
df = df.reset_index(drop=True)
 df
if USE_OPTIMIZE:
space = [
Real(0.1, 0.7, name='iou_thr'),
Real(0.2, 0.7, name='skip_box_thr'),
Real(1, 10, name='resweight'),
Real(1, 10, name='edetweight'),
]
opt_result = optimize(
 space,
all_predictions,
n_calls=10, 
)
 best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
else:
best_final_score = 0.68687
best_iou_thr = 0.41485
best_skip_box_thr = 0.4159
best_resweight = 3.62106
best_edetweight = 6.50667
print('-'*13 + 'WBF' + '-'*14)
 print(f'[Best Score]: {best_final_score:.4f}')
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Best Resnest weight]: {best_resweight:.3f}')
 print(f'[Best Best Effdet weight]: {best_edetweight:.3f}')
 print('-'*30 )<predict_on_test>
df['target'].value_counts() / len(df )
def make_predictions(
 models,
images,
score_threshold=0.25, 
):
predictions = []
for fold_number, net in enumerate(models):
with torch.no_grad() :
net.eval()
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*1023
return boxes, scores, labels
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def run_wbf_tta(predictions, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes_1 = []
scores_1 = []
labels_1 = []
for i in range(2):
bb =(predictions['boxes'][i]/(image_size-1)).tolist()
ss = predictions['scores'][i].tolist()
ls = predictions['labels'][i].tolist()
boxes_1.append(bb)
 scores_1.append(ss)
 labels_1.append(ls)
 boxes_1, scores_1, labels_1 = weighted_boxes_fusion(boxes_1, scores_1, labels_1,
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes_2 = []
scores_2 = []
labels_2 = []
for i in range(2,4):
bb =(predictions['boxes'][i]/(image_size-1)).tolist()
ss = predictions['scores'][i].tolist()
ls = predictions['labels'][i].tolist()
boxes_2.append(bb)
 scores_2.append(ss)
 labels_2.append(ls)
 boxes_2, scores_2, labels_2 = weighted_boxes_fusion(boxes_2, scores_2, labels_2,
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = [boxes_1.tolist() , boxes_2.tolist() ]
scores = [scores_1.tolist() , scores_2.tolist() ]
labels = [labels_1.tolist() , labels_2.tolist() ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels,
weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<load_pretrained>
def null_table(data):
null_list = []
for i in data:
if data[i].notnull().any() :
null_list.append(data[i].notnull().value_counts())
 return pd.DataFrame(pd.concat(null_list, axis=1 ).T )
test_dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{TEST_ROOT_PATH}/*.jpg')]),
path=TEST_ROOT_PATH 
)
 test_data_loader = DataLoader(
 test_dataset,
batch_size=1,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)<load_from_csv>
null_table(df )
if len(os.listdir(TEST_ROOT_PATH)) > 10:
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True)
 marking_p = marking.copy()
results = []
for images, image_ids in test_data_loader:
image = images[0]
height, width, _ = image.shape
image = cv2.resize(image,(512, 512))
image_res = cv2.resize(image,(1024, 1024))
predictions_tta = {
"boxes": [],
"scores": [],
"labels": []
}
for index in range(4):
roated = TTAImage(image, index)
 roated = to_tensor([roated] ).cuda()
predictions = make_predictions(MODEL["effdet"], roated)
 boxes, scores, labels = run_wbf(predictions, image_size=512, image_index=0)
 roated = TTAImage(image_res, index)
 roated = to_tensor([roated] ).cuda()
predictions_1 = MODEL["resnest"][0](roated)
 predictions_2 = MODEL["resnest"][1](roated)
 boxes_r = predictions_1[0]["boxes"].tolist() + predictions_2[0]["boxes"].tolist()
scores_r = predictions_1[0]["scores"].tolist() + predictions_2[0]["scores"].tolist()
labels_r = predictions_1[0]["labels"].tolist() + predictions_2[0]["labels"].tolist()
boxes = [boxes_r, boxes]
scores = [scores_r, scores]
labels = [labels_r, labels]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels,
weights=[best_resweight, best_edetweight],
iou_thr=best_iou_thr,
skip_box_thr=best_skip_box_thr)
 for _ in range(3-index):
boxes = rotBoxes90(boxes, 1024, 1024)
 if index == 3:
boxes = boxes.astype(np.int32)
 predictions_tta["boxes"].append(boxes)
 predictions_tta["scores"].append(scores)
 predictions_tta["labels"].append(labels)
 boxes, scores, labels = run_wbf_tta(predictions_tta, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes[:, [0, 1]] = boxes[:, [0, 1]] - boxes[:, [0, 1]]*0.01
boxes[:, [2, 3]] = boxes[:, [2, 3]] + boxes[:, [2, 3]]*0.01
boxes =(boxes*1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for bb, ss in zip(boxes, scores):
marking_p = marking_p.append({
'image_id':image_ids[0],
'width':1024,
'height':1024,
'source':"arvalis_1",
'x':bb[0], 'y':bb[1], 'w':bb[2], 'h':bb[3]
}, ignore_index=True)
 marking_p = marking_p[~marking_p['image_id'].isin(marking.image_id)]<define_variables>
null_table(df_test )
N_FOLD = 2
USE_OPTIMIZE = False
USE_TTA = True
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/test'
TEST_ROOT_PATH = '.. /input/global-wheat-detection/test'<init_hyperparams>
text = df['text']
target = df['target']
test_text = df_test['text']
for i in np.random.randint(500, size=5):
print(f'Tweet
' * 2 )
class TrainGlobalConfig:
num_workers = 8
batch_size = 4
n_epochs = 5
lr = 0.0002
folder = 'retrains'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
<train_model>
lookup_dict = {
'abt' : 'about',
'afaik' : 'as far as i know',
'bc' : 'because',
'bfn' : 'bye for now',
'bgd' : 'background',
'bh' : 'blockhead',
'br' : 'best regards',
'btw' : 'by the way',
'cc': 'carbon copy',
'chk' : 'check',
'dam' : 'do not annoy me',
'dd' : 'dear daughter',
'df': 'dear fiance',
'ds' : 'dear son',
'dyk' : 'did you know',
'em': 'email',
'ema' : 'email address',
'ftf' : 'face to face',
'fb' : 'facebook',
'ff' : 'follow friday',
'fotd' : 'find of the day',
'ftw': 'for the win',
'fwiw' : 'for what it is worth',
'gts' : 'guess the song',
'hagn' : 'have a good night',
'hand' : 'have a nice day',
'hotd' : 'headline of the day',
'ht' : 'heard through',
'hth' : 'hope that helps',
'ic' : 'i see',
'icymi' : 'in case you missed it',
'idk' : 'i do not know',
'ig': 'instagram',
'iirc' : 'if i remember correctly',
'imho' : 'in my humble opinion',
'imo' : 'in my opinion',
'irl' : 'in real life',
'iwsn' : 'i want sex now',
'jk' : 'just kidding',
'jsyk' : 'just so you know',
'jv' : 'joint venture',
'kk' : 'cool cool',
'kyso' : 'knock your socks off',
'lmao' : 'laugh my ass off',
'lmk' : 'let me know',
'lo' : 'little one',
'lol' : 'laugh out loud',
'mm' : 'music monday',
'mirl' : 'meet in real life',
'mrjn' : 'marijuana',
'nbd' : 'no big deal',
'nct' : 'nobody cares though',
'njoy' : 'enjoy',
'nsfw' : 'not safe for work',
'nts' : 'note to self',
'oh' : 'overheard',
'omg': 'oh my god',
'oomf' : 'one of my friends',
'orly' : 'oh really',
'plmk' : 'please let me know',
'pnp' : 'party and play',
'qotd' : 'quote of the day',
're' : 'in reply to in regards to',
'rtq' : 'read the question',
'rt' : 'retweet',
'sfw' : 'safe for work',
'smdh' : 'shaking my damn head',
'smh' : 'shaking my head',
'so' : 'significant other',
'srs' : 'serious',
'tftf' : 'thanks for the follow',
'tftt' : 'thanks for this tweet',
'tj' : 'tweetjack',
'tl' : 'timeline',
'tldr' : 'too long did not read',
'tmb' : 'tweet me back',
'tt' : 'trending topic',
'ty' : 'thank you',
'tyia' : 'thank you in advance',
'tyt' : 'take your time',
'tyvw' : 'thank you very much',
'w': 'with',
'wtv' : 'whatever',
'ygtr' : 'you got that right',
'ykwim' : 'you know what i mean',
'ykyat' : 'you know you are addicted to',
'ymmv' : 'your mileage may vary',
'yolo' : 'you only live once',
'yoyo' : 'you are on your own',
'yt': 'youtube',
'yw' : 'you are welcome',
'zomg' : 'oh my god to the maximum'
}
def re_train(path, marker, level, folder=None):
df_folds = get_df_folds(marker)
 device = torch.device('cuda:0')
 net = get_net(level)
 if folder:
TrainGlobalConfig.folder = folder
train_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == 0].index.values,
marking=marker,
transforms=get_train_transforms(512),
test=True, 
)
 validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == 0].index.values[:len(df_folds)// 5],
marking=marker,
transforms=get_valid_transforms(512),
test=True, 
)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.load(path, device)
 fitter.fit(train_loader, val_loader)
 del fitter<load_pretrained>
def lemmatize_text(text, nlp=nlp):
doc = nlp(text)
 lemma_sent = [i.lemma_ for i in doc if not i.is_stop]
return ' '.join(lemma_sent)
 def abbrev_conversion(text):
words = text.split()
abbrevs_removed = []
for i in words:
if i in lookup_dict:
i = lookup_dict[i]
abbrevs_removed.append(i)
 return ' '.join(abbrevs_removed)
 def standardize_text(text_data):
entity_pattern = re.compile(r'(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)')
 url_pattern = re.compile(r'(?:\@|http?\://|https?\://|www)\S+')
 retweet_pattern = re.compile(r'^(RT|RT:)\s+')
 digit_pattern = re.compile(r'[\d]+')
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002500-\U00002BEF"
u"\U00002702-\U000027B0"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
u"\U0001f926-\U0001f937"
u"\U00010000-\U0010ffff"
u"\u2640-\u2642"
u"\u2600-\u2B55"
u"\u200d"
u"\u23cf"
u"\u23e9"
u"\u231a"
u"\ufe0f"
u"\u3030"
"]+", flags=re.UNICODE)
 url_strip = text_data.apply(lambda x: re.sub(url_pattern, '', x)if pd.isna(x)!= True else x)
 html_parse = url_strip.apply(lambda x: bs(x, 'html.parser' ).get_text() if pd.isna(x)!= True else x)
 retweet_strip = html_parse.apply(lambda x: re.sub(retweet_pattern, '', x)if pd.isna(x)!= True else x)
 emoji_strip = retweet_strip.apply(lambda x: re.sub(emoji_pattern, '', x)if pd.isna(x)!= True else x)
 entity_strip = emoji_strip.apply(lambda x: re.sub(entity_pattern, '', x)if pd.isna(x)!= True else x)
 lowercase = entity_strip.apply(lambda x: str.lower(x)if pd.isna(x)!= True else x)
 punct_strip = lowercase.apply(lambda x: re.sub(f'[{re.escape(string.punctuation)}]', '', x)if pd.isna(x)!= True else x)
 abbrev_converted = punct_strip.apply(lambda x: abbrev_conversion(x)if pd.isna(x)!= True else x)
 digit_strip = abbrev_converted.apply(lambda x: re.sub(digit_pattern, '', x)if pd.isna(x)!= True else x)
 lemma_and_stop = digit_strip.apply(lambda x: lemmatize_text(x)if pd.isna(x)!= True else x)
 return lemma_and_stop
if len(os.listdir(TEST_ROOT_PATH)) > 10:
re_train(".. /input/effdetd5sourcelee/best-retrain-epoch51.bin", marking_p, 5, "retrains")
 MODEL["effdet"][0] = load_net_eval("retrains/best-retrain.bin", 5)
<categorify>
df['clean_text'] = pd.DataFrame(clean_text)
 df_test['clean_text'] = pd.DataFrame(test_clean_text )
results = []
for images, image_ids in test_data_loader:
image = images[0]
height, width, _ = image.shape
image = cv2.resize(image,(512, 512))
image_res = cv2.resize(image,(1024, 1024))
predictions_tta = {
"boxes": [],
"scores": [],
"labels": []
}
for index in range(4):
roated = TTAImage(image, index)
 roated = to_tensor([roated] ).cuda()
predictions = make_predictions(MODEL["effdet"], roated)
 boxes, scores, labels = run_wbf(predictions, image_size=512, image_index=0)
 roated = TTAImage(image_res, index)
 roated = to_tensor([roated] ).cuda()
predictions_1 = MODEL["resnest"][0](roated)
 predictions_2 = MODEL["resnest"][1](roated)
 boxes_r = predictions_1[0]["boxes"].tolist() + predictions_2[0]["boxes"].tolist()
scores_r = predictions_1[0]["scores"].tolist() + predictions_2[0]["scores"].tolist()
labels_r = predictions_1[0]["labels"].tolist() + predictions_2[0]["labels"].tolist()
boxes = [boxes_r, boxes]
scores = [scores_r, scores]
labels = [labels_r, labels]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels,
weights=[best_resweight, best_edetweight],
iou_thr=best_iou_thr,
skip_box_thr=best_skip_box_thr)
 for _ in range(3-index):
boxes = rotBoxes90(boxes, 1024, 1024)
 if index == 3:
boxes = boxes.astype(np.int32)
 predictions_tta["boxes"].append(boxes)
 predictions_tta["scores"].append(scores)
 predictions_tta["labels"].append(labels)
 boxes, scores, labels = run_wbf_tta(predictions_tta, image_size=1024, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes[:, [0, 1]] = boxes[:, [0, 1]] - boxes[:, [0, 1]]*0.01
boxes[:, [2, 3]] = boxes[:, [2, 3]] + boxes[:, [2, 3]]*0.01
scores = scores.clip(min=0, max=1.0)
 boxes =(boxes*1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_ids[0],
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
df['clean_text'] = df['clean_text'].apply(lambda x: re.sub(pattern_new, '', x)if pd.isna(x)!= True else x)
 df_test['clean_text'] = df_test['clean_text'].apply(lambda x: re.sub(pattern_new, '', x)if pd.isna(x)!= True else x )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df<install_modules>
print('Training Counts of 'new': ', len(re.findall(pattern_new, ' '.join(df['clean_text']))))
print('Test Counts of 'new': ', len(re.findall(pattern_new, ' '.join(df_test['clean_text']))))
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<categorify>
sentence_enc = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4' )
def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
def extract_keywords(text, nlp=nlp):
potential_keywords = []
TOP_KEYWORD = -1
pos_tag = ['ADJ', 'NOUN', 'PROPN']
doc = nlp(text)
 for i in doc:
if i.pos_ in pos_tag:
potential_keywords.append(i.text)
 document_embed = sentence_enc([text])
 potential_embed = sentence_enc(potential_keywords)
 vector_distances = cosine_similarity(document_embed, potential_embed)
 keyword = [potential_keywords[i] for i in vector_distances.argsort() [0][TOP_KEYWORD:]]
return keyword
def keyword_filler(keyword, text):
if pd.isnull(keyword):
try:
keyword = extract_keywords(text)[0]
except:
keyword = ''
return keyword
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
df['keyword_fill'] = pd.DataFrame(list(map(keyword_filler, df['keyword'], df['clean_text'])) ).astype(str)
 df_test['keyword_fill'] = pd.DataFrame(list(map(keyword_filler, df_test['keyword'], df_test['clean_text'])) ).astype(str)
 print('Null Training Keywords => ', df['keyword_fill'].isnull().any())
 print('Null Test Keywords => ', df_test['keyword_fill'].isnull().any() )
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
df['keyword_fill'] = pd.DataFrame(standardize_text(df['keyword_fill']))
df_test['keyword_fill'] = pd.DataFrame(standardize_text(df_test['keyword_fill']))
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('.. /input/wheat-effdet5-fold0-best-checkpoint/fold0-best-all-states.bin' )<categorify>
keyword_count_0 = pd.DataFrame(df['keyword_fill'][df['target']==0].value_counts().reset_index())
 keyword_count_1 = pd.DataFrame(df['keyword_fill'][df['target']==1].value_counts().reset_index() )
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<categorify>
train_features = df[['clean_text','keyword_fill']]
test_features = df_test[['clean_text', 'keyword_fill']]
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes ).clip(min=0, max=511 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores<concatenate>
train_x, val_x, train_y, val_y = train_test_split(
 train_features,
target,
test_size=0.2,
random_state=RANDOM_SEED, 
)
 print(train_x.shape)
 print(train_y.shape)
 print(val_x.shape)
 print(val_y.shape )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<categorify>
train_ds = tf.data.Dataset.from_tensor_slices(( dict(train_x), train_y))
val_ds = tf.data.Dataset.from_tensor_slices(( dict(val_x), val_y))
test_ds = tf.data.Dataset.from_tensor_slices(dict(test_features))
def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<define_variables>
AUTOTUNE = tf.data.experimental.AUTOTUNE
BUFFER_SIZE = 1000
BATCH_SIZE = 32
def configure_dataset(dataset, shuffle=False, test=False):
if shuffle:
dataset = dataset.cache() \ 
.shuffle(BUFFER_SIZE, seed=RANDOM_SEED, reshuffle_each_iteration=True)\ 
.batch(BATCH_SIZE, drop_remainder=True ).prefetch(AUTOTUNE)
 elif test:
dataset = dataset.cache() \ 
.batch(BATCH_SIZE, drop_remainder=False ).prefetch(AUTOTUNE)
 else:
dataset = dataset.cache() \ 
.batch(BATCH_SIZE, drop_remainder=True ).prefetch(AUTOTUNE)
 return dataset
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
train_ds = configure_dataset(train_ds, shuffle=True)
 val_ds = configure_dataset(val_ds)
 test_ds = configure_dataset(test_ds, test=True )
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
bert_preprocessor = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', name='BERT_preprocesser')
 bert_encoder = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/4', trainable=True, name='BERT_encoder')
 nnlm_embed = hub.KerasLayer('https://tfhub.dev/google/nnlm-en-dim50/2', name='embedding_layer' )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<import_modules>
def build_model() :
text_input = layers.Input(shape=() , dtype=tf.string, name='clean_text')
 encoder_inputs = bert_preprocessor(text_input)
 encoder_outputs = bert_encoder(encoder_inputs)
 pooled_output = encoder_outputs["pooled_output"]
bert_dropout = layers.Dropout(0.1, name='BERT_dropout' )(pooled_output)
 key_input = layers.Input(shape=() , dtype=tf.string, name='keyword_fill')
 key_embed = nnlm_embed(key_input)
 key_flat = layers.Flatten()(key_embed)
 key_dense = layers.Dense(128, activation='elu', kernel_regularizer=regularizers.l2(1e-4))(key_flat)
 key_dropout = layers.Dropout(0.5, name='dense_dropout' )(key_dense)
 merge = layers.concatenate([bert_dropout, key_dropout])
 dense = layers.Dense(128, activation='elu', kernel_regularizer=regularizers.l2(1e-4))(merge)
 dropout = layers.Dropout(0.5, name='merged_dropout' )(dense)
 clf = layers.Dense(1, activation='sigmoid', name='classifier' )(dropout)
 return Model([text_input, key_input], clf, name='BERT_classifier' )
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping
from sklearn.model_selection import KFold, StratifiedKFold
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import log_loss
import tensorflow.keras.backend as K
import tensorflow.keras.layers as L
import tensorflow.keras.models as M
from tensorflow.keras import regularizers
import tensorflow_addons as tfa
from tqdm.notebook import tqdm
import tensorflow as tf
import pandas as pd
import numpy as np
import random
import time
import collections
import itertools
from sklearn.decomposition import PCA<load_from_csv>
EPOCHS = 2
LEARNING_RATE = 5e-5
STEPS_PER_EPOCH = int(train_ds.unbatch().cardinality().numpy() / BATCH_SIZE)
 VAL_STEPS = int(val_ds.unbatch().cardinality().numpy() / BATCH_SIZE)
 TRAIN_STEPS = STEPS_PER_EPOCH * EPOCHS
WARMUP_STEPS = int(TRAIN_STEPS * 0.1)
 adamw_optimizer = create_optimizer(
 init_lr=LEARNING_RATE,
num_train_steps=TRAIN_STEPS,
num_warmup_steps=WARMUP_STEPS
)
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 train_targets = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 del test_features['sig_id'], train_features['sig_id'], train_targets['sig_id'] 
<categorify>
bert_classifier.compile(
 loss=BinaryCrossentropy(from_logits=True),
optimizer= adamw_optimizer,
metrics=[BinaryAccuracy(name='accuracy')] 
)
 history = bert_classifier.fit(
 train_ds,
epochs=EPOCHS,
steps_per_epoch=STEPS_PER_EPOCH,
validation_data= val_ds,
validation_steps=VAL_STEPS
)
categorical = ['cp_type', 'cp_dose']
for feature in categorical:
trans = LabelEncoder()
train_features[feature] = trans.fit_transform(train_features[feature])
 test_features[feature] = trans.fit_transform(test_features[feature])
 time_mapping = {24:1, 48:2, 72:3}
train_features['cp_time'] = train_features['cp_time'].map(time_mapping)
 test_features['cp_time'] = test_features['cp_time'].map(time_mapping)
 train_features<create_dataframe>
train_loss = history.history['loss']
val_loss = history.history['val_loss']
train_acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
pca = PCA(n_components=800)
 train_features_pca = pca.fit_transform(train_features)
 test_features_pca = pca.transform(test_features)
 train_features_pca = pd.DataFrame(train_features_pca)
 test_features_pca = pd.DataFrame(test_features_pca)
 train_features = train_features_pca
test_features = test_features_pca<sort_values>
val_target = np.asarray([i[1] for i in list(val_ds.unbatch().as_numpy_iterator())])
 print(val_target.shape)
 val_target[:5]
correlations = train_features.corr().abs().unstack().sort_values(kind='quicksort', ascending=False ).reset_index()
correlations = correlations[correlations['level_0'] != correlations['level_1']].reset_index()
correlations = correlations[correlations.iloc[:, 3] > 0.92]
c = collections.Counter(correlations.iloc[:, 3])
 print(correlations)
 drop_columns = correlations[::2]['level_0'].values
train_features = train_features.drop(columns=drop_columns)
 test_features = test_features.drop(columns=drop_columns)
 train = train_features.copy()
train<choose_model_class>
val_predict = bert_classifier.predict(val_ds )
def create_model(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1400, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(840, activation="sigmoid")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(588, activation="sigmoid")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(470, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.1),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=5),
loss='binary_crossentropy', metrics=["accuracy"] 
)
 return model<choose_model_class>
predictions = bert_classifier.predict(test_ds)
 print(predictions.shape)
 print(predictions[:5] )
N_STARTS = 3
tf.random.set_seed(43)
 res = train_targets.copy()
submission.loc[:, train_targets.columns] = 0
res.loc[:, train_targets.columns] = 0
for seed in range(N_STARTS):
for n,(tr, te)in enumerate(KFold(n_splits=5, random_state=seed, shuffle=True ).split(train_targets)) :
print(f'Fold {n}')
 model = create_model(len(train.columns))
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.15, patience=3, verbose=1, epsilon=1e-4, mode='min')
 early_stop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, mode= 'min')
 model.fit(train.values[tr],
train_targets.values[tr],
validation_data=(train.values[te], train_targets.values[te]),
epochs=45, batch_size=64,
callbacks=[reduce_lr_loss, early_stop], verbose=2 
)
 test_predict = model.predict(test_features.values)
 val_predict = model.predict(train.values[te])
 submission.loc[:, train_targets.columns] += test_predict
res.loc[te, train_targets.columns] += val_predict
print('')
 submission.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res.loc[:, train_targets.columns] /= N_STARTS<compute_test_metric>
predictions = np.where(predictions > THRESHOLD, 1, 0)
 df_predictions = pd.DataFrame(predictions)
 df_predictions.columns = ['target']
print(df_predictions.shape)
 df_predictions.head()
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(train_targets.loc[:, _target], res.loc[:, _target]))
print(np.mean(metrics))<save_to_csv>
submission = pd.concat([df_test['id'], df_predictions], axis=1)
 submission.to_csv('submission.csv', index=False )