Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<save_to_csv>
sys.path.insert(0, "/kaggle/input/weightedboxesfusion")
sample_submission['id'] = ids
sample_submission['label'] = 0.47*test_preds_xgb_1+0.53*test_preds_EB7_lr
sample_submission.to_csv('submission_1.csv', index=False)
 sample_submission.head()<save_to_csv>
DATA_DIR = "/kaggle/input/global-wheat-detection"
MODELS_IN_DIR = "/kaggle/input/frcnnfoldthree"
sample_submission['id'] = ids
sample_submission['label'] = 0.52*test_preds_xgb_1+0.48*test_preds_EB7_lr
sample_submission.to_csv('submission_2.csv', index=False)
 sample_submission.head()<save_to_csv>
test_df = pd.read_csv(os.path.join(DATA_DIR, "sample_submission.csv"))
test_df.shape
sample_submission['id'] = ids
sample_submission['label'] = 0.55*test_preds_xgb_1+0.45*test_preds_EB7_lr
sample_submission.to_csv('submission_3.csv', index=False)
 sample_submission.head()<save_to_csv>
class WheatDataset(Dataset):
def __init__(self, dataframe, image_dir, transforms=None):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.transforms = transforms
def __len__(self)-> int:
return len(self.image_ids)
 def __getitem__(self, idx: int):
image_id = self.image_ids[idx]
image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.df[self.df['image_id'] == image_id]
if self.transforms:
sample = {"image": image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
sample_submission['id'] = ids
sample_submission['label'] = 0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr
sample_submission.to_csv('submission_EB4_0.csv', index=False)
 sample_submission.head()<save_to_csv>
def get_model() :

 backbone = resnet_fpn_backbone('resnet101', pretrained=False)
 model = FasterRCNN(backbone, num_classes=2)
 return model
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.01*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.01*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_0.csv', index=False)
 sample_submission.head()<save_to_csv>
DEVICE = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 model = get_model()
model.load_state_dict(torch.load(os.path.join(MODELS_IN_DIR, "best_model.pth")))
 model.eval()
model.to(DEVICE)
 1 == 1
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.05*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.05*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_1.csv', index=False)
 sample_submission.head()<save_to_csv>
def get_test_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.08*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.08*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_2.csv', index=False)
 sample_submission.head()<save_to_csv>
def collate_fn(batch):
return tuple(zip(*batch))
test_dataset = WheatDataset(test_df, os.path.join(DATA_DIR, "test"), get_test_transforms())
 test_data_loader = DataLoader(
 test_dataset,
batch_size=4,
shuffle=False,
num_workers=1,
drop_last=False,
collate_fn=collate_fn
)
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.08*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.08*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_2.csv', index=False)
 sample_submission.head()<save_to_csv>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
sample_submission['id'] = ids
subs = np.clip(1.1*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.1*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission['label'] = subs
sample_submission.to_csv('submission_EB7_EB4_3.csv', index=False)
 sample_submission.head()<save_to_csv>
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]]
res_boxes[:, [1,3]] = boxes[:, [0,2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
sample_submission['id'] = ids
subs = np.clip(1.1*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.1*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 subs[subs > 0.9976] = 1
sample_submission['label'] = subs
sample_submission.to_csv('submission_EB7_EB4_3_b.csv', index=False)
 sample_submission.head()<save_to_csv>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
sample_submission['id'] = ids
subs = np.clip(1.1*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.1*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 subs[subs > 0.9973] = 1
sample_submission['label'] = subs
sample_submission.to_csv('submission_EB7_EB4_3_c.csv', index=False)
 sample_submission.head()<save_to_csv>
def make_tta_predictions(images, score_threshold=0.5):
with torch.no_grad() :
images = torch.stack(images ).float().to(DEVICE)
 predictions = []
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
sample_submission['id'] = ids
subs = np.clip(1.1*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.1*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 subs[subs > 0.9973] = 1
subs[subs < 0.0018] = 0
sample_submission['label'] = subs
sample_submission.to_csv('submission_EB7_EB4_3_d.csv', index=False)
 sample_submission.head()<save_to_csv>
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.5, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
sample_submission['id'] = ids
sample_submission['label'] = np.clip(1.12*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.12*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission.to_csv('submission_EB7_EB4_4.csv', index=False)
 sample_submission.head()<save_to_csv>
results = []
for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
sample_submission['id'] = ids
subs = np.clip(1.12*(0.5*test_preds_xgb_1+0.5*test_preds_EB7_lr)-0.12*(0.5*test_preds_xgb_2+0.5*test_preds_EB4_lr), 0, 1)
 sample_submission['label'] = subs
subs[subs > 0.9974] = 1
subs[subs < 0.0018] = 0
sample_submission.to_csv('submission_EB7_EB4_4_b.csv', index=False)
 sample_submission.head()<import_modules>
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.head()
<load_from_csv><EOS> 
test_df.to_csv('submission.csv', index=False )