Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<train_model>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
history = model.fit_generator(
 generator = batch_data_gen(b_train_csv, TRAIN_DIR, BATCH_SIZE, augmentation=None),
validation_data = batch_data_gen(b_valid_csv, TRAIN_DIR, BATCH_SIZE),
validation_steps = 50,
steps_per_epoch = 500,
epochs = 20,
verbose = 2,
callbacks=[ checkpoint, reduce_lr ]
)<load_pretrained>
def get_valid_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
model.load_weights('best_model.h5' )<define_variables>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
lrs = LearningRateScheduler(lambda epoch: 0.001 * 0.3 **(epoch // 2))
history = model.fit_generator(
 generator = batch_data_gen(b_train_csv, TRAIN_DIR, BATCH_SIZE, augmentation=augmentor),
validation_data = batch_data_gen(b_valid_csv, TRAIN_DIR, BATCH_SIZE),
validation_steps = 20,
steps_per_epoch = 150,
epochs = 10,
verbose = 2,
callbacks=[ checkpoint, lrs ],
)<load_pretrained>
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)
model.load_weights('best_model.h5' )<predict_on_test>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('.. /input/wheateffdet5bestcheckpoint/best-checkpoint-038epoch.bin' )
fig, axs = plt.subplots(ncols=3, nrows=5, figsize=(10, 15), sharex=True, sharey=True)
 for i, img_id in enumerate(np.random.choice(train_csv.dropna() ['ImageId'].unique() , 5)) :
print(img_id)
 img_df = train_csv.set_index('ImageId' ).loc[[img_id]]
x, y = load_paired_data(img_df, TRAIN_DIR)
 yp = model.predict(np.expand_dims(preprocess_input(x), axis=0))
axs[i,0].set_ylabel(img_id)
 axs[i,0].imshow(x)
 axs[i,1].imshow(y[:,:,0])
 axs[i,2].imshow(yp[0,:,:,0])
 axs[0,0].set_title('Input')
 axs[0,1].set_title('Mask')
 axs[0,2].set_title('Predict')
 plt.xticks([])
 plt.yticks([])
 plt.show()<load_from_csv>
def make_predictions(images, score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 return [predictions]
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
sub_csv = pd.DataFrame(columns=['ImageId', 'EncodedPixels'])
 for test_id in tqdm(os.listdir(TEST_DIR)) :
fp = os.path.join(TEST_DIR, test_id)
 img = skiImgRead(fp)
 assert(img.shape ==(IMG_HW, IMG_HW, 3)) , 'Bad Shape in image: "{}"'.format(fp)
 img = resize(img,(ZOOM_HW, ZOOM_HW))
imgTTA1 = preprocess_input(img ).reshape(1, ZOOM_HW, ZOOM_HW, 3)
 imgTTA1 = imgTTA1[:, :: 1, :: 1, :]
imgTTA2 = imgTTA1[:, :: 1, ::-1, :]
imgTTA3 = imgTTA1[:, ::-1, :: 1, :]
imgTTA4 = imgTTA1[:, ::-1, ::-1, :] 
(rTTA1,rTTA2,rTTA3,rTTA4)= model.predict(np.concatenate([imgTTA1, imgTTA2, imgTTA3, imgTTA4])) [:,:,:,0]
result =(
 rTTA1[:: 1, :: 1] +
rTTA2[:: 1, ::-1] +
rTTA3[::-1, :: 1] +
rTTA4[::-1, ::-1] 
)/4
result = resize(result,(IMG_HW, IMG_HW))
labels = label(( result>0.5)+0)
 if labels.max() == 0:
sub_csv = sub_csv.append({'ImageId':test_id}, ignore_index=True)
 else:
for k in np.unique(labels[labels>0]):
sub_csv = sub_csv.append(
 {
'ImageId' : test_id,
'EncodedPixels': rle_encode(labels==k)
 }, ignore_index=True)
<save_to_csv>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
sub_csv.to_csv('submission.csv', index=False )<install_modules>
results = []
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
<import_modules><EOS> 
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()