Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<train_model>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
reg = StackingRegressor(
 estimators=estimators,
final_estimator=lightgbm.LGBMRegressor() ,
n_jobs=-1,
cv=5 
)
 reg.fit(train_feature, train['target'])
<save_to_csv>
img_sz = 640
device = 'cuda' if torch.cuda.is_available() else 'cpu'
device
test_pred = reg.predict(test.drop("id",axis = 1))
submission = pd.DataFrame({
"id": test["id"],
"target":test_pred
})
 submission.to_csv('stacking_sub.csv', index=False )<install_modules>
def get_valid_transforms() :
return A.Compose([A.Resize(height=img_sz, width=img_sz, p=1.0),
ToTensorV2(p=1.0)], p=1.0 )
!pip install python-vivid<import_modules>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
import seaborn as sns
import matplotlib.pyplot as plt
from vivid.estimators.base import MetaBlock
from vivid.estimators.boosting.mixins import BoostingEarlyStoppingMixin
from catboost import CatBoostRegressor<import_modules>
dataset = DatasetRetriever(image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms())
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(dataset, batch_size=2, shuffle=False, num_workers=4,
drop_last=False, collate_fn=collate_fn )
from vivid.features.base import BinningCountBlock
from vivid.features.base import CountEncodingBlock
from vivid.features.base import FilterBlock
from vivid.estimators.boosting import XGBRegressorBlock
from vivid.estimators.boosting import LGBMRegressorBlock
from vivid.estimators.boosting.block import create_boosting_seed_blocks
from vivid.estimators.linear import TunedRidgeBlock
from vivid.estimators.svm import SVRBlock
from vivid.estimators.ensumble import RFRegressorBlock
from vivid.estimators.base import EnsembleBlock, BaseBlock<load_from_csv>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=img_sz
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path, map_location=torch.device(device))
if 'model_state_dict' in checkpoint.keys() :
net.load_state_dict(checkpoint['model_state_dict'])
 else:
net.load_state_dict(checkpoint)
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
nets = [load_net('.. /input/global-wheat-models/best-fold-4-640sz.bin'),
load_net('.. /input/global-wheat-models/best-fold-1-640sz.bin'),
load_net('.. /input/global-wheat-models/best-fold-2-640sz.bin'),
load_net('.. /input/global-wheat-models/best-fold-3-640sz.bin')]
train_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 test_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 sample_submission_df = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv')
 feature_columns = [
'cont1',
'cont2',
'cont3',
'cont4',
'cont5',
'cont6',
'cont7',
'cont8',
'cont9',
'cont10',
'cont11',
'cont12',
'cont13',
'cont14',
]
y = train_df['target'].values<install_modules>
class BaseWheatTTA:
image_size = img_sz
def augment(self, image): raise NotImplementedError
def batch_augment(self, images): raise NotImplementedError
def deaugment_boxes(self, boxes): raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):
def augment(self, image): return image.flip(1)
 def batch_augment(self, images): return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):
def augment(self, image): return image.flip(2)
 def batch_augment(self, images): return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):
def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):
def __init__(self, transforms): self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
!pip install ptitprince<install_modules>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
!pip freeze | grep seaborn<install_modules>
def make_predictions(net, images, score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
det = net(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 return [predictions]
def make_tta_predictions(net, images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).cuda().float()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=img_sz, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
!pip install -U seaborn<import_modules>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
from ptitprince import RainCloud<define_search_space>
results = []
for images, image_ids in data_loader:
predictions = []
for net in nets:
predictions += make_tta_predictions(net, images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*(1024/img_sz)).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
<import_modules><EOS> 
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()