Kaggle competition: Global Wheat Detection
params = {'num_leaves': 30,
'min_data_in_leaf': 20,
'objective': 'regression',
'max_depth': 5,
'learning_rate': 0.01,
"boosting": "gbdt",
"feature_fraction": 0.9,
"bagging_freq": 1,
"bagging_fraction": 0.9,
"bagging_seed": 11,
"metric": 'rmse',
"lambda_l1": 0.2,
"verbosity": -1}
lgb_model = lgb.LGBMRegressor(**params, n_estimators = 20000, nthread = 4, n_jobs = -1)
 lgb_model.fit(X_train, y_train,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )<predict_on_test>
!pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext"./
lgb_model.fit(X, y,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200)
 y_pred=lgb_model.predict(X_test )<save_to_csv>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
y_pred=np.expm1(y_pred)
 pd.DataFrame({'id': test.id, 'revenue': y_pred} ).to_csv('submission_LGB.csv', index=False )<choose_model_class>
trained_weight_path = '.. /input/efficidentdet7fullsize/best-checkpoint-054epoch.bin'
xgb_params = {'eta': 0.01,
'objective': 'reg:linear',
'max_depth': 7,
'subsample': 0.8,
'colsample_bytree': 0.8,
'eval_metric': 'rmse',
'seed': 11,
'silent': True}
xgb_model = xgb.XGBRegressor(**xgb_params, n_estimators = 20000,
nthread = 4, n_jobs = -1 )<train_model>
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
xgb_model.fit(X, y,
eval_set=[(X_train, y_train),(X_valid, y_valid)], eval_metric='rmse',
verbose=1000, early_stopping_rounds=200 )<predict_on_test>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
y_pred = xgb_model.predict(X_test )<save_to_csv>
def make_tta_predictions(images, nets, score_threshold=0.25):
all_predictions = []
images = torch.stack(images ).float().cuda()
for fold_num, net in enumerate(nets):
with torch.no_grad() :
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 all_predictions.append(predictions)
 return all_predictions
def run_wbf(all_predictions, image_index, image_size=1024, iou_thr=0.44, skip_box_thr=0.43, weights=None):
for i in range(len(nets)) :
predictions = all_predictions[i]
if i == 0:
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
else:
boxes += [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores += [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels += [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
y_pred=np.expm1(y_pred)
 pd.DataFrame({'id': test.id, 'revenue': y_pred} ).to_csv('submission_XGB.csv', index=False )<train_model>
def create_model_efficientdet_test(compound_coef:int, weight_path=None,
num_classes=1, image_size=1024, pretrained=False, model_ema=False):
config = get_efficientdet_config(f'tf_efficientdet_d{compound_coef}')
 model = EfficientDet(config, pretrained_backbone=pretrained)
 config.num_classes = num_classes
config.image_size = image_size
model.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(weight_path)
 if model_ema:
model.load_state_dict(checkpoint['state_dict_ema'])
 else:
model.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
model = DetBenchEval(model, config)
 anchors.MAX_DETECTIONS_PER_IMAGE=250
model.eval() ;
return model.cuda()
nets = [create_model_efficientdet_test(compound_coef=4,
weight_path='.. /input/efficidentdet4-fullsize/best-checkpoint-058epoch.bin',
pretrained=False,
model_ema=True),
create_model_efficientdet_test(compound_coef=5,
weight_path='.. /input/efficientdet5-fullsize/best-checkpoint-052epoch.bin',
pretrained=False),
create_model_efficientdet_test(compound_coef=7,
weight_path=trained_weight_path,
pretrained=False,
model_ema=True)]
model_gboost = GradientBoostingRegressor()
model_gboost.fit(X_train, y_train)
 y_pred = model_gboost.predict(X_test )<save_to_csv>
def get_test_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
y_pred=np.expm1(y_pred)
 pd.DataFrame({'id': test.id, 'revenue': y_pred} ).to_csv('submission_GradientBoosting.csv', index=False )<set_options>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetrieverTest(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
print(os.listdir(".. /input"))
warnings.filterwarnings('ignore')
 %matplotlib inline
random_seed = 2018<load_from_csv>
test_dataset = DatasetRetrieverTest(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
test_data_loader = DataLoader(
 test_dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)
train = pd.read_csv('.. /input/tmdb-box-office-prediction/train.csv')
 test = pd.read_csv('.. /input/tmdb-box-office-prediction/test.csv')
 sub = pd.read_csv('.. /input/tmdb-box-office-prediction/sample_submission.csv')
 print(train.columns)
 print(test.columns)
<filter>
def predict_test(data_loader,nets):
results = []
testdf_psuedo = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images=images, nets=nets)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': image_id,
'source': 'BULLBULL',
'width':1024,
'height':1024,
'x': box[0],
'y': box[1],
'w': box[2],
'h': box[3]
}
testdf_psuedo.append(result)
 test_df_pseudo = pd.DataFrame(testdf_psuedo, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h'])
 return test_df_pseudo
print("Number of NA Train for the variable budget :
" + str(train[train.budget == 0].shape[0])+ "
")
 print("Number of NA Test for the variable budget :
" + str(test[test.budget == 0].shape[0]))<feature_engineering>
test_df_pseudo = predict_test(test_data_loader, nets )
train.loc[train['id'] == 16,'revenue'] = 192864
train.loc[train['id'] == 90,'budget'] = 30000000
train.loc[train['id'] == 118,'budget'] = 60000000
train.loc[train['id'] == 149,'budget'] = 18000000
train.loc[train['id'] == 313,'revenue'] = 12000000
train.loc[train['id'] == 451,'revenue'] = 12000000
train.loc[train['id'] == 464,'budget'] = 20000000
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 930000
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 819,'budget'] = 90000000
train.loc[train['id'] == 850,'budget'] = 90000000
train.loc[train['id'] == 1007,'budget'] = 2
train.loc[train['id'] == 1112,'budget'] = 7500000
train.loc[train['id'] == 1131,'budget'] = 4300000
train.loc[train['id'] == 1359,'budget'] = 10000000
train.loc[train['id'] == 1542,'budget'] = 1
train.loc[train['id'] == 1570,'budget'] = 15800000
train.loc[train['id'] == 1571,'budget'] = 4000000
train.loc[train['id'] == 1714,'budget'] = 46000000
train.loc[train['id'] == 1721,'budget'] = 17500000
train.loc[train['id'] == 1865,'revenue'] = 25000000
train.loc[train['id'] == 1885,'budget'] = 12
train.loc[train['id'] == 2091,'budget'] = 10
train.loc[train['id'] == 2268,'budget'] = 17500000
train.loc[train['id'] == 2491,'budget'] = 6
train.loc[train['id'] == 2602,'budget'] = 31000000
train.loc[train['id'] == 2612,'budget'] = 15000000
train.loc[train['id'] == 2696,'budget'] = 10000000
train.loc[train['id'] == 2801,'budget'] = 10000000
train.loc[train['id'] == 335,'budget'] = 2
train.loc[train['id'] == 348,'budget'] = 12
train.loc[train['id'] == 470,'budget'] = 13000000
train.loc[train['id'] == 513,'budget'] = 1100000
train.loc[train['id'] == 640,'budget'] = 6
train.loc[train['id'] == 696,'budget'] = 1
train.loc[train['id'] == 797,'budget'] = 8000000
train.loc[train['id'] == 850,'budget'] = 1500000
train.loc[train['id'] == 1199,'budget'] = 5
train.loc[train['id'] == 1282,'budget'] = 9
train.loc[train['id'] == 1347,'budget'] = 1
train.loc[train['id'] == 1755,'budget'] = 2
train.loc[train['id'] == 1801,'budget'] = 5
train.loc[train['id'] == 1918,'budget'] = 592
train.loc[train['id'] == 2033,'budget'] = 4
train.loc[train['id'] == 2118,'budget'] = 344
train.loc[train['id'] == 2252,'budget'] = 130
train.loc[train['id'] == 2256,'budget'] = 1
train.loc[train['id'] == 2696,'budget'] = 10000000<feature_engineering>
def generate_train(test_df_pseudo, fold_=1):
print("Loading training set...")
 marking = pd.read_csv('.. /input/wheat-cleaned-dataset/clean_train_v2.csv')
 marking = marking[marking['area']> 300]
df_with_pseudo = pd.concat([marking, test_df_pseudo])
 index = list(set(marking.image_id))
fold = 0
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
train_index = index[:len(index)*fold//5] + index[len(index)*(fold+1)//5:]
train_df_pseudo_fold = test_df_pseudo['image_id'].unique()
train_fold = np.concatenate(( train_index, train_df_pseudo_fold))
print('train dataset size:', len(train_fold))
val_fold = np.array(val_index)
 print('val dataset size:', len(val_fold))
return df_with_pseudo, train_fold, val_fold
test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4829,'budget'] = 20
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6759,'budget'] = 50
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 6858,'budget'] = 100
test.loc[test['id'] == 6876,'budget'] = 250
test.loc[test['id'] == 6972,'budget'] = 1
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 7231,'budget'] = 85
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test.loc[test['id'] == 5926,'budget'] = 300
test.loc[test['id'] == 5927,'budget'] = 4
test.loc[test['id'] == 5986,'budget'] = 1
test.loc[test['id'] == 6053,'budget'] = 20
test.loc[test['id'] == 6104,'budget'] = 1
test.loc[test['id'] == 6130,'budget'] = 30
test.loc[test['id'] == 6301,'budget'] = 150
test.loc[test['id'] == 6276,'budget'] = 100
test.loc[test['id'] == 6473,'budget'] = 100
test.loc[test['id'] == 6842,'budget'] = 30<merge>
df_with_pesudo, train_fold, val_fold = generate_train(test_df_pseudo )
release_dates = pd.read_csv('.. /input/release-dates/release_dates_per_country.csv')
 release_dates['id'] = range(1,7399)
 release_dates.drop(['original_title','title'],axis = 1,inplace = True)
 release_dates.index = release_dates['id']
train = pd.merge(train, release_dates, how='left', on=['id'])
 test = pd.merge(test, release_dates, how='left', on=['id'] )<load_from_csv>
def get_train_transforms() :
return A.Compose(
 [
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.RandomRotate90(p=0.5),
A.Resize(height=1024, width=1024, p=1),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
 )
trainAdditionalFeatures = pd.read_csv('.. /input/additionnal-features/TrainAdditionalFeatures.csv')[['imdb_id','popularity2','rating','totalVotes']]
testAdditionalFeatures = pd.read_csv('.. /input/additionnal-features/TestAdditionalFeatures.csv')[['imdb_id','popularity2','rating','totalVotes']]
train = pd.merge(train, trainAdditionalFeatures, how='left', on=['imdb_id'])
 test = pd.merge(test, testAdditionalFeatures, how='left', on=['imdb_id'] )<sort_values>
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False, mixup=True):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
self.mixup = mixup
self.mosaic_border = [-1024//2, -1024//2]
def __getitem__(self, index: int):
image_id = self.image_ids[index]
origin_image = False
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 origin_image = True
else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 if(random.random() < 0.9)and(self.test == False):
image, boxes = cutout(image, boxes)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['labels'] = torch.ones(( len(sample['bboxes']),), dtype=torch.int64)
 target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
if os.path.exists(f'{TRAIN_ROOT_PATH}/{image_id}.jpg'):
image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 else:
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image = cv2.resize(image,(1024, 1024))
image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize
xc, yc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize*2, imsize*2, 3), 114/255, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:, 2] - result_boxes[:, 0])*(result_boxes[:, 3] - result_boxes[:, 1])> 300.0)]
result_boxes = result_boxes[np.where(( result_boxes[:, 2] - result_boxes[:, 0])>= 10.0)]
result_boxes = result_boxes[np.where(( result_boxes[:, 3] - result_boxes[:, 1])>= 10.0)]
result_image, result_boxes = random_affine(result_image, targets=result_boxes)
 return result_image, result_boxes
corr_mat = train.corr()
corr_mat.revenue.sort_values(ascending=False )<sort_values>
train_dataset = DatasetRetriever(
 image_ids=train_fold,
marking=df_with_pesudo,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=val_fold,
marking=df_with_pesudo,
transforms=get_valid_transforms() ,
test=True,
)
Train = train.copy()
Train.sort_values('revenue',ascending=False,inplace=True)
 Train =Train.head(20)
 Train[['title','popularity','budget','genres','revenue','release_date','production_companies']]<data_type_conversions>
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def prepare(df):
global json_cols
global train_dict
df[['release_month','release_day','release_year']]=df['release_date'].str.split('/',expand=True ).replace(np.nan, 0 ).astype(int)
 df['release_year'] = df['release_year']
df.loc[(df['release_year'] <= 18)&(df['release_year'] < 100), "release_year"] += 2000
df.loc[(df['release_year'] > 18)&(df['release_year'] < 100), "release_year"] += 1900
rating_na = df.groupby(["release_year","original_language"])['rating'].mean().reset_index()
df[df.rating.isna() ]['rating'] = df.merge(rating_na, how = 'left' ,on = ["release_year","original_language"])
 vote_count_na = df.groupby(["release_year","original_language"])['totalVotes'].mean().reset_index()
df[df.totalVotes.isna() ]['totalVotes'] = df.merge(vote_count_na, how = 'left' ,on = ["release_year","original_language"])
 df['budget'] = np.log1p(df['budget'])
 df['genders_0_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
df['_collection_name'] = df['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 le = LabelEncoder()
le.fit(list(df['_collection_name'].fillna('')))
 df['_collection_name'] = le.transform(df['_collection_name'].fillna('' ).astype(str))
df['_num_Keywords'] = df['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 df['_num_cast'] = df['cast'].apply(lambda x: len(x)if x != {} else 0)
 releaseDate = pd.to_datetime(df['release_date'])
 df['release_dayofweek'] = releaseDate.dt.dayofweek
df['release_quarter'] = releaseDate.dt.quarter
df['_budget_runtime_ratio'] = df['budget']/df['runtime']
df['_budget_popularity_ratio'] = df['budget']/df['popularity']
df['_budget_year_ratio'] = df['budget']/(df['release_year']*df['release_year'])
 df['_releaseYear_popularity_ratio'] = df['release_year']/df['popularity']
df['_releaseYear_popularity_ratio2'] = df['popularity']/df['release_year']
df['meanruntimeByYear'] = df.groupby("release_year")["runtime"].aggregate('mean')
 df['meanPopularityByYear'] = df.groupby("release_year")["popularity"].aggregate('mean')
 df['meanBudgetByYear'] = df.groupby("release_year")["budget"].aggregate('mean')
 df['_popularity_theatrical_ratio'] = df['theatrical']/df['popularity']
df['_budget_theatrical_ratio'] = df['budget']/df['theatrical']
df['_popularity_totalVotes_ratio'] = df['totalVotes']/df['popularity']
df['_totalVotes_releaseYear_ratio'] = df['totalVotes']/df['release_year']
df['_budget_totalVotes_ratio'] = df['budget']/df['totalVotes']
df['_rating_popularity_ratio'] = df['rating']/df['popularity']
df['_rating_totalVotes_ratio'] = df['totalVotes']/df['rating']
df['_budget_rating_ratio'] = df['budget']/df['rating']
df['_runtime_rating_ratio'] = df['runtime']/df['rating']
df['has_homepage'] = 0
df.loc[pd.isnull(df['homepage']),"has_homepage"] = 1
df['isbelongs_to_collectionNA'] = 0
df.loc[pd.isnull(df['belongs_to_collection']),"isbelongs_to_collectionNA"] = 1
df['isTaglineNA'] = 0
df.loc[df['tagline'] == 0 ,"isTaglineNA"] = 1
df['isOriginalLanguageEng'] = 0
df.loc[ df['original_language'] == "en" ,"isOriginalLanguageEng"] = 1
df['isTitleDifferent'] = 1
df.loc[ df['original_title'] == df['title'] ,"isTitleDifferent"] = 0
df['isMovieReleased'] = 1
df.loc[ df['status'] != "Released" ,"isMovieReleased"] = 0
df['collection_id'] = df['belongs_to_collection'].apply(lambda x : np.nan if len(x)==0 else x[0]['id'])
 df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['overview_word_count'] = df['overview'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))
df['cast_count'] = df['cast'].apply(lambda x : len(x))
df['crew_count'] = df['crew'].apply(lambda x : len(x))
for col in ['genres', 'production_countries', 'spoken_languages', 'production_companies'] :
df[col] = df[col].map(lambda x: sorted(list(set([n if n in train_dict[col] else col+'_etc' for n in [d['name'] for d in x]])))).map(lambda x: ','.join(map(str, x)))
 temp = df[col].str.get_dummies(sep=',')
 df = pd.concat([df, temp], axis=1, sort=False)
 df.drop(['genres_etc'], axis = 1, inplace = True)
 df = df.drop(['belongs_to_collection','genres','homepage','imdb_id','overview','runtime'
,'poster_path','production_companies','production_countries','release_date','spoken_languages'
,'status','title','Keywords','cast','crew','original_language','original_title','tagline', 'collection_id','movie_id'
],axis=1)
 df.fillna(value=0.0, inplace = True)
 return df<feature_engineering>
class TrainGlobalConfig:
num_workers = 2
batch_size = 2
use_amp = True
use_ema = True
n_epochs = 8
total_batch_size = 8
clip_grad = True
accumulate_steps = int(total_batch_size / batch_size)
 lr = 0.0085
folder = 'effdet7'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = CosineLRScheduler
scheduler_params = dict(
 t_initial=n_epochs,
t_mul=1.0,
lr_min = 1e-8, 
)
test['revenue'] = np.nan
json_cols = ['genres', 'production_companies', 'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def get_dictionary(s):
try:
d = eval(s)
 except:
d = {}
return d
for col in tqdm(json_cols + ['belongs_to_collection']):
train[col] = train[col].apply(lambda x : get_dictionary(x))
test[col] = test[col].apply(lambda x : get_dictionary(x))
print(train.shape)
 train.head()<feature_engineering>
def is_parallel(model):
return type(model)in(nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)
 class ModelEMA:

 def __init__(self, model, decay=0.9999, updates=0):
self.ema = deepcopy(model.module if is_parallel(model)else model ).eval()
self.updates = updates
self.decay = lambda x: decay *(1 - math.exp(-x / 2000))
for p in self.ema.parameters() :
p.requires_grad_(False)
 def update(self, model):
with torch.no_grad() :
self.updates += 1
d = self.decay(self.updates)
 msd = model.module.state_dict() if is_parallel(model)else model.state_dict()
for k, v in self.ema.state_dict().items() :
if v.dtype.is_floating_point:
v *= d
v +=(1.- d)* msd[k].detach()
def update_attr(self, model, include=() , exclude=('process_group', 'reducer')) :
copy_attr(self.ema, model, include, exclude )
def get_json_dict(df):
global json_cols
result = dict()
for e_col in json_cols :
d = dict()
rows = df[e_col].values
for row in rows :
if row is None : continue
for i in row :
if i['name'] not in d :
d[i['name']] = 0
d[i['name']] += 1
result[e_col] = d
return result
train_dict = get_json_dict(train)
 test_dict = get_json_dict(test)
 for col in json_cols :
remove = []
train_id = set(list(train_dict[col].keys()))
test_id = set(list(test_dict[col].keys()))
remove += list(train_id - test_id)+ list(test_id - train_id)
 for i in train_id.union(test_id)- set(remove):
if train_dict[col][i] < 10 or i == '' :
remove += [i]
for i in remove :
if i in train_dict[col] :
del train_dict[col][i]
if i in test_dict[col] :
del test_dict[col][i]
print(col, 'size :', len(train_id.union(test_id)) , '->', len(train_dict[col]))<concatenate>
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
self.use_amp = config.use_amp
self.accumulate_steps = config.accumulate_steps
self.clip_grad = config.clip_grad
self.max_norm = 10.0
self.use_ema = config.use_ema
self.model_ema = None
self.model_ema_decay = 0.999
pg0, pg1, pg2 = [], [], []
for k, v in self.model.named_parameters() :
if v.requires_grad:
if '.bias' in k:
pg2.append(v)
 elif '.weight' in k and '.bn' not in k:
pg1.append(v)
 else:
pg0.append(v)
 self.optimizer = torch.optim.SGD(pg0, lr=config.lr, momentum=0.9, nesterov=True)
 self.optimizer.add_param_group({'params': pg1, 'weight_decay': 5e-4})
 self.optimizer.add_param_group({'params': pg2})
 self.model.to(self.device)
 if self.use_amp:
self.model, self.optimizer = amp.initialize(self.model, self.optimizer, opt_level="O1",verbosity=0)
 if self.use_ema:
self.model_ema = ModelEMA(self.model, decay=self.model_ema_decay)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
best_epoch = 0
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
if self.use_ema:
summary_loss = self.validation(self.model_ema.ema.module if hasattr(self.model_ema.ema, 'module')else self.model_ema.ema,
validation_loader)
 else:
summary_loss = self.validation(self.model, validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 best_epoch = e
if self.config.validation_scheduler:
self.scheduler.step(e+1)
 self.epoch += 1
return best_epoch
def validation(self, model, val_loader):
model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images ).to(self.device ).float()
batch_size = images.shape[0]
target_input = {'bbox':[target['boxes'].to(self.device ).float() for target in targets],
'cls': [target['labels'].to(self.device ).float() for target in targets]}
loss = model(images, target_input['bbox'], target_input['cls'])
 loss, class_loss, box_loss = loss
summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
self.optimizer.zero_grad()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images ).to(self.device ).float()
batch_size = images.shape[0]
target_input = {'bbox':[target['boxes'].to(self.device ).float() for target in targets],
'cls': [target['labels'].to(self.device ).float() for target in targets]}
loss = self.model(images, target_input['bbox'], target_input['cls'])
 loss, class_loss, box_loss = loss
if self.use_amp:
with amp.scale_loss(loss, self.optimizer)as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
if(step+1)% self.accumulate_steps == 0:
if self.clip_grad:
if self.use_amp:
torch.nn.utils.clip_grad_norm_(amp.master_params(self.optimizer), self.max_norm)
 else:
torch.nn.utils.clip_grad_norm_(self.model.parameters() , self.max_norm)
 self.optimizer.step()
self.optimizer.zero_grad()
if self.model_ema is not None:
self.model_ema.update(self.model)
 summary_loss.update(loss.detach().item() , batch_size)
 if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
if self.use_ema:
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'state_dict_ema': self.model_ema.ema.module.state_dict() if hasattr(self.model_ema.ema, 'module')else self.model_ema.ema.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 else:
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )
all_data = prepare(pd.concat([train, test] ).reset_index(drop = True))
train = all_data.loc[:train.shape[0] - 1,:]
test = all_data.loc[train.shape[0]:,:]
print(train.shape)
 train.head()<define_variables>
def collate_fn(batch):
return tuple(zip(*batch))
def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 best_epoch = fitter.fit(train_loader, val_loader)
 return best_epoch
features = list(train.columns)
 features = [i for i in features if i != 'id' and i != 'revenue']<compute_test_metric>
def get_net() :
config = get_efficientdet_config('tf_efficientdet_d7')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(trained_weight_path)
 net.load_state_dict(checkpoint['state_dict_ema'])
 return DetBenchTrain(net, config)
 net = get_net()
def score(data, y):
validation_res = pd.DataFrame(
 {"id": data["id"].values,
"transactionrevenue": data["revenue"].values,
"predictedrevenue": np.expm1(y)})
 validation_res = validation_res.groupby("id")["transactionrevenue", "predictedrevenue"].sum().reset_index()
return np.sqrt(mean_squared_error(np.log1p(validation_res["transactionrevenue"].values),
np.log1p(validation_res["predictedrevenue"].values)) )<split>
best_epoch = None
if len(os.listdir(".. /input/global-wheat-detection/test/")) > 11:
best_epoch = run_training()
class KFoldValidation() :
def __init__(self, data, n_splits=5):
unique_vis = np.array(sorted(data['id'].astype(str ).unique()))
folds = GroupKFold(n_splits)
 ids = np.arange(data.shape[0])
 self.fold_ids = []
for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
self.fold_ids.append([
ids[data['id'].astype(str ).isin(unique_vis[trn_vis])],
ids[data['id'].astype(str ).isin(unique_vis[val_vis])]
])
 def validate(self, train, test, features, model, name="", prepare_stacking=False,
fit_params={"early_stopping_rounds": 500, "verbose": 100, "eval_metric": "rmse"}):
model.FI = pd.DataFrame(index=features)
 full_score = 0
if prepare_stacking:
test[name] = 0
train[name] = np.NaN
for fold_id,(trn, val)in enumerate(self.fold_ids):
devel = train[features].iloc[trn]
y_devel = np.log1p(train["revenue"].iloc[trn])
 valid = train[features].iloc[val]
y_valid = np.log1p(train["revenue"].iloc[val])
 print("Fold ", fold_id, ":")
 model.fit(devel, y_devel, eval_set=[(valid, y_valid)], **fit_params)
 if len(model.feature_importances_)== len(features):
model.FI['fold' + str(fold_id)] = model.feature_importances_ / model.feature_importances_.sum()
predictions = model.predict(valid)
 predictions[predictions < 0] = 0
print("Fold ", fold_id, " error: ", mean_squared_error(y_valid, predictions)**0.5)
 fold_score = score(train.iloc[val], predictions)
 full_score += fold_score / len(self.fold_ids)
 print("Fold ", fold_id, " score: ", fold_score)
 if prepare_stacking:
train[name].iloc[val] = predictions
test_predictions = model.predict(test[features])
 test_predictions[test_predictions < 0] = 0
test[name] += test_predictions / len(self.fold_ids)
 print("Final score: ", full_score)
 return full_score 
<choose_model_class>
if best_epoch is not None:
print("load retrained model {}".format(best_epoch))
net = create_model_efficientdet_test(compound_coef=7,
weight_path=f'./effdet7/best-checkpoint-{str(best_epoch ).zfill(3)}epoch.bin',
image_size=1024,
model_ema=True)
 else:
net = create_model_efficientdet_test(compound_coef=7,
weight_path=trained_weight_path,
image_size=1024,
model_ema=True )
Kfolder = KFoldValidation(train )<choose_model_class>
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes)
 tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
def make_tta_predictions(images, net, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
lgbmodel = lgb.LGBMRegressor(n_estimators=10000,
objective='regression',
metric='rmse',
max_depth = 5,
num_leaves=30,
min_child_samples=100,
learning_rate=0.01,
boosting = 'gbdt',
min_data_in_leaf= 10,
feature_fraction = 0.9,
bagging_freq = 1,
bagging_fraction = 0.9,
importance_type='gain',
lambda_l1 = 0.2,
bagging_seed=random_seed,
subsample=.8,
colsample_bytree=.9,
use_best_model=True )<train_model>
def get_test_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
Kfolder.validate(train, test, features , lgbmodel, name="lgbfinal", prepare_stacking=True )<save_to_csv>
test_dataset = DatasetRetrieverTest(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
test_data_loader = DataLoader(
 test_dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)
test['revenue'] = np.expm1(test["lgbfinal"])
 test[['id','revenue']].to_csv('submission_lgb.csv', index=False)
 test[['id','revenue']].head()<choose_model_class>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 results = []
for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images, net=net)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
<compute_test_metric><EOS> 
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()