Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<load_from_csv>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
train = pd.read_csv(DATA / "train.csv")
 test = pd.read_csv(DATA / "test.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 test['target']=pd.read_csv(".. /input/tabularplayground-previsionio/tabular-playground-series-jan-2021_train-tabular-playground-series-jan-2021_test-BLEND-LR-2-06964.csv")['target']
train=pd.concat([train,test],0 ).reset_index()
print("train: {}, test: {}, sample sub: {}".format(
 train.shape, test.shape, smpl_sub.shape 
))<prepare_x_and_y>
SEED = 100
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
class TreeModel:

 def __init__(self, model_type: str):
self.model_type = model_type
self.trn_data = None
self.val_data = None
self.model = None
def train(self,
params: dict,
X_train: pd.DataFrame, y_train: np.ndarray,
X_val: pd.DataFrame, y_val: np.ndarray,
train_weight: tp.Optional[np.ndarray] = None,
val_weight: tp.Optional[np.ndarray] = None,
train_params: dict = {}):
if self.model_type == "lgb":
self.trn_data = lgb.Dataset(X_train, label=y_train, weight=train_weight)
 self.val_data = lgb.Dataset(X_val, label=y_val, weight=val_weight)
 self.model = lgb.train(params=params,
train_set=self.trn_data,
valid_sets=[self.trn_data, self.val_data],
**train_params)
 elif self.model_type == "xgb":
self.trn_data = xgb.DMatrix(X_train, y_train, weight=train_weight)
 self.val_data = xgb.DMatrix(X_val, y_val, weight=val_weight)
 self.model = xgb.train(params=params,
dtrain=self.trn_data,
evals=[(self.trn_data, "train"),(self.val_data, "val")],
**train_params)
 elif self.model_type == "cat":
self.trn_data = Pool(X_train, label=y_train, group_id=[0] * len(X_train))
self.val_data = Pool(X_val, label=y_val, group_id=[0] * len(X_val))
self.model = CatBoost(params)
 self.model.fit(
 self.trn_data, eval_set=[self.val_data], use_best_model=True, **train_params)
 else:
raise NotImplementedError
def predict(self, X: pd.DataFrame):
if self.model_type == "lgb":
return self.model.predict(
 X, num_iteration=self.model.best_iteration)
 elif self.model_type == "xgb":
X_DM = xgb.DMatrix(X)
 return self.model.predict(
 X_DM, ntree_limit=self.model.best_ntree_limit)
 elif self.model_type == "cat":
return self.model.predict(X)
 else:
raise NotImplementedError
@property
def feature_names_(self):
if self.model_type == "lgb":
return self.model.feature_name()
elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).keys())
 elif self.model_type == "cat":
return self.model.feature_names_
else:
raise NotImplementedError
@property
def feature_importances_(self):
if self.model_type == "lgb":
return self.model.feature_importance(importance_type="gain")
 elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).values())
 elif self.model_type == "cat":
return self.model.feature_importances_
else:
raise NotImplementedError<init_hyperparams>
def collate_fn(batch):
return tuple(zip(*batch))
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [3,1]]
res_boxes[:, [1,3]] = boxes[:, [0,2]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes)
 tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
ID_COL = "id"
FEAT_COLS = [f"cont{i}" for i in range(1, 15)]
TGT_COL = "target"
N_SPLITS = 5
RANDOM_SEED_LIST = [
42,
2021, 0, 1086, 39
]
MODEL_PARAMS = {
"lgb": {
"objective": "root_mean_squared_error",
"boosting": "gbdt",
"learning_rate": 0.1,
"seed": RANDOM_SEED_LIST[0],
'max_depth': 7,
'colsample_bytree':.85,
"subsample":.85,
"n_jobs": 2,
},
"xgb": {
"objective": "reg:squarederror",
"learning_rate": 0.05,
"seed": RANDOM_SEED_LIST[0],
"max_depth": 7,
"subsample":.85,
"colsample_bytree":.85,
"n_jobs": 2,
'tree_method': "gpu_hist",
"gpu_id": 0,
},
"cat": {
'loss_function': 'RMSE',
'learning_rate': 0.1,
'max_depth': 7,
'random_state': RANDOM_SEED_LIST[0],
"thread_count": 2,
'num_boost_round': 20000
}
}
TRAIN_PARAMS = {
"lgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 100,
"verbose_eval": 100,
},
"xgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 100,
"verbose_eval": 100,
},
"cat": {'early_stopping_rounds': 100, 'verbose_eval': 100}
}<define_variables>
MODELS_PATHS = ['.. /input/frcnnfold012best/frcnn-fold0-best100.pth',
'.. /input/frcnnfold012best/frcnn_fold1_best50.pth',
'.. /input/frcnnfold012best/frcnn_fold2_best50.pth']
def get_model() :
backbone = resnet_fpn_backbone('resnet101', pretrained=False)
 model = FasterRCNN(backbone, num_classes=2)
 return model
frcnn_models = []
DEVICE = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 for path in MODELS_PATHS:
model = get_model()
model.load_state_dict(torch.load(path))
model.eval()
model.to(DEVICE)
 frcnn_models.append(model )
use_feat_cols = []
train_feat = train[[ID_COL]].copy()
test_feat = test[[ID_COL]].copy()<concatenate>
class WheatDataset(Dataset):
def __init__(self, dataframe, image_dir, transforms=None):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.transforms = transforms
def __len__(self)-> int:
return len(self.image_ids)
 def __getitem__(self, idx: int):
image_id = self.image_ids[idx]
image = cv2.imread(f'{self.image_dir}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.df[self.df['image_id'] == image_id]
if self.transforms:
sample = {"image": image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
train_feat = pd.concat([
train_feat, train[FEAT_COLS]], axis=1)
 test_feat = pd.concat([
test_feat, test[FEAT_COLS]], axis=1)
 use_feat_cols.extend(FEAT_COLS )<define_variables>
def get_train_transform() :
return A.Compose([
A.Flip(0.5),
ToTensorV2(p=1.0)
 ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})
 def get_valid_transform() :
return A.Compose([
ToTensorV2(p=1.0)
 ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})
 def get_test_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
def run_train_and_inference(X, X_test, y, use_model, model_params, train_params, seed_list, n_splits):
oof_pred_arr = np.zeros(len(X))
test_pred_arr = np.zeros(len(X_test))
feature_importances = pd.DataFrame()
score_list = []
for seed in seed_list:
if use_model == "cat":
model_params['random_state'] = seed
else:
model_params["seed"] = seed
kf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
 tmp_oof_pred = np.zeros(len(X))
tmp_test_pred = np.zeros(len(X_test))
for fold,(trn_idx, val_idx)in enumerate(kf.split(X, y)) :
print("*" * 100)
 print(f"Seed: {seed} - Fold: {fold}")
 X_trn = X.loc[trn_idx].reset_index(drop=True)
 X_val = X.loc[val_idx].reset_index(drop=True)
 y_trn = y[trn_idx]
y_val = y[val_idx]
model = TreeModel(model_type=use_model)
 with timer(prefix="Model training"):
model.train(
 params=model_params, X_train=X_trn, y_train=y_trn,
X_val=X_val, y_val=y_val, train_params=train_params)
 fi_tmp = pd.DataFrame()
fi_tmp["feature"] = model.feature_names_
fi_tmp["importance"] = model.feature_importances_
fi_tmp["fold"] = fold
fi_tmp["seed"] = seed
feature_importances = feature_importances.append(fi_tmp)
 val_pred = model.predict(X_val)
 score = mean_squared_error(y_val, val_pred, squared=False)
 print(f"score: {score:.5f}")
 score_list.append([seed, fold, score])
 tmp_oof_pred[val_idx] = val_pred
tmp_test_pred += model.predict(X_test)
 oof_score = mean_squared_error(y, tmp_oof_pred, squared=False)
 print(f"oof score: {oof_score: 5f}")
 score_list.append([seed, "oof", oof_score])
 oof_pred_arr += tmp_oof_pred
test_pred_arr += tmp_test_pred / n_splits
oof_pred_arr /= len(seed_list)
 test_pred_arr /= len(seed_list)
 oof_score = mean_squared_error(y, oof_pred_arr, squared=False)
 score_list.append(["avg", "oof", oof_score])
 score_df = pd.DataFrame(
 score_list, columns=["seed", "fold", "rmse score"])
 return oof_pred_arr, test_pred_arr, score_df, feature_importances<prepare_x_and_y>
def make_tta_predictions(images, model, score_threshold=0.40):
with torch.no_grad() :
images = torch.stack(images ).float().to(DEVICE)
 predictions = []
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
X = train_feat[use_feat_cols]
X_test = test_feat[use_feat_cols]
y = train[TGT_COL].values
print(f"train_feat: {X.shape}, test_feat: {X_test.shape}" )<split>
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.45, skip_box_thr=0.45, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
oof_pred_lgb, test_pred_lgb, score_lgb, feat_imps_lgb = run_train_and_inference(
 X, X_test, y, "lgb", MODEL_PARAMS["lgb"], TRAIN_PARAMS["lgb"], RANDOM_SEED_LIST, N_SPLITS )<filter>
DATA_DIR = '.. /input/global-wheat-detection'
device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu')
 test_df = pd.read_csv(os.path.join(DATA_DIR, "sample_submission.csv"))
testdf_pseudos = []
for model in frcnn_models:
results = []
testdf_pseudo = []
test_dataset = WheatDataset(test_df, os.path.join(DATA_DIR, "test"), get_test_transforms())
 test_data_loader = DataLoader(
 test_dataset,
batch_size=4,
shuffle=False,
num_workers=0,
drop_last=False,
collate_fn=collate_fn 
)
 for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images,model,score_threshold=0.70)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': 'nvnn'+image_id,
'width': 1024,
'height': 1024,
'source': 'nvnn',
'x': box[0],
'y': box[1],
'w': box[2],
'h': box[3]
}
testdf_pseudo.append(result)
 testdf_pseudos.append(testdf_pseudo )
score_lgb.loc[score_lgb.fold == "oof"]<split>
new_train_dfs = []
for result in testdf_pseudos:
df = pd.DataFrame(result, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h'])
 new_train_dfs.append(df )
oof_pred_xgb, test_pred_xgb, score_xgb, feat_imps_xgb = run_train_and_inference(
 X, X_test, y, "xgb", MODEL_PARAMS["xgb"], TRAIN_PARAMS["xgb"], RANDOM_SEED_LIST, N_SPLITS )<filter>
train_df = pd.read_csv(f'{DATA_DIR}/train.csv')
 train_df['x'] = -1
train_df['y'] = -1
train_df['w'] = -1
train_df['h'] = -1
def expand_bbox(x):
r = np.array(re.findall("([0-9]+[.]?[0-9]*)", x))
if len(r)== 0:
r = [-1, -1, -1, -1]
return r
train_df[['x', 'y', 'w', 'h']] = np.stack(train_df['bbox'].apply(lambda x: expand_bbox(x)))
 train_df.drop(columns=['bbox'], inplace=True)
 train_df['x'] = train_df['x'].astype(np.float)
 train_df['y'] = train_df['y'].astype(np.float)
 train_df['w'] = train_df['w'].astype(np.float)
 train_df['h'] = train_df['h'].astype(np.float)
 skf = StratifiedKFold(n_splits=30, shuffle=True, random_state=10)
 fold_df = train_df[['image_id']].copy()
fold_df.loc[:, 'bbox_count'] = 1
fold_df = fold_df.groupby('image_id' ).count()
fold_df.loc[:, 'source'] = train_df[['image_id', 'source']].groupby('image_id' ).min() ['source']
fold_df.loc[:, 'group'] = np.char.add(fold_df['source'].values.astype('str'),
fold_df['bbox_count'].apply(lambda x: f'_{x//15}' ).values.astype('str'))
fold_df.loc[:, 'fold'] = 0
for i,(train_index, val_index)in enumerate(skf.split(X=fold_df.index, y=fold_df['group'])) :
fold_df.loc[fold_df.iloc[val_index].index, 'fold'] = i
fold_number = 0
valid_image_ids = fold_df[fold_df['fold']==fold_number].index.values
valid_df = train_df[train_df['image_id'].isin(valid_image_ids)]
valid_df.shape
score_xgb.loc[score_xgb.fold == "oof"]<split>
class WheatDataset(Dataset):
def __init__(self, dataframe, image_dir=DATA_DIR, transforms=None):
super().__init__()
self.image_ids = dataframe['image_id'].unique()
self.df = dataframe
self.image_dir = image_dir
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
records = self.df[self.df['image_id'] == image_id]
if 'nvnn' in image_id:
image_id = image_id[4:]
image = cv2.imread(f'{self.image_dir}/test/{image_id}.jpg', cv2.IMREAD_COLOR)
 else:
image = cv2.imread(f'{self.image_dir}/train/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
area =(boxes[:, 3] - boxes[:, 1])*(boxes[:, 2] - boxes[:, 0])
 area = torch.as_tensor(area, dtype=torch.float32)
 labels = torch.ones(( records.shape[0],), dtype=torch.int64)
 iscrowd = torch.zeros(( records.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 target['area'] = area
target['iscrowd'] = iscrowd
if self.transforms:
sample = {
'image': image,
'bboxes': target['boxes'],
'labels': labels
}
sample = self.transforms(**sample)
 image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'] = target['boxes'].float()
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
oof_pred_cat, test_pred_cat, score_cat, feat_imps_cat = run_train_and_inference(
 X, X_test, y, "cat", MODEL_PARAMS["cat"], TRAIN_PARAMS["cat"], RANDOM_SEED_LIST, N_SPLITS )<filter>
class Averager:
def __init__(self):
self.current_total = 0.0
self.iterations = 0.0
def send(self, value):
self.current_total += value
self.iterations += 1
@property
def value(self):
if self.iterations == 0:
return 0
else:
return 1.0 * self.current_total / self.iterations
def reset(self):
self.current_total = 0.0
self.iterations = 0.0
score_cat.loc[score_cat.fold == "oof"]<define_variables>
frcnn_models_trained = []
for model, train_df in zip(frcnn_models, new_train_dfs):
train_dataset = WheatDataset(train_df, image_dir=DATA_DIR, transforms=get_train_transform())
 valid_dataset = WheatDataset(valid_df, image_dir=DATA_DIR, transforms=get_valid_transform())
 indices = torch.randperm(len(train_dataset)).tolist()
train_data_loader = DataLoader(
 train_dataset,
batch_size=8,
shuffle=False,
num_workers=0,
collate_fn=collate_fn 
)
 valid_data_loader = DataLoader(
 valid_dataset,
batch_size=4,
shuffle=False,
num_workers=0,
collate_fn=collate_fn 
)
 model.train()
model.to(device)
 params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params, lr=0.01, momentum=0.9, weight_decay=0.0001)
 lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.5)
 num_epochs = 10
loss_hist = Averager()
itr = 1
for epoch in range(num_epochs):
loss_hist.reset()
for images, targets, image_ids in train_data_loader:
images = list(image.to(device)for image in images)
 targets = [{k: v.to(device)for k, v in t.items() } for t in targets]
loss_dict = model(images, targets)
 losses = sum(loss for loss in loss_dict.values())
 loss_value = losses.item()
loss_hist.send(loss_value)
 optimizer.zero_grad()
losses.backward()
optimizer.step()
if itr % 50 == 0:
print(f"Iteration
itr += 1
if lr_scheduler is not None:
lr_scheduler.step()
print(f"Epoch
frcnn_models_trained.append(model)
 frcnn_models_trained = [model.eval() for model in frcnn_models_trained]
model_names = ["lgb", "xgb", "cat"]<create_dataframe>
USE_OPTIMIZE = False
pd.DataFrame(
 np.corrcoef([
oof_pred_lgb,
oof_pred_xgb,
oof_pred_cat
]),
columns=model_names, index=model_names )<create_dataframe>
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )
pd.DataFrame(
 np.corrcoef([
test_pred_lgb,
test_pred_xgb,
test_pred_cat
]),
columns=model_names, index=model_names )<compute_test_metric>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']]
df_folds['bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds['source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds['stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 train_index, holdout_index = next(skf.split(X=df_folds.index, y=df_folds['stratify_group']))
df_holdout = df_folds.iloc[holdout_index].copy()
df_folds = df_folds.iloc[train_index].copy()
df_folds['fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
oof_pred_avg =(oof_pred_lgb + oof_pred_xgb + oof_pred_cat)/ 3
oof_score_avg = mean_squared_error(y, oof_pred_avg, squared=False)
 print(f"oof score avg: {oof_score_avg:.5f}" )<compute_test_metric>
holdout_dataset = DatasetRetriever(
 image_ids=df_holdout.index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 holdout_loader = DataLoader(
 holdout_dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)
test_pred_avg =(test_pred_lgb + test_pred_xgb + test_pred_cat)/ 3<save_to_csv>
def load_net(checkpoint_path, version):
config = get_efficientdet_config(f'tf_efficientdet_d{version}')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
models = [
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold0-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold1-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold2-epo50last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold3-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold4-epo50last.bin', 5),
]
sub = smpl_sub.copy()
sub[TGT_COL] = test_pred_avg
sub.to_csv("submission.csv", index=False)
 sub.head()<set_options>
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes*2 ).clip(min=0, max=1023 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores
if USE_OPTIMIZE:
all_predictions = []
for images, targets, image_ids in tqdm(holdout_loader, total=len(holdout_loader)) :
with torch.no_grad() :
images = torch.stack(images)
 images = images.cuda().float()
fold_predictions = {}
for fold_number in range(2):
fold_predictions[fold_number] = models[fold_number](images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
image_predictions = {
'image_id': image_ids[i],
'gt_boxes':(targets[i]['boxes'].cpu().numpy() *2 ).clip(min=0, max=1023 ).astype(int),
}
for fold_number in range(2):
boxes, scores = process_det(i, fold_predictions[fold_number])
 image_predictions[f'pred_boxes_fold{fold_number}'] = boxes
image_predictions[f'scores_fold{fold_number}'] = scores
all_predictions.append(image_predictions )
sns.set_style("darkgrid")
 %matplotlib inline
warnings.filterwarnings("ignore" )<load_from_csv>
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'{TRAIN_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth")
 iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x )
%%time
train = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/train.csv')
 test = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/test.csv')
 sample_sub = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv')
 org_len = len(train.drop('id',axis=1))<count_values>
def calculate_final_score(
 all_predictions,
iou_thr,
skip_box_thr,
method,
sigma=0.5, 
):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
image_id = all_predictions[i]['image_id']
folds_boxes, folds_scores, folds_labels = [], [], []
for fold_number in range(2):
pred_boxes = all_predictions[i][f'pred_boxes_fold{fold_number}'].copy()
scores = all_predictions[i][f'scores_fold{fold_number}'].copy()
folds_boxes.append(pred_boxes)
 folds_scores.append(scores)
 folds_labels.append(np.ones(pred_boxes.shape[0]))
if method == 'weighted_boxes_fusion':
boxes, scores, labels = weighted_boxes_fusion(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 elif method == 'nms':
boxes, scores, labels = nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr)
 elif method == 'soft_nms':
boxes, scores, labels = soft_nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, thresh=skip_box_thr, sigma=sigma)
 elif method == 'non_maximum_weighted':
boxes, scores, labels = non_maximum_weighted(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 else:
raise
image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )
print("Train data set dtypes:
")
 print(f"Shape : {train.shape}")
 print(f"{train.dtypes.value_counts() }")
 print('*'*30)
 print("Test data set dtypes:
")
 print(f"Shape : {test.shape}")
 print(f"{test.dtypes.value_counts() }" )<save_to_csv>
if USE_OPTIMIZE:
print('[WBF]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='weighted_boxes_fusion', 
))
print('[NMS]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='nms', 
))
print('[SOFT NMS]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
sigma=0.1,
method='soft_nms', 
))
print('[NMW]: ', calculate_final_score(
 all_predictions,
iou_thr=0.55,
skip_box_thr=0.0001,
method='non_maximum_weighted', 
))
profile = pandas_profiling.ProfileReport(train,minimal=True)
 profile.to_file(output_file="output.html")
 profile<drop_column>
def log(text):
with open('opt.log', 'a+')as logger:
logger.write(f'{text}
')
 def optimize(space, all_predictions, method, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*5 + f'{method}' + '-'*5)
 log(params)
 final_score = calculate_final_score(all_predictions, method=method, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )
train_cont = train.drop('id',axis=1 )<feature_engineering>
space = [
Real(0, 1, name='iou_thr'),
Real(0.25, 1, name='skip_box_thr'),
]
if USE_OPTIMIZE:
opt_result = optimize(
 space,
all_predictions,
method='weighted_boxes_fusion',
n_calls=50,
)
train_cont['new'] = train_cont['cont2']*train_cont['cont3']*train_cont['cont6']*train_cont['cont7']*train_cont['cont11']*train_cont['cont12']
test['new'] = test['cont2']*test['cont3']*test['cont6']*test['cont7']*test['cont11']*test['cont12']<feature_engineering>
if USE_OPTIMIZE:
best_final_score = -opt_result.fun
best_iou_thr = opt_result.x[0]
best_skip_box_thr = opt_result.x[1]
else:
best_final_score = 0.7197
best_iou_thr = 0.450
best_skip_box_thr = 0.450
print('-'*13 + 'WBF' + '-'*14)
 print(f'[Best Iou Thr]: {best_iou_thr:.3f}')
 print(f'[Best Skip Box Thr]: {best_skip_box_thr:.3f}')
 print(f'[Best Score]: {best_final_score:.4f}')
 print('-'*30 )
train_cont['new1'] = train_cont['cont9']*train_cont['cont10']*train_cont['cont1']
test['new1'] = test['cont9']*test['cont10']*test['cont1']<feature_engineering>
if USE_OPTIMIZE:
all_predictions = []
for fold_number in range(5):
validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True, 
)
 validation_loader = DataLoader(
 validation_dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn 
)
 for images, targets, image_ids in tqdm(validation_loader, total=len(validation_loader)) :
with torch.no_grad() :
images = torch.stack(images)
 images = images.cuda().float()
det = models[fold_number](images, torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
all_predictions.append({
'pred_boxes':(boxes*2 ).clip(min=0, max=1023 ).astype(int),
'scores': scores,
'gt_boxes':(targets[i]['boxes'].cpu().numpy() *2 ).clip(min=0, max=1023 ).astype(int),
'image_id': image_ids[i],
} )
features = train_cont.drop('target',axis=1 ).columns
train_cont['mean'] = train_cont[features].mean(axis=1)
 test['mean'] = test[features].mean(axis=1 )<define_variables>
def calculate_final_score(all_predictions, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
pred_boxes = all_predictions[i]['pred_boxes'].copy()
scores = all_predictions[i]['scores'].copy()
image_id = all_predictions[i]['image_id']
indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )
low_cont = ['target', 'cont10', 'cont9', 'cont7']
up_cont = ['cont10']
n999 = [ np.percentile(train_cont[i],99.9)for i in train_cont[up_cont]]
n001 = [ np.percentile(train_cont[i],0.1)for i in train_cont[low_cont]]<set_options>
best_final_score, best_score_threshold = 0, 0
if USE_OPTIMIZE:
for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
else:
best_score_threshold = 0.40
for i, j in enumerate(low_cont):
train_cont = train_cont[train_cont[j] > n001[i]]
gc.collect()
for i, j in enumerate(up_cont):
train_cont = train_cont[train_cont[j] < n999[i]]
gc.collect()<prepare_x_and_y>
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30 )
X_train = train_cont.drop('target',axis=1)
 y_train = train_cont['target']<find_best_params>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def objective(trial):
params = {
'n_estimators': trial.suggest_int('n_estimators', 350, 1000),
'max_depth': trial.suggest_int('max_depth', 6, 13),
'learning_rate': trial.suggest_uniform('learning_rate', 0.009, 0.10),
'subsample': trial.suggest_uniform('subsample', 0.50, 1),
'colsample_bytree': trial.suggest_uniform('colsample_bytree', 0.50, 1),
'gamma': trial.suggest_int('gamma', 0, 0.05),
'objective':'reg:squarederror',
'eval_metric' : 'rmse',
'tree_method':'gpu_hist',
}
clf = xgb.XGBRegressor(**params)
 rmse_scores = []
X_train_k = X_train.values
y_train_k = y_train.values
skf = KFold(n_splits=3,shuffle=True)
 for train_idx, valid_idx in skf.split(X_train_k,y_train_k):
train_data = X_train_k[train_idx, :], y_train_k[train_idx]
valid_data = X_train_k[valid_idx, :], y_train_k[valid_idx]
clf.fit(X_train_k[train_idx, :], y_train_k[train_idx])
 pred = clf.predict(X_train_k[valid_idx, :])
 rmse = np.sqrt(mean_squared_error(y_train_k[valid_idx],pred))
rmse_scores.append(rmse)
 print(f'Trial done: Accuracy values on folds: {rmse_scores}')
 return np.average(rmse_scores )<find_best_params>
def get_valid_transforms512() :
return A.Compose([A.Resize(height=512, width=512, p=1.0), ToTensorV2(p=1.0)], p=1.0)
 def get_valid_transforms1024() :
return A.Compose([ToTensorV2(p=1.0)], p=1.0)
 dataset512 = TestDatasetRetriever(image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms512())
 data_loader512 = DataLoader(
 dataset512,
batch_size=2,
shuffle=False,
num_workers=0,
drop_last=False,
collate_fn=collate_fn 
)
 dataset1024 = TestDatasetRetriever(image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms1024())
 data_loader1024 = DataLoader(
 dataset1024,
batch_size=2,
shuffle=False,
num_workers=0,
drop_last=False,
collate_fn=collate_fn
)
n_trials = 5
FIT_XGB = True
if FIT_XGB:
study = optuna.create_study(direction="minimize")
 study.optimize(objective, n_trials=n_trials)
 print("Number of finished trials: {}".format(len(study.trials)))
 print("Best trial:")
 trial = study.best_trial
print(" Value: {}".format(trial.value))
print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value))<init_hyperparams>
def make_frcnn_predictions(images, score_threshold=0.50):
with torch.no_grad() :
images = torch.stack(images ).cuda().float()
predictions = []
for fold_number, model in enumerate(frcnn_models_trained):
for tta_transform in tta_transforms:
result = []
outputs = model(tta_transform.batch_augment(images.clone()))
for i, image in enumerate(images):
boxes = outputs[i]['boxes'].data.cpu().numpy()
scores = outputs[i]['scores'].data.cpu().numpy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
best_param = {'n_estimators': 751, 'max_depth': 10,
'learning_rate': 0.019789645280696613,
'subsample': 0.8730019407814834,
'colsample_bytree': 0.6012295369579667,'gamma':0}
best_param['objective'] ='reg:squarederror'
best_param['tree_method'] ='gpu_hist'
best_param['eval_metric'] ='rmse'<train_model>
for i, model in enumerate(models):
print(type(model))
model = xgb.XGBRegressor(**best_param)
 model.fit(X_train,y_train )<predict_on_test>
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes*2 ).clip(min=0, max=1023 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores
predictions_final = model.predict(test.drop('id',axis=1))<save_to_csv>
def make_predictions(images, score_threshold=best_score_threshold):
with torch.no_grad() :
images = torch.stack(images ).cuda().float()
predictions = []
for fold_number, net in enumerate(models):
for tta_transform in tta_transforms:
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
submission = pd.DataFrame({
"id": test["id"],
"target":predictions_final
})
 submission.to_csv('my_submission.csv', index=False )<load_from_csv>
def run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
%%time
train = pd.read_csv(".. /input/tabular-playground-series-jan-2021/train.csv")
 test = pd.read_csv(".. /input/tabular-playground-series-jan-2021/test.csv")
 df=pd.concat([train,test],0 ).reset_index(drop=True)
 df.sort_values(by=['id'],inplace=True)
 df<feature_engineering>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
cols=df.drop(['id','target'],1 ).columns
df['mean']=df[cols].mean(axis=1)
 df['min']=df[cols].min(axis=1)
 df['max']=df[cols].max(axis=1)
 df['std']=df[cols].std(axis=1)
 df['kurt']=df[cols].kurt(axis=1)
 df['mad']=df[cols].mad(axis=1)
 df['median']=df[cols].median(axis=1)
 df['skew']=df[cols].skew(axis=1)
 df['sum']=df[cols].sum(axis=1)
 df['sem']=df[cols].sem(axis=1)
 df['idxmin']=df[cols].idxmin(axis=1 ).str.replace("cont","" ).astype(int)
 df['idxmax']=df[cols].idxmax(axis=1 ).str.replace("cont","" ).astype(int)
 df<feature_engineering>
results = []
for(images_effdet, image_ids_effdet),(images_frcnn, image_ids_frcnn)in zip(data_loader512, data_loader1024):
if image_ids_effdet == image_ids_frcnn:
frcnn_predictions = make_frcnn_predictions(images_frcnn)
 effdet_predictions = make_effdet_predictions(images_effdet)
 images = image_ids_effdet
predictions = frcnn_predictions + effdet_predictions
for i, image in enumerate(images):
boxes, scores, labels = run_wbf(effdet_predictions, image_index=i)
 boxes =(boxes * 2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
 else:
print('Error')
 break
for c1 in cols:
print(c1)
 for c2 in cols:
if c1<c2:
df[f'{c1}MOINS{c2}']=df[f'{c1}']-df[f'{c2}']
df[f'{c1}PLUS{c2}']=df[f'{c1}']+df[f'{c2}']
df[f'{c1}FOIS{c2}']=df[f'{c1}']*df[f'{c2}']
df[f'{c1}DIV{c2}']=df[f'{c1}']/df[f'{c2}'] 
<prepare_x_and_y>
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )
target='target'
train=df[df['target'].notnull() ]
test=df[~(df['target'].notnull())]
print(train.shape,test.shape)
 y = train[target].copy()
X = train.drop([target,'id'],1)
 features=X.columns
FOLDS=13
SEED=2021
oof=np.zeros(y.shape[0])
 preds=np.zeros(test.shape[0])
 skf = KFold(n_splits=FOLDS, shuffle=True, random_state=SEED)
 for fold,(idxT,idxV)in enumerate(skf.split(X)) :
print("FOLD",fold)
 X_train,y_train=X.loc[idxT,features],y[idxT]
X_test,y_test=X.loc[idxV,features],y[idxV]
fit_params={"early_stopping_rounds":300,
"eval_metric" : ['rmse'],
"eval_set" : [(X_test,y_test)],
'eval_names': ['valid'],
'verbose': 100,
'feature_name': 'auto',
'categorical_feature': 'auto'
}
evals_result = {}
clf = lgb.LGBMRegressor(num_leaves= 255, max_depth=-1,
seed=42,
objective="regression",
silent=True,
verbose=1,
num_threads=-1,
n_estimators=100000,
colsample_bytree=0.7,
subsample=0.8,
learning_rate=0.01)
 START=time.time()
clf.fit(X_train, y_train, **fit_params)
 oof[idxV]=clf.predict(X_test)
 preds+=clf.predict(test[features])/FOLDS
print("RMSE",mean_squared_error(y_test,oof[idxV]))
END=time.time()
print("time to train ",round(END-START,1),"secondes")
 feat_imp = pd.Series(clf.feature_importances_, index=X_train.columns)
 feat_imp=pd.DataFrame(feat_imp ).reset_index()
feat_imp.columns=['id','importance']
feat_imp.sort_values('importance',ascending=False,inplace=True)
 joblib.dump(clf, f'{fold}.pkl')
 print(feat_imp[:50])
 print("RMSE Global",mean_squared_error(y, oof)) 
<save_to_csv>
def load_net(checkpoint_path, version):
config = get_efficientdet_config(f'tf_efficientdet_d{version}')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
models = [
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold0-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold1-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold2-epo50last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold3-epo40last.bin', 5),
load_net('.. /input/efficientdet5allfoldlast/effdet5-fold4-epo50last.bin', 5),
]
sample_submission=pd.read_csv(".. /input/tabular-playground-series-jan-2021/sample_submission.csv")
 sample_submission['target'] = preds
sample_submission.to_csv('submission.csv', index=False )<set_options>
best_score_threshold = 0.40
best_iou_thr = 0.45
best_skip_box_thr = 0.45
%matplotlib inline<load_from_csv>
class BaseWheatTTA:
image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
train = pd.read_csv(DATA / "train.csv")
 test = pd.read_csv(DATA / "test.csv")
 smpl_sub = pd.read_csv(DATA / "sample_submission.csv")
 print("train: {}, test: {}, sample sub: {}".format(
 train.shape, test.shape, smpl_sub.shape 
))<prepare_x_and_y>
transform = TTACompose([
TTARotate90() ,
TTAVerticalFlip() ,
] )
class TreeModel:

 def __init__(self, model_type: str):
self.model_type = model_type
self.trn_data = None
self.val_data = None
self.model = None
def train(self,
params: dict,
X_train: pd.DataFrame, y_train: np.ndarray,
X_val: pd.DataFrame, y_val: np.ndarray,
train_weight: tp.Optional[np.ndarray] = None,
val_weight: tp.Optional[np.ndarray] = None,
train_params: dict = {}):
if self.model_type == "lgb":
self.trn_data = lgb.Dataset(X_train, label=y_train, weight=train_weight)
 self.val_data = lgb.Dataset(X_val, label=y_val, weight=val_weight)
 self.model = lgb.train(params=params,
train_set=self.trn_data,
valid_sets=[self.trn_data, self.val_data],
**train_params)
 elif self.model_type == "xgb":
self.trn_data = xgb.DMatrix(X_train, y_train, weight=train_weight)
 self.val_data = xgb.DMatrix(X_val, y_val, weight=val_weight)
 self.model = xgb.train(params=params,
dtrain=self.trn_data,
evals=[(self.trn_data, "train"),(self.val_data, "val")],
**train_params)
 elif self.model_type == "cat":
self.trn_data = Pool(X_train, label=y_train, group_id=[0] * len(X_train))
self.val_data = Pool(X_val, label=y_val, group_id=[0] * len(X_val))
self.model = CatBoost(params)
 self.model.fit(
 self.trn_data, eval_set=[self.val_data], use_best_model=True, **train_params)
 else:
raise NotImplementedError
def predict(self, X: pd.DataFrame):
if self.model_type == "lgb":
return self.model.predict(
 X, num_iteration=self.model.best_iteration)
 elif self.model_type == "xgb":
X_DM = xgb.DMatrix(X)
 return self.model.predict(
 X_DM, ntree_limit=self.model.best_ntree_limit)
 elif self.model_type == "cat":
return self.model.predict(X)
 else:
raise NotImplementedError
@property
def feature_names_(self):
if self.model_type == "lgb":
return self.model.feature_name()
elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).keys())
 elif self.model_type == "cat":
return self.model.feature_names_
else:
raise NotImplementedError
@property
def feature_importances_(self):
if self.model_type == "lgb":
return self.model.feature_importance(importance_type="gain")
 elif self.model_type == "xgb":
return list(self.model.get_score(importance_type="gain" ).values())
 elif self.model_type == "cat":
return self.model.feature_importances_
else:
raise NotImplementedError<init_hyperparams>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0, 
)
 dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=2,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)
ID_COL = "id"
FEAT_COLS = [f"cont{i}" for i in range(1, 15)]
TGT_COL = "target"
N_SPLITS = 7
RANDOM_SEED_LIST = [
42,
]
MODEL_PARAMS = {
"lgb": {
"objective": "root_mean_squared_error",
"boosting": "gbdt",
"learning_rate": 0.1,
"seed": RANDOM_SEED_LIST[0],
'max_depth': 7,
'colsample_bytree':.85,
"subsample":.85,
"n_jobs": 2,
},
"xgb": {
"objective": "reg:squarederror",
"learning_rate": 0.05,
"seed": RANDOM_SEED_LIST[0],
"max_depth": 7,
"subsample":.85,
"colsample_bytree":.85,
"n_jobs": 2,
'tree_method': "gpu_hist",
"gpu_id": 0,
},
"cat": {
'loss_function': 'RMSE',
'learning_rate': 0.1,
'max_depth': 7,
'random_state': RANDOM_SEED_LIST[0],
"thread_count": 2,
'num_boost_round': 20000
}
}
TRAIN_PARAMS = {
"lgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 100,
"verbose_eval": 100,
},
"xgb": {
"num_boost_round": 20000,
"early_stopping_rounds": 100,
"verbose_eval": 100,
},
"cat": {'early_stopping_rounds': 100, 'verbose_eval': 100}
}<define_variables>
def make_predictions(images, score_threshold=best_score_threshold):
with torch.no_grad() :
images = torch.stack(images ).cuda().float()
predictions = []
for fold_number, net in enumerate(models):
for tta_transform in tta_transforms:
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 result = []
for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
use_feat_cols = []
train_feat = train[[ID_COL]].copy()
test_feat = test[[ID_COL]].copy()<concatenate>
def run_wbf(predictions, image_index, image_size=512, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).tolist() for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
train_feat = pd.concat([
train_feat, train[FEAT_COLS]], axis=1)
 test_feat = pd.concat([
test_feat, test[FEAT_COLS]], axis=1)
 use_feat_cols.extend(FEAT_COLS )<define_variables>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
def run_train_and_inference(X, X_test, y, use_model, model_params, train_params, seed_list, n_splits):
oof_pred_arr = np.zeros(len(X))
test_pred_arr = np.zeros(len(X_test))
feature_importances = pd.DataFrame()
score_list = []
for seed in seed_list:
if use_model == "cat":
model_params['random_state'] = seed
else:
model_params["seed"] = seed
kf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)
 tmp_oof_pred = np.zeros(len(X))
tmp_test_pred = np.zeros(len(X_test))
for fold,(trn_idx, val_idx)in enumerate(kf.split(X, y)) :
print("*" * 100)
 print(f"Seed: {seed} - Fold: {fold}")
 X_trn = X.loc[trn_idx].reset_index(drop=True)
 X_val = X.loc[val_idx].reset_index(drop=True)
 y_trn = y[trn_idx]
y_val = y[val_idx]
model = TreeModel(model_type=use_model)
 with timer(prefix="Model training"):
model.train(
 params=model_params, X_train=X_trn, y_train=y_trn,
X_val=X_val, y_val=y_val, train_params=train_params)
 fi_tmp = pd.DataFrame()
fi_tmp["feature"] = model.feature_names_
fi_tmp["importance"] = model.feature_importances_
fi_tmp["fold"] = fold
fi_tmp["seed"] = seed
feature_importances = feature_importances.append(fi_tmp)
 val_pred = model.predict(X_val)
 score = mean_squared_error(y_val, val_pred, squared=False)
 print(f"score: {score:.5f}")
 score_list.append([seed, fold, score])
 tmp_oof_pred[val_idx] = val_pred
tmp_test_pred += model.predict(X_test)
 oof_score = mean_squared_error(y, tmp_oof_pred, squared=False)
 print(f"oof score: {oof_score: 5f}")
 score_list.append([seed, "oof", oof_score])
 oof_pred_arr += tmp_oof_pred
test_pred_arr += tmp_test_pred / n_splits
oof_pred_arr /= len(seed_list)
 test_pred_arr /= len(seed_list)
 oof_score = mean_squared_error(y, oof_pred_arr, squared=False)
 score_list.append(["avg", "oof", oof_score])
 score_df = pd.DataFrame(
 score_list, columns=["seed", "fold", "rmse score"])
 return oof_pred_arr, test_pred_arr, score_df, feature_importances<prepare_x_and_y>
results = []
for images, image_ids in data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
X = train_feat[use_feat_cols]
X_test = test_feat[use_feat_cols]
y = train[TGT_COL].values
print(f"train_feat: {X.shape}, test_feat: {X_test.shape}" )<split>
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df