Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<load_from_csv>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID' )<count_values>
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
Y_train.value_counts()<split>
TRAIN_DATA_PATH = '.. /input/global-wheat-detection/train/'
TRAIN_CSV_PATH = '.. /input/global-wheat-detection/train.csv'
TEST_DATA_PATH = '.. /input/global-wheat-detection/test/'
if len(os.listdir('.. /input/global-wheat-detection/test/')) >11:
PL_OPT = True
else:
PL_OPT = False
warmup_opt = True
warmup_epoch = 1
PL_lr_sche = 'plat'
cos_lr_min = 1e-8
PL_lr = 0.0001
PL_batchsize = 1
PL_epoch = 5
PL_thr = 0.25
img_size = 1024
OOF_thr = 0.3
WBF_iou_thr = 0.44
WBF_skip_thr = 0.43
path1 = [
'.. /input/212121/21_all - 0.39332 train - epoch89.bin'
]
path2 = [
'.. /input/292929/29_0 - 0.40681 train - 0.37596 val - epoch94.bin',
]
EPOCHS = 5
y_pred = np.zeros(sample_submission.shape[0])
 y_oof = np.zeros(X_train.shape[0])
 kf = KFold(n_splits = EPOCHS , shuffle = True)
 for x_train_index , x_val_index in kf.split(X_train , Y_train):
clf = xgb.XGBClassifier(
 n_estimators=500,
max_depth=4,
learning_rate=0.005,
subsample=0.2,
colsample_bytree = 0.2 
)
 x_tr , x_val = X_train.iloc[x_train_index , :] , X_train.iloc[x_val_index,:]
y_tr , y_val = Y_train.iloc[x_train_index] , Y_train.iloc[x_val_index]
clf.fit(x_tr,y_tr)
 y_pred_train = clf.predict_proba(x_val)[:,1]
y_oof[x_val_index] = y_pred_train
print('ROC AUC {}'.format(roc_auc_score(y_val, y_pred_train)))
 y_pred+= clf.predict_proba(X_test)[:,1] / EPOCHS 
<save_to_csv>
def get_train_transforms() :
return A.Compose(
 [
A.RandomSizedCrop(min_max_height=(800, 800), height=1024, width=1024, p=0.5),
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=img_size, width=img_size, p=1),
A.Cutout(num_holes=8, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=img_size, width=img_size, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_test_transforms() :
return A.Compose([
A.Resize(height=img_size, width=img_size, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
sample_submission['isFraud'] = y_pred
sample_submission.to_csv('second_simple_xgboost_Fraud_Baseline.csv' )<load_from_csv>
marking = pd.read_csv(TRAIN_CSV_PATH)
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )
sub1 = pd.read_csv('.. /input/stacking/stack_median.csv')
 sub2 = pd.read_csv('.. /input/stacking-higher-and-higher/stack_median.csv')
 sub3 = pd.read_csv('.. /input/ensemble/submission_p2_1.csv')
 sub4 = pd.read_csv('.. /input/safe-box/blend02.csv')
 sub5 = pd.read_csv('.. /input/stackers-blend-top-4/All_Blends.csv')
 sample = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv' )<save_to_csv>
class DatasetT(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{TEST_DATA_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
dataset = DatasetT(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{TEST_DATA_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)
sample['isFraud'] =(0.35*sub1['isFraud'] + 0.30*sub2['isFraud'] + 0.25*sub3['isFraud'] + 0.15*sub4['isFraud'] + 0.15*sub5['isFraud'])
 sample.to_csv('submission.csv', index=False )<define_variables>
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(image_id, cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes
sub_path = ".. /input/ieeesubmissions"
all_files = os.listdir(sub_path)
 all_files<load_from_csv>
def load_test_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=img_size
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
outs = [pd.read_csv(os.path.join(sub_path, f), index_col=0)for f in all_files]
concat_sub = pd.concat(outs, axis=1)
 cols = list(map(lambda x: "ieee" + str(x), range(len(concat_sub.columns))))
concat_sub.columns = cols
concat_sub.reset_index(inplace=True)
 concat_sub.head()
ncol = concat_sub.shape[1]<feature_engineering>
def load_train_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = img_size
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
return DetBenchTrain(net, config )
concat_sub['ieee_max'] = concat_sub.iloc[:, 1:ncol].max(axis=1)
 concat_sub['ieee_min'] = concat_sub.iloc[:, 1:ncol].min(axis=1)
 concat_sub['ieee_mean'] = concat_sub.iloc[:, 1:ncol].mean(axis=1)
 concat_sub['ieee_median'] = concat_sub.iloc[:, 1:ncol].median(axis=1 )<define_variables>
class BaseWheatTTA:

 image_size = img_size
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
cutoff_lo = 0.8
cutoff_hi = 0.2<save_to_csv>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
concat_sub['isFraud'] = concat_sub['ieee_mean']
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_mean.csv',
index=False, float_format='%.6f' )<save_to_csv>
def make_tta_predictions(images, models, score_threshold):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
assert images.shape[0] == 1
predictions = []
boxes_All = []
scores_All = []
for tta_transform in tta_transforms:
for net in models:
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 boxes = det[0].detach().cpu().numpy() [:,:4]
scores = det[0].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
scores = scores[indexes]
if len(boxes_All)== 0:
boxes_All = boxes
scores_All = scores
else:
boxes_All = np.concatenate(( boxes_All,boxes),0)
 scores_All = np.concatenate(( scores_All,scores),0)
 result = {
'boxes': boxes_All,
'scores': scores_All,
}
predictions.append([result])
 return predictions
concat_sub['isFraud'] = concat_sub['ieee_median']
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_median.csv',
index=False, float_format='%.6f' )<save_to_csv>
class TrainGlobalConfig:
num_workers = 2
batch_size = PL_batchsize
n_epochs = PL_epoch
lr = PL_lr
folder = 'plabel_model'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
if PL_lr_sche == 'cos':
SchedulerClass = torch.optim.lr_scheduler.CosineAnnealingLR
scheduler_params = dict(
 T_max=PL_epoch,
eta_min=cos_lr_min, 
)
 elif PL_lr_sche == 'plat':
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)
concat_sub['isFraud'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1), 1,
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
0, concat_sub['ieee_median']))
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_pushout_median.csv',
index=False, float_format='%.6f' )<feature_engineering>
class WarmUp(_LRScheduler):

 def __init__(self, optimizer, total_iters, last_epoch=-1):
self.total_iters = total_iters
super(WarmUp, self ).__init__(optimizer, last_epoch)
 def get_lr(self):

 return [base_lr * self.last_epoch /(self.total_iters + 1e-8)for base_lr in self.base_lrs]
concat_sub['isFraud'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['ieee_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['ieee_min'],
concat_sub['ieee_mean']))
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_minmax_mean.csv',
index=False, float_format='%.6f' )<feature_engineering>
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config, train_loader_length):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 if warmup_opt:
iter_per_epoch = train_loader_length
total_iters = iter_per_epoch * warmup_epoch
self.WarmUp_scheduler = WarmUp(self.optimizer, total_iters)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint1.bin')
 if warmup_opt and(self.epoch+1)<= warmup_epoch:
pass
else:
if self.config.validation_scheduler:
if PL_lr_sche == 'plat':
self.scheduler.step(metrics=summary_loss.avg)
 elif PL_lr_sche == 'cos':
self.scheduler.step()
self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
self.optimizer.zero_grad()
loss, _, _ = self.model(images, boxes, labels)
 loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if warmup_opt and(self.epoch+1)<= warmup_epoch:
self.WarmUp_scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )
concat_sub['isFraud'] = np.where(np.all(concat_sub.iloc[:,1:ncol] > cutoff_lo, axis=1),
concat_sub['ieee_max'],
np.where(np.all(concat_sub.iloc[:,1:ncol] < cutoff_hi, axis=1),
concat_sub['ieee_min'],
concat_sub['ieee_median']))
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_minmax_median.csv',
index=False, float_format='%.6f' )<save_to_csv>
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
concat_sub['isFraud'] = concat_sub['ieee0'].rank(method ='min')+ concat_sub['ieee1'].rank(method ='min')+ concat_sub['ieee2'].rank(method ='min')+concat_sub['ieee3'].rank(method ='min')
 concat_sub['isFraud'] =(concat_sub['isFraud']-concat_sub['isFraud'].min())/(concat_sub['isFraud'].max() - concat_sub['isFraud'].min())
 concat_sub.describe()
concat_sub[['TransactionID', 'isFraud']].to_csv('stack_rank.csv', index=False, float_format='%.8f' )<define_variables>
if PL_OPT:
test_models = []
for p in path2:
test_models.append(load_test_net(p))
results_plabel = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images, test_models, PL_thr)
 for i, image in enumerate(images):
assert i == 0
image_id = image_ids[i]
image_ = cv2.imread(f'{TEST_DATA_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 h,w,_ = np.shape(image_)
 boxes, scores, labels = run_wbf(predictions, img_size, WBF_iou_thr, WBF_skip_thr)
 if img_size == 512:
boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 elif img_size == 1024:
boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result_p = {
'image_id': image_id,
'width':w,
'height':h,
'source':'usask_1',
'x':box[0],
'y':box[1],
'w':box[2],
'h':box[3],
}
results_plabel.append(result_p)
 del test_models
gc.collect()
results_df = pd.DataFrame(results_plabel, columns=['image_id', 'width','height','source','x','y','w','h'])
 results_df['image_id'] = results_df['image_id'].apply(lambda x: TEST_DATA_PATH+'/'+ x+'.jpg')
 marking['image_id'] = marking['image_id'].apply(lambda x: TRAIN_DATA_PATH+'/'+ x+'.jpg')
 train_data_plabel = pd.concat([results_df,marking], axis=0)
 skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = train_data_plabel[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
fold_number = 0
train_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] != fold_number].index.values,
marking=train_data_plabel,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=train_data_plabel,
transforms=get_valid_transforms() ,
test=True, 
)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 train_models = []
for p in path1:
train_models.append(load_train_net(p))
device = torch.device('cuda:0')
 fitter = Fitter(model=train_models[0].to(device), device=device, config=TrainGlobalConfig, train_loader_length=len(train_loader))
fitter.fit(train_loader, val_loader)
 del train_models
gc.collect()
fake_users = [3675092,
3675220,
3675222,
3675224,
3675664,
3675671,
3676611,
3678832,
3678915,
3680489,
3686898,
3696177,
3696197,
3706230,
3706233,
3728046,
3728371,
3728378,
3728380,
3736552,
3739847,
3746563,
3751307,
3751312,
3752776,
3754293,
3754888,
3755063,
3766462,
3773321,
3773324,
3773327,
3773331,
3774072,
3774628,
3774641,
3778645,
3778835,
3780007,
3782022,
3782029,
3784627,
3786361,
3793491,
3802384,
3804591,
3804598,
3804602,
3804605,
3804610,
3804614,
3813532,
3814787,
3870028,
3870030,
3870160,
3872008,
3874367,
3878358,
3885616,
3885782,
3886152,
3886162,
3890934,
3890938,
3890942,
3890943,
3890951,
3890953,
3891400,
3919708,
3920857,
3941138,
3942591,
3950989,
3951346,
3953392,
3954152,
3956200,
3956208,
3956212,
3957622,
3957646,
3957647,
3962111,
3963078,
3964729,
3974966,
3976537,
3976549,
3976581,
3978840,
3981018,
3986769,
3990744,
3991684,
3991785,
3992349,
3992365,
3992371,
3994323,
3994685,
4001389,
4001670,
4001792,
4001969,
4003304,
4003306,
4003349,
4008952,
4009339,
4011062,
4011789,
4012423,
4012512,
4013026,
4015663,
4017735,
4018119,
4023200,
4023810,
4023849,
4025569,
4025585,
4033888,
4038711,
4039648,
4043140,
4044092,
4046075,
4046088,
4047811,
4047814,
4049130,
4049455,
4052338,
4052382,
4052416,
4052838,
4052894,
4054484,
4063978,
4065054,
4069816,
4072304,
4072318,
4074954,
4076564,
4077862,
4077896,
4080555,
4080560,
4080750,
4080758,
4080767,
4083310,
4084704,
4086831,
4089011,
4089018,
4091311,
4091608,
4098204,
4102399,
4104099,
4104546,
4105756,
4106102,
4106104,
4106265,
4108498,
4112290,
4113512,
4113992,
4114021,
4114073,
4120021,
4122335,
4122503,
4123436,
4124809,
4124830,
4126213,
4127131,
4129482,
4133128,
4133382,
4137019,
4137651,
4138422,
4139290,
4139736,
4140415,
4140866,
4140876,
4140885,
4140888,
4142722,
4142857,
4146328,
4147304,
4149421,
4149429,
4154121,
4154484,
4155526,
4157307,
4158172,
4159899,
4159911,
4161494,
4161824,
4162840,
4162863,
4162881,
4163437,
4163633,
4163827,
4165162,
4165203,
4165205,
4165211,
4165225,
4167358,
4168182,
4168420]<define_variables>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
fraud_tran_users = [3671888,
3676402,
3676410,
3677776,
3679661,
3679667,
3682263,
3685848,
3686652,
3689369,
3689599,
3697831,
3697893,
3698063,
3700660,
3700663,
3701086,
3703681,
3707336,
3708386,
3711944,
3711956,
3715012,
3729146,
3729719,
3730919,
3736667,
3742126,
3753511,
3756488,
3762286,
3764738,
3764741,
3764742,
3764746,
3765703,
3777492,
3779360,
3790961,
3791376,
3791418,
3800966,
3804339,
3804974,
3811777,
3811834,
3817323,
3817332,
3817337,
3819508,
3820639,
3825564,
3830237,
3830645,
3830648,
3833416,
3839619,
3839621,
3839628,
3840265,
3853868,
3855783,
3856122,
3856126,
3858739,
3858741,
3858913,
3864705,
3865304,
3869334,
3878178,
3878818,
3880519,
3880795,
3880938,
3880978,
3884824,
3885935,
3888408,
3891513,
3893682,
3895696,
3899183,
3909492,
3910095,
3911943,
3918744,
3918752,
3919563,
3919569,
3919574,
3919867,
3920437,
3922735,
3929343,
3936498,
3938032,
3939861,
3940245,
3941736,
3942864,
3948602,
3950394,
3950401,
3950403,
3950415,
3950418,
3950421,
3950671,
3957368,
3958894,
3960455,
3961144,
3961650,
3962873,
3962877,
3962881,
3962909,
3962956,
3962959,
3964831,
3969750,
3970268,
3970688,
3971072,
3971571,
3975220,
3980008,
3980013,
3980645,
3985298,
3985306,
3985313,
3985672,
3985677,
3985682,
3990116,
3990119,
3991837,
3994482,
3999698,
3999702,
3999703,
4002412,
4004976,
4007409,
4007428,
4007433,
4009468,
4009473,
4010733,
4011164,
4011181,
4013047,
4014514,
4015014,
4015787,
4018153,
4018173,
4022610,
4023223,
4023849,
4025540,
4025641,
4026248,
4029004,
4029005,
4030751,
4044848,
4045018,
4045803,
4050099,
4050105,
4050117,
4050130,
4050636,
4059726,
4061664,
4063724,
4067067,
4076997,
4077309,
4078114,
4080429,
4082621,
4082894,
4085298,
4089316,
4094551,
4095410,
4096324,
4097033,
4097039,
4097272,
4098072,
4098491,
4098947,
4098974,
4101056,
4101425,
4101526,
4106264,
4106703,
4108065,
4108286,
4109485,
4113667,
4114458,
4117756,
4117771,
4118261,
4120923,
4122948,
4124677,
4125805,
4126117,
4126222,
4126226,
4126254,
4126258,
4127375,
4127380,
4127395,
4127590,
4131644,
4132416,
4132885,
4133012,
4133331,
4133359,
4133411,
4134322,
4134495,
4134499,
4134509,
4134642,
4136286,
4136534,
4137182,
4138950,
4139722,
4140204,
4141069,
4142428,
4152461,
4154580,
4156319,
4158363,
4158526,
4158615,
4158777,
4159713,
4162245,
4162612,
4163621,
4164780,
4168047,
4169416]<save_to_csv>
if PL_OPT:
final_models = [
load_test_net(f'plabel_model/last-checkpoint1.bin')
 ]
else:
final_models = []
for p in path1:
final_models.append(load_test_net(p))
df = pd.read_csv(".. /input/009429/stack_median(1 ).csv")
<create_dataframe>
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images, final_models, OOF_thr)
 for i, image in enumerate(images):
assert i == 0
boxes, scores, labels = run_wbf(predictions, img_size, WBF_iou_thr, WBF_skip_thr)
 if img_size == 512:
boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 elif img_size == 1024:
boxes = boxes.astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
dff = df.copy()<feature_engineering>
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head(10 )