Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<feature_engineering>
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
cols_tmp = []
for fold in range(NUM_FOLD):
cols_tmp.append("{}_pred_fold{}".format(col_targets[-1], fold+1))
df_pred_image[col_targets[-1]] = df_pred_image[cols_tmp].values.mean(axis=1)
 df_pred_image.head()<compute_test_metric>
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )
def solve_conflict(df_pred_s, df_pred, TH_NEGATIVE=0.5, TH_INDETERMINATE = 0.5, verbose=True):
index_indeterminate = df_pred_s['indeterminate']>TH_INDETERMINATE
index_negative =(index_indeterminate==False)&(df_pred_s['negative_exam_for_pe']>TH_NEGATIVE)
 index_positive =(index_indeterminate==False)&(index_negative==False)
 index_negative_and_negative_lte_05 = index_negative &(df_pred_s['negative_exam_for_pe']<=0.5)
 df_pred_s['negative_exam_for_pe'][index_negative_and_negative_lte_05] = 0.5001
index_indeterminate_and_indeterminate_lte_05 = index_indeterminate &(df_pred_s['indeterminate']<=0.5)
 df_pred_s['indeterminate'][index_indeterminate_and_indeterminate_lte_05] = 0.5001
index_indeterminate_and_negative_gt_05 = index_indeterminate &(df_pred_s['negative_exam_for_pe']>0.5)
 df_pred_s['negative_exam_for_pe'][index_indeterminate_and_negative_gt_05] = 0.5
index_negative_and_indeterminate_gt_05 = index_negative &(df_pred_s['indeterminate']>0.5)
 df_pred_s['indeterminate'][index_negative_and_indeterminate_gt_05] = 0.5
index_positive_and_negative_gt_05 = index_positive &(df_pred_s['negative_exam_for_pe']>0.5)
 df_pred_s['negative_exam_for_pe'][index_positive_and_negative_gt_05] = 0.5
index_positive_and_indeterminate_gt_05 = index_positive &(df_pred_s['indeterminate']>0.5)
 df_pred_s['indeterminate'][index_positive_and_indeterminate_gt_05] = 0.5
index_negative_and_rv_lv_ratio_lt_1_gt_05 =(index_positive==False)&(df_pred_s['rv_lv_ratio_lt_1']>0.5)
 df_pred_s['rv_lv_ratio_lt_1'][index_negative_and_rv_lv_ratio_lt_1_gt_05] = 0.5
index_negative_and_rv_lv_ratio_gte_1_gt_05 =(index_positive==False)&(df_pred_s['rv_lv_ratio_gte_1']>0.5)
 df_pred_s['rv_lv_ratio_gte_1'][index_negative_and_rv_lv_ratio_gte_1_gt_05] = 0.5
index_negative_and_central_pe_gt_05 =(index_positive==False)&(df_pred_s['central_pe']>0.5)
 index_negative_and_rightsided_pe_gt_05 =(index_positive==False)&(df_pred_s['rightsided_pe']>0.5)
 df_pred_s['rightsided_pe'][index_negative_and_rightsided_pe_gt_05] = 0.5
index_negative_and_leftsided_pe_gt_05 =(index_positive==False)&(df_pred_s['leftsided_pe']>0.5)
 df_pred_s['leftsided_pe'][index_negative_and_leftsided_pe_gt_05] = 0.5
index_negative_and_chronic_pe_gt_05 =(index_positive==False)&(df_pred_s['chronic_pe']>0.5)
 df_pred_s['chronic_pe'][index_negative_and_chronic_pe_gt_05] = 0.5
index_negative_and_acute_and_chronic_pe_gt_05 =(index_positive==False)&(df_pred_s['acute_and_chronic_pe']>0.5)
 df_pred_s['acute_and_chronic_pe'][index_negative_and_acute_and_chronic_pe_gt_05] = 0.5
index_positive_and_rv_gte_lv = index_positive &(df_pred_s['rv_lv_ratio_lt_1']<=df_pred_s['rv_lv_ratio_gte_1'])
 index_positive_and_rv_lt_lv = index_positive &(df_pred_s['rv_lv_ratio_lt_1']>df_pred_s['rv_lv_ratio_gte_1'])
 index_positive_and_rv_gte_lv_and_rv_lv_ratio_gte_1_lte_05 =\ 
(index_positive_and_rv_gte_lv)&(df_pred_s['rv_lv_ratio_gte_1']<=0.5)
 df_pred_s['rv_lv_ratio_gte_1'][index_positive_and_rv_gte_lv_and_rv_lv_ratio_gte_1_lte_05] = 0.5001
index_positive_and_rv_gte_lv_and_rv_lv_ratio_lt_1_gt_05 =\ 
(index_positive_and_rv_gte_lv)&(df_pred_s['rv_lv_ratio_lt_1']>0.5)
 df_pred_s['rv_lv_ratio_lt_1'][index_positive_and_rv_gte_lv_and_rv_lv_ratio_lt_1_gt_05] = 0.5
index_positive_and_rv_lt_lv_and_rv_lv_ratio_lt_1_lte_05 =\ 
(index_positive_and_rv_lt_lv)&(df_pred_s['rv_lv_ratio_lt_1']<=0.5)
 df_pred_s['rv_lv_ratio_lt_1'][index_positive_and_rv_lt_lv_and_rv_lv_ratio_lt_1_lte_05] = 0.5001
index_positive_and_rv_lt_lv_and_rv_lv_ratio_gte_1_gt_05 =\ 
(index_positive_and_rv_lt_lv)&(df_pred_s['rv_lv_ratio_gte_1']>0.5)
 df_pred_s['rv_lv_ratio_gte_1'][index_positive_and_rv_lt_lv_and_rv_lv_ratio_gte_1_gt_05] = 0.5
index_positive_and_central_is_greatest = index_positive &(df_pred_s['central_pe']>=df_pred_s['rightsided_pe'])&(df_pred_s['central_pe']>=df_pred_s['leftsided_pe'])
 index_positive_and_right_is_greatest = index_positive &(index_positive_and_central_is_greatest==False)&(df_pred_s['rightsided_pe']>=df_pred_s['leftsided_pe'])
 index_positive_and_left_is_greatest = index_positive &(index_positive_and_central_is_greatest==False)&(index_positive_and_right_is_greatest==False)
 index_positive_and_central_is_greatest_and_central_pe_lte_05 =(index_positive_and_central_is_greatest)&(df_pred_s['central_pe']<=0.5)
 df_pred_s['central_pe'][index_positive_and_central_is_greatest_and_central_pe_lte_05] = 0.5001
index_positive_and_right_is_greatest_and_rightsided_pe_lte_05 =(index_positive_and_right_is_greatest)&(df_pred_s['rightsided_pe']<=0.5)
 df_pred_s['rightsided_pe'][index_positive_and_right_is_greatest_and_rightsided_pe_lte_05] = 0.5001
index_positive_and_left_is_greatest_and_leftsided_pe_lte_05 =(index_positive_and_left_is_greatest)&(df_pred_s['leftsided_pe']<=0.5)
 df_pred_s['leftsided_pe'][index_positive_and_left_is_greatest_and_leftsided_pe_lte_05] = 0.5001
index_double_positive = index_positive &(df_pred_s['chronic_pe']>0.5)&(df_pred_s['acute_and_chronic_pe']>0.5)
 index_double_positive_and_chronic_lte_acute_and_chronic = index_double_positive &(df_pred_s['chronic_pe']<=df_pred_s['acute_and_chronic_pe'])
 df_pred_s['chronic_pe'][index_double_positive_and_chronic_lte_acute_and_chronic] = 0.5
index_double_positive_and_chronic_gt_acute_and_chronic = index_double_positive &(df_pred_s['chronic_pe']>df_pred_s['acute_and_chronic_pe'])
 df_pred_s['acute_and_chronic_pe'][index_double_positive_and_chronic_gt_acute_and_chronic] = 0.5
df_pred_s['positive'] = 0
df_pred_s['positive'][index_positive] = 1
df_pred2 = pd.merge(df_pred, df_pred_s[[col_groupby, 'positive']], on=col_groupby, how='left')
 df_agg = df_pred.groupby(col_groupby)['pe_present_on_image'].agg('max' ).reset_index()
df_agg.columns = [col_groupby, 'pe_present_on_image_pred_max']
df_pred2 = pd.merge(df_pred2, df_agg, on=col_groupby, how='left')
 df_pred2['peak'] = df_pred2['pe_present_on_image']==df_pred2['pe_present_on_image_pred_max']
index_positive_i = df_pred2['positive']==1
index_negative_and_pe_present_on_image_gt_05_i =(index_positive_i==False)&(df_pred2['pe_present_on_image']>0.5)
 df_pred['pe_present_on_image'][index_negative_and_pe_present_on_image_gt_05_i] = 0.5
index_positive_and_peak_and_pe_present_on_image_lte_05_i = index_positive_i &(df_pred2['peak'])&(df_pred2['pe_present_on_image']<=0.5)
 df_pred['pe_present_on_image'][index_positive_and_peak_and_pe_present_on_image_lte_05_i] = 0.5001
if verbose:
print("num study", len(df_pred_s))
print("num image", len(df_pred))
print("split to 3 classes")
 print(" num predicted_as_negative:", index_negative.sum())
 print(" num predicted_as_indeterminate:", index_indeterminate.sum())
 print(" num predicted_as_positive:", index_positive.sum())
 print("process 3 class conflict")
 print(" num predicted_as_negative and negative<=0.5:", index_negative_and_negative_lte_05.sum())
 print(" num predicted_as_indeterminate and indeterminate<=0.5:", index_indeterminate_and_indeterminate_lte_05.sum())
 print(" num predicted_as_indeterminate and negative_exam_for_pe>0.5:", index_indeterminate_and_negative_gt_05.sum())
 print(" num predicted_as_negative and indeterminate>0.5:", index_negative_and_indeterminate_gt_05.sum())
 print(" num predicted_as_positive and negative_exam_for_pe>0.5:", index_positive_and_negative_gt_05.sum())
 print(" num predicted_as_positive and indeterminate>0.5:", index_positive_and_indeterminate_gt_05.sum())
 print("process negative case")
 print(" num predicted_as_not_positive and rv_lv_ratio_lt_1>0.5:", index_negative_and_rv_lv_ratio_lt_1_gt_05.sum())
 print(" num predicted_as_not_positive and rv_lv_ratio_gte_1>0.5:", index_negative_and_rv_lv_ratio_gte_1_gt_05.sum())
 print(" num predicted_as_not_positive and central_pe>0.5:", index_negative_and_central_pe_gt_05.sum())
 print(" num predicted_as_not_positive and central_pe>0.5:", index_negative_and_rightsided_pe_gt_05.sum())
 print(" num predicted_as_not_positive and leftsided_pe>0.5:", index_negative_and_leftsided_pe_gt_05.sum())
 print(" num predicted_as_not_positive and chronic_pe>0.5:", index_negative_and_chronic_pe_gt_05.sum())
 print(" num predicted_as_not_positive and acute_and_chronic_pe>0.5:", index_negative_and_acute_and_chronic_pe_gt_05.sum())
 print("process positive case")
 print(" num predicted_as_positive and rv_lv_ratio_lt_1<=rv_lv_ratio_gte_1:", index_positive_and_rv_gte_lv.sum())
 print(" num predicted_as_positive and rv_lv_ratio_lt_1>rv_lv_ratio_gte_1:", index_positive_and_rv_lt_lv.sum())
 print(" num predicted_as_positive and(rv_lv_ratio_lt_1<=rv_lv_ratio_gte_1)and(rv_lv_ratio_gte_1<=0.5): ",
index_positive_and_rv_gte_lv_and_rv_lv_ratio_gte_1_lte_05.sum())
 print(" num predicted_as_positive and(rv_lv_ratio_lt_1<=rv_lv_ratio_gte_1)and rv_lv_ratio_lt_1>0.5: ",
index_positive_and_rv_gte_lv_and_rv_lv_ratio_lt_1_gt_05.sum())
 print(" num predicted_as_positive and(rv_lv_ratio_lt_1>rv_lv_ratio_gte_1)and rv_lv_ratio_lt_1<=0.5: ",
index_positive_and_rv_lt_lv_and_rv_lv_ratio_lt_1_lte_05.sum())
 print(" num predicted_as_positive and(rv_lv_ratio_lt_1>rv_lv_ratio_gte_1)and rv_lv_ratio_gte_1>0.5: ",
index_positive_and_rv_lt_lv_and_rv_lv_ratio_gte_1_gt_05.sum())
 print(" num predicted_as_positive and central is greatest:", index_positive_and_central_is_greatest.sum())
 print(" num predicted_as_positive and right is greatest:", index_positive_and_right_is_greatest.sum())
 print(" num predicted_as_positive and left is greatest:", index_positive_and_left_is_greatest.sum())
 print(" num predicted_as_positive and central is greatest and central_pe<=0.5:", index_positive_and_central_is_greatest_and_central_pe_lte_05.sum())
 print(" num predicted_as_positive and right is greatest and rightsided_pe<=0.5:", index_positive_and_right_is_greatest_and_rightsided_pe_lte_05.sum())
 print(" num predicted_as_positive and left is greatest and leftsided_pe<=0.5:", index_positive_and_left_is_greatest_and_leftsided_pe_lte_05.sum())
 print(" num both chronic_pe and acute_and_chronic_pe is positive:", index_double_positive.sum())
 print(" num both chronic_pe and acute_and_chronic_pe is positive and chronic<=acute_and_chronic:", index_double_positive_and_chronic_lte_acute_and_chronic.sum())
 print(" num both chronic_pe and acute_and_chronic_pe is positive and chronic>acute_and_chronic:", index_double_positive_and_chronic_gt_acute_and_chronic.sum())
 print("process image level")
 print(" num img of predicted_as_positive:", index_positive_i.sum())
 print(" num img of predicted_as_negative:",(index_positive_i==0 ).sum())
 print(" num img of peak:", df_pred2['peak'].sum())
 print(" num img of predicted_as_negative and pe_present_on_image>0.5:", index_negative_and_pe_present_on_image_gt_05_i.sum())
 print(" num img of predicted_as_positive and peak and pe_present_on_image<=0.5:", index_positive_and_peak_and_pe_present_on_image_lte_05_i.sum())
 return df_pred_s, df_pred 
<merge>
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True)
 marking.sample(10 )
df_pred_study_const, df_pred_image_const = solve_conflict(df_pred_study, df_pred_image)
 df_pred_study_const.head()<concatenate>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
 df_folds = marking[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str)
)
 df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
df_folds.sample(10 )
df_sub_pred = copy.deepcopy(df_pred_image_const[[col_index, col_targets[-1]]])
 df_sub_pred.columns = ['id', 'label']
for i, col in enumerate(col_targets[:-1]):
df_tmp = df_pred_study_const[[col_groupby, col]]
df_tmp.columns = ['id', 'label']
df_tmp['id'] = df_tmp['id'] + '_{}'.format(col)
 df_sub_pred = pd.concat([df_sub_pred, df_tmp])
 df_sub_pred = df_sub_pred.reset_index(drop=True)
 print(df_sub_pred.shape)
 df_sub_pred.head()<load_from_csv>
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
class WheatDataset(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize = 1024):
w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes
df_sub = pd.read_csv(df_sub_path)
 print(df_sub.shape)
 df_sub.head()<merge>
fold_number = 0
train_dataset = WheatDataset(
 image_ids=df_folds[df_folds['fold'] != fold_number].index.values,
marking=marking,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = WheatDataset(
 image_ids=df_folds[df_folds['fold'] == fold_number].index.values,
marking=marking,
transforms=get_valid_transforms() ,
test=True,
)
df_sub = pd.merge(df_sub[['id']], df_sub_pred, on='id', how='left')
 print(df_sub.shape)
 df_sub.head(30 )<feature_engineering>
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
mean_targets = [
0.674681,
0.021569,
0.040115,
0.019920,
0.055090,
0.212117,
0.257590,
0.129139,
0.174612,
0.289885,
]
df_sub_mean = copy.deepcopy(df_test_full[[col_index]])
 df_sub_mean.columns = ['id']
df_sub_mean['label'] = mean_targets[-1]
for i, col in enumerate(col_targets[:-1]):
df_tmp = df_test_full_study[[col_groupby]]
df_tmp.columns = ['id']
df_tmp['label'] = mean_targets[i]
df_tmp['id'] = df_tmp['id'] + '_{}'.format(col)
 df_sub_mean = pd.concat([df_sub_mean, df_tmp])
 df_sub_mean = df_sub_mean.reset_index(drop=True)
 print(df_sub_mean.shape )<merge>
class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
summary_loss = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint.bin')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
self.optimizer.zero_grad()
loss, _, _ = self.model(images, boxes, labels)
 loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )
df_sub['label'][pd.isna(df_sub['label'])] = pd.merge(df_sub[['id']], df_sub_mean, on='id', how='left')['label'][pd.isna(df_sub['label'])]
df_sub.head(20 )<merge>
class TrainGlobalConfig:
num_workers = 2
batch_size = 4
n_epochs = 3
lr = 0.0002
folder = 'effdet5-cutmix-augmix'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)
def check_consistency2(df_exam, df_image, test):

 df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')
 ids = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']
labels = [c for c in df.columns if c not in ids]
df = df[ids + labels]
df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('Found', len(errors), 'inconsistent predictions')
 return errors<compute_test_metric>
def collate_fn(batch):
return tuple(zip(*batch))
error = check_consistency2(df_pred_study_const, df_pred_image_const, df_test )<save_to_csv>
def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, val_loader )
if len(error)==0:
df_sub.to_csv('submission.csv', index=None)
 else:
print("error!" )<feature_engineering>
def get_net() :
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 checkpoint = torch.load('.. /input/efficientdet/efficientdet_d5-ef44aea8.pth')
 net.load_state_dict(checkpoint)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
return DetBenchTrain(net, config)
 net = get_net()
3989*3/3600*2<install_modules>
run_training()
package_path = '.. /input/efficientnet-pytorch-07/efficientnet_pytorch-0.7.0'
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2<define_variables>
def get_test_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )
CFG = {
'train': False,
'train_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/train',
'test_img_path': '.. /input/rsna-str-pulmonary-embolism-detection/test',
'cv_fold_path': '.. /input/stratified-validation-strategy/rsna_train_splits_fold_20.csv',
'train_path': '.. /input/rsna-str-pulmonary-embolism-detection/train.csv',
'test_path': '.. /input/rsna-str-pulmonary-embolism-detection/test.csv',
'image_target_cols': [
'pe_present_on_image',
],
'exam_target_cols': [
'negative_exam_for_pe',
'rv_lv_ratio_gte_1',
'rv_lv_ratio_lt_1',
'leftsided_pe',
'chronic_pe',
'rightsided_pe',
'acute_and_chronic_pe',
'central_pe',
'indeterminate'
],
'img_num': 200,
'img_size': 256,
'lr': 0.0005,
'epochs': 2,
'device': 'cuda',
'train_bs': 2,
'accum_iter': 8,
'verbose_step': 1,
'num_workers': 4,
'efbnet': 'efficientnet-b0',
'train_folds': [np.arange(0,16),
np.concatenate([np.arange(0,12), np.arange(16,20)]),
np.concatenate([np.arange(0,8), np.arange(12,20)]),
np.concatenate([np.arange(0,4), np.arange(8,20)]),
np.arange(4,20),
],
'valid_folds': [np.arange(16,20),
np.arange(12,16),
np.arange(8,12),
np.arange(4,8),
np.arange(0,4)
 ],
'model_path': '.. /input/kh-rsna-model',
'tag': 'efb0_stage2_multilabel'
}
SEED = 42321
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
def window(img, WL=50, WW=350):
upper, lower = WL+WW//2, WL-WW//2
X = np.clip(img.copy() , lower, upper)
 X = X - np.min(X)
 X = X / np.max(X)
 return X
def get_img(path):
d = pydicom.read_file(path)


 img =(d.pixel_array * d.RescaleSlope)+ d.RescaleIntercept
r = window(img, -600, 1500)
 g = window(img, 100, 700)
 b = window(img, 40, 400)
 res = np.concatenate([r[:, :, np.newaxis],
g[:, :, np.newaxis],
b[:, :, np.newaxis]], axis=-1)
 res = zoom(res, [CFG['img_size']/res.shape[0], CFG['img_size']/res.shape[1], 1.], prefilter=False, order=1)
 return res
class RSNADatasetStage1(Dataset):
def __init__(
 self, df, label_smoothing, data_root,
image_subsampling=True, transforms=None, output_label=True 
):
super().__init__()
self.df = df.reset_index(drop=True ).copy()
self.label_smoothing = label_smoothing
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index: int):
if self.output_label:
target = self.df[CFG['image_target_cols']].values[index]
path = "{}/{}/{}/{}.dcm".format(self.data_root,
self.df.iloc[index]['StudyInstanceUID'],
self.df.iloc[index]['SeriesInstanceUID'],
self.df.iloc[index]['SOPInstanceUID'])
 img = get_img(path)
 if self.transforms:
img = self.transforms(image=img)['image']
if self.output_label == True:
target = np.clip(target, self.label_smoothing, 1 - self.label_smoothing)
 return img, target
else:
return img
class RSNADataset(Dataset):
def __init__(
 self, df, label_smoothing, data_root,
image_subsampling=True, transforms=None, output_label=True 
):
super().__init__()
self.df = df
self.patients = self.df['StudyInstanceUID'].unique()
self.image_subsampling = image_subsampling
self.label_smoothing = label_smoothing
self.transforms = transforms
self.data_root = data_root
self.output_label = output_label
def get_patients(self):
return self.patients
def __len__(self):
return len(self.patients)
 def __getitem__(self, index: int):
patient = self.patients[index]
df_ = self.df.loc[self.df.StudyInstanceUID == patient]
per_image_feats = get_stage1_columns()
if self.image_subsampling:
img_num = min(CFG['img_num'], df_.shape[0])
 img_ix = np.random.choice(np.arange(df_.shape[0]), replace=False, size=img_num)
 imgs = np.zeros(( CFG['img_num'],), np.float32)
 per_image_preds = np.zeros(( CFG['img_num'], len(per_image_feats)) , np.float32)
 locs = np.zeros(( CFG['img_num'],), np.float32)
 image_masks = np.zeros(( CFG['img_num'],), np.float32)
 image_masks[:img_num] = 1.
 if self.output_label:
exam_label = df_[CFG['exam_target_cols']].values[0]
image_labels = np.zeros(( CFG['img_num'], len(CFG['image_target_cols'])) , np.float32)
 else:
img_num = df_.shape[0]
img_ix = np.arange(df_.shape[0])
 imgs = np.zeros(( img_num,), np.float32)
 per_image_preds = np.zeros(( img_num, len(per_image_feats)) , np.float32)
 locs = np.zeros(( img_num,), np.float32)
 image_masks = np.zeros(( img_num,), np.float32)
 image_masks[:img_num] = 1.
 if self.output_label:
exam_label = df_[CFG['exam_target_cols']].values[0]
image_labels = np.zeros(( img_num, len(CFG['image_target_cols'])) , np.float32)
 for i, im_ix in enumerate(img_ix):
path = "{}/{}/{}/{}.dcm".format(self.data_root,
df_['StudyInstanceUID'].values[im_ix],
df_['SeriesInstanceUID'].values[im_ix],
df_['SOPInstanceUID'].values[im_ix])
 d = pydicom.read_file(path)
 locs[i] = d.ImagePositionPatient[2]
per_image_preds[i,:] = df_[per_image_feats].values[im_ix,:]
if self.output_label == True:
image_labels[i] = df_[CFG['image_target_cols']].values[im_ix]
seq_ix = np.argsort(locs)

locs = locs[seq_ix]
locs[1:img_num] = locs[1:img_num]-locs[0:img_num-1]
locs[0] = 0
per_image_preds = per_image_preds[seq_ix]
if self.output_label == True:
image_labels = image_labels[seq_ix]
image_labels = np.clip(image_labels, self.label_smoothing, 1 - self.label_smoothing)
 exam_label = np.clip(exam_label, self.label_smoothing, 1 - self.label_smoothing)
 return imgs, per_image_preds, locs, image_labels, exam_label, image_masks
else:
return imgs, per_image_preds, locs, img_num, index, seq_ix
HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90,
Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,
IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine,
IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout 
)
 def get_train_transforms() :
return Compose([
ToTensorV2(p=1.0),
], p=1.)

 def get_valid_transforms() :
return Compose([
ToTensorV2(p=1.0),
], p=1.)

 class RNSAImageFeatureExtractor(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = EfficientNet.from_name(CFG['efbnet'])
 self.pooling = nn.AdaptiveAvgPool2d(1)
 def get_dim(self):
return self.cnn_model._fc.in_features
def forward(self, x):
feats = self.cnn_model.extract_features(x)
 return self.pooling(feats ).view(x.shape[0], -1)
 class RSNAImgClassifierSingle(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = RNSAImageFeatureExtractor()
self.image_predictors = nn.Linear(self.cnn_model.get_dim() , 1)
 def forward(self, imgs):
imgs_embdes = self.cnn_model(imgs)
 image_preds = self.image_predictors(imgs_embdes)
 return image_preds
class RSNAImgClassifier(nn.Module):
def __init__(self):
super().__init__()
self.cnn_model = RNSAImageFeatureExtractor()
self.image_predictors = nn.Linear(self.cnn_model.get_dim() , 9)
 def forward(self, imgs):
imgs_embdes = self.cnn_model(imgs)
 image_preds = self.image_predictors(imgs_embdes)
 return image_preds
class TimeDistributed(nn.Module):
def __init__(self, module, batch_first=True):
super(TimeDistributed, self ).__init__()
self.module = module
self.batch_first = batch_first
def forward(self, x):

 x_size= x.size()
c_in = x.contiguous().view(x_size[0] * x_size[1], *x_size[2:])
 c_out = self.module(c_in)
 r_in = c_out.view(x_size[0], x_size[1], -1)
 if self.batch_first is False:
r_in = r_in.permute(1, 0, 2)
 return r_in
class RSNAClassifier(nn.Module):
def __init__(self, hidden_size=64):
super().__init__()
self.gru = nn.GRU(len(get_stage1_columns())+1, hidden_size, bidirectional=True, batch_first=True, num_layers=2)
 self.image_predictors = TimeDistributed(nn.Linear(hidden_size*2, 1))
self.exam_predictor = nn.Linear(hidden_size*2*2, 9)
 def forward(self, img_preds, locs):
embeds = torch.cat([img_preds, locs.view(locs.shape[0], locs.shape[1], 1)], dim=2)
 embeds, _ = self.gru(embeds)
 image_preds = self.image_predictors(embeds)
 avg_pool = torch.mean(embeds, 1)
 max_pool, _ = torch.max(embeds, 1)
 conc = torch.cat([avg_pool, max_pool], 1)
 exam_pred = self.exam_predictor(conc)
 return image_preds, exam_pred
def rsna_wloss_inference(y_true_img, y_true_exam, y_pred_img, y_pred_exam, chunk_sizes):

 y_true_img, y_true_exam, y_pred_img, y_pred_exam = torch.tensor(y_true_img, dtype=torch.float32), torch.tensor(y_true_exam, dtype=torch.float32), torch.tensor(y_pred_img, dtype=torch.float32), torch.tensor(y_pred_exam, dtype=torch.float32)
 y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks = torch.split(y_true_img, chunk_sizes, dim=0), torch.split(y_true_exam, chunk_sizes, dim=0), torch.split(y_pred_img, chunk_sizes, dim=0), torch.split(y_pred_exam, chunk_sizes, dim=0)
 label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988] ).view(1, -1)
 img_w = 0.07361963
bce_func = torch.nn.BCELoss(reduction='none')
 total_loss = torch.tensor(0, dtype=torch.float32)
 total_weights = torch.tensor(0, dtype=torch.float32)
 for i,(y_true_img_, y_true_exam_, y_pred_img_, y_pred_exam_)in enumerate(zip(y_true_img_chunks, y_true_exam_chunks, y_pred_img_chunks, y_pred_exam_chunks)) :
exam_loss = bce_func(y_pred_exam_[0, :], y_true_exam_[0, :])
 exam_loss = torch.sum(exam_loss*label_w, 1)[0]
image_loss = bce_func(y_pred_img_, y_true_img_)
 img_num = chunk_sizes[i]
qi = torch.sum(y_true_img_)/img_num
image_loss = torch.sum(img_w*qi*image_loss)
 total_loss += exam_loss+image_loss
total_weights += label_w.sum() + img_w*qi*img_num
final_loss = total_loss/total_weights
return final_loss
def rsna_wloss_train(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):
label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988] ).view(1, -1 ).to(device)
 img_w = 0.07361963
bce_func = torch.nn.BCEWithLogitsLoss(reduction='none' ).to(device)
 total_loss = torch.tensor(0, dtype=torch.float32 ).to(device)
 total_weights = torch.tensor(0, dtype=torch.float32 ).to(device)
 for i in range(y_true_img.shape[0]):
exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])
 exam_loss = torch.sum(exam_loss*label_w, 1)[0]
img_mask = image_masks[i]
image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:] ).flatten()
image_loss = image_loss*img_mask
img_num = torch.sum(img_mask)
 qi = torch.sum(y_true_img[i,:])/img_num
image_loss = torch.sum(img_w*qi*image_loss)
 total_loss += exam_loss+image_loss
total_weights += label_w.sum() + img_w*qi*img_num
final_loss = total_loss/total_weights
return final_loss, total_loss, total_weights
def rsna_wloss_valid(y_true_img, y_true_exam, y_pred_img, y_pred_exam, image_masks, device):
label_w = torch.tensor([0.0736196319, 0.2346625767, 0.0782208589, 0.06257668712, 0.1042944785, 0.06257668712, 0.1042944785, 0.1877300613, 0.09202453988] ).view(1, -1 ).to(device)
 img_w = 0.07361963
bce_func = torch.nn.BCEWithLogitsLoss(reduction='none' ).to(device)
 total_loss = torch.tensor(0, dtype=torch.float32 ).to(device)
 total_weights = torch.tensor(0, dtype=torch.float32 ).to(device)
 for i in range(y_true_img.shape[0]):
exam_loss = bce_func(y_pred_exam[i, :], y_true_exam[i, :])
 exam_loss = torch.sum(exam_loss*label_w, 1)[0]
img_mask = image_masks[i]
image_loss = bce_func(y_pred_img[i,:], y_true_img[i,:] ).flatten()
image_loss = image_loss*img_mask
img_num = torch.sum(img_mask)
 qi = torch.sum(y_true_img[i,:])/img_num
image_loss = torch.sum(img_w*qi*image_loss)
 total_loss += exam_loss+image_loss
total_weights += label_w.sum() + img_w*qi*img_num
final_loss = total_loss/total_weights
return final_loss, total_loss, total_weights
def prepare_train_dataloader(train, cv_df, train_fold, valid_fold):
train_patients = cv_df.loc[cv_df.fold.isin(train_fold), 'StudyInstanceUID'].unique()
valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
train_ = train.loc[train.StudyInstanceUID.isin(train_patients),:].reset_index(drop=True)
 valid_ = train.loc[train.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)
 train_ds = RSNADataset(train_, 0.0, CFG['train_img_path'], image_subsampling=True, transforms=get_train_transforms() , output_label=True)
 valid_ds = RSNADataset(valid_, 0.0, CFG['train_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=True)
 train_loader = torch.utils.data.DataLoader(
 train_ds,
batch_size=CFG['train_bs'],
pin_memory=True,
drop_last=False,
shuffle=True,
num_workers=CFG['num_workers'], 
)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=1,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=True, 
)
 return train_loader, val_loader
def train_one_epoch(epoch, model, device, scaler, optimizer, train_loader):
model.train()
t = time.time()
loss_sum = 0
loss_w_sum = 0
for step,(imgs, per_image_preds, locs, image_labels, exam_label, image_masks)in enumerate(train_loader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
image_masks = image_masks.to(device ).float()
image_labels = image_labels.to(device ).float()
exam_label = exam_label.to(device ).float()
with autocast() :
image_preds, exam_pred = model(per_image_preds, locs)
 loss, total_loss, total_weights = rsna_wloss_train(image_labels, exam_label, image_preds, exam_pred, image_masks, device)
 scaler.scale(loss ).backward()
loss_sum += total_loss.detach().item()
loss_w_sum += total_weights.detach().item()
if(( step + 1)% CFG['accum_iter'] == 0)or(( step + 1)== len(train_loader)) :
scaler.step(optimizer)
 scaler.update()
optimizer.zero_grad()
if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(train_loader)) :
print(
 f'epoch {epoch} train step {step+1}/{len(train_loader)}, ' + \
f'loss: {loss_sum/loss_w_sum:.4f}, ' + \
f'time: {(time.time() - t):.4f}', end= '\r' if(step + 1)!= len(train_loader)else '
' 
)
 def post_process(exam_pred, image_pred):
rv_lv_ratio_lt_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_lt_1')
 rv_lv_ratio_gte_1_ix = CFG['exam_target_cols'].index('rv_lv_ratio_gte_1')
 central_pe_ix = CFG['exam_target_cols'].index('central_pe')
 rightsided_pe_ix = CFG['exam_target_cols'].index('rightsided_pe')
 leftsided_pe_ix = CFG['exam_target_cols'].index('leftsided_pe')
 acute_and_chronic_pe_ix = CFG['exam_target_cols'].index('acute_and_chronic_pe')
 chronic_pe_ix = CFG['exam_target_cols'].index('chronic_pe')
 negative_exam_for_pe_ix = CFG['exam_target_cols'].index('negative_exam_for_pe')
 indeterminate_ix = CFG['exam_target_cols'].index('indeterminate')
 has_pe_image = torch.max(image_pred, 1)[0][0] > 0
rv_lv_ratios = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]]
rv_lv_ratios_1_a = nn.functional.softmax(rv_lv_ratios, dim=1)
 rv_lv_ratios_1_a = torch.log(rv_lv_ratios_1_a/(1-rv_lv_ratios_1_a))
exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix]] = torch.where(has_pe_image, rv_lv_ratios_1_a, rv_lv_ratios)
 crl_pe = exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]]
has_no_pe = torch.max(crl_pe ,1)[0] <= 0
max_val = torch.max(crl_pe, 1)[0]
crl_pe_1_b = torch.where(crl_pe==max_val, 0.0001-crl_pe+crl_pe, crl_pe)
 exam_pred[:, [central_pe_ix, rightsided_pe_ix, leftsided_pe_ix]] = torch.where(has_pe_image*has_no_pe, crl_pe_1_b, crl_pe)
 ac_pe = exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]]
both_ac_ch = torch.min(ac_pe ,1)[0] > 0
ac_pe_1_c = nn.functional.softmax(ac_pe, dim=1)
 ac_pe_1_c = torch.log(ac_pe_1_c/(1-ac_pe_1_c))
exam_pred[:, [acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(has_pe_image*both_ac_ch, ac_pe_1_c, ac_pe)
 neg_ind = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]
neg_ind_1d = torch.clamp(neg_ind, max=0)
 exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(has_pe_image, neg_ind_1d, neg_ind)
 ne_inde = exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]]
ne_inde_2_a = nn.functional.softmax(ne_inde, dim=1)
 ne_inde_2_a = torch.log(ne_inde_2_a/(1-ne_inde_2_a))
exam_pred[:, [negative_exam_for_pe_ix, indeterminate_ix]] = torch.where(~has_pe_image, ne_inde_2_a, ne_inde)
 all_other_exam_labels = exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,
central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,
acute_and_chronic_pe_ix, chronic_pe_ix]]
all_other_exam_labels_2_b = torch.clamp(all_other_exam_labels, max=0)
 exam_pred[:, [rv_lv_ratio_lt_1_ix, rv_lv_ratio_gte_1_ix,
central_pe_ix, rightsided_pe_ix, leftsided_pe_ix,
acute_and_chronic_pe_ix, chronic_pe_ix]] = torch.where(~has_pe_image, all_other_exam_labels_2_b, all_other_exam_labels)
 return exam_pred, image_pred
def valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=False):
model.eval()
t = time.time()
loss_sum = 0
loss_w_sum = 0
for step,(imgs, per_image_preds, locs, image_labels, exam_label, image_masks)in enumerate(val_loader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
image_masks = image_masks.to(device ).float()
image_labels = image_labels.to(device ).float()
exam_label = exam_label.to(device ).float()
image_preds, exam_pred = model(per_image_preds, locs)
 exam_pred, image_preds= post_process(exam_pred, image_preds)
 loss, total_loss, total_weights = rsna_wloss_valid(image_labels, exam_label, image_preds, exam_pred, image_masks, device)
 loss_sum += total_loss.detach().item()
loss_w_sum += total_weights.detach().item()
if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(val_loader)) :
print(
 f'epoch {epoch} valid Step {step+1}/{len(val_loader)}, ' + \
f'loss: {loss_sum/loss_w_sum:.4f}, ' + \
f'time: {(time.time() - t):.4f}', end='\r' if(step + 1)!= len(val_loader)else '
' 
)
 if schd_loss_update:
scheduler.step(loss_sum/loss_w_sum)
 else:
scheduler.step()
def check_label_consistency(checking_df):
df = checking_df.copy()
print(df.shape)
 df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('label in-consistency counts:', errors.shape)
 if errors.shape[0] > 0:
print(errors.broken_rule.value_counts())
 print(errors)
 assert False
def inference(model, device, df, root_path):
model.eval()
t = time.time()
ds = RSNADataset(df, 0.0, root_path, image_subsampling=False, transforms=get_valid_transforms() , output_label=False)
 dataloader = torch.utils.data.DataLoader(
 ds,
batch_size=1,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=True, 
)
 patients = ds.get_patients()
res_dfs = []
for step,(imgs, per_image_preds, locs, img_num, index, seq_ix)in enumerate(dataloader):
imgs = imgs.to(device ).float()
per_image_preds = per_image_preds.to(device ).float()
locs = locs.to(device ).float()
index = index.detach().numpy() [0]
seq_ix = seq_ix.detach().numpy() [0,:]
patient_filt =(df.StudyInstanceUID == patients[index])
 patient_df = pd.DataFrame()
patient_df['SOPInstanceUID'] = df.loc[patient_filt, 'SOPInstanceUID'].values[seq_ix]
patient_df['SeriesInstanceUID'] = df.loc[patient_filt, 'SeriesInstanceUID'].values
patient_df['StudyInstanceUID'] = patients[index]
for c in CFG['image_target_cols']+CFG['exam_target_cols']:
patient_df[c] = 0.0
image_preds, exam_pred = model(per_image_preds, locs)
 exam_pred, image_preds = post_process(exam_pred, image_preds)
 exam_pred = torch.sigmoid(exam_pred ).cpu().detach().numpy()
image_preds = torch.sigmoid(image_preds ).cpu().detach().numpy()
patient_df[CFG['exam_target_cols']] = exam_pred[0]
patient_df[CFG['image_target_cols']] = image_preds[0,:]
res_dfs += [patient_df]


if(( step + 1)% CFG['verbose_step'] == 0)or(( step + 1)== len(dataloader)) :
print(
 f'Inference Step {step+1}/{len(dataloader)}, ' + \
f'time: {(time.time() - t):.4f}', end='\r' if(step + 1)!= len(dataloader)else '
' 
)
 res_dfs = pd.concat(res_dfs, axis=0 ).reset_index(drop=True)
 res_dfs = df[['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID']].merge(res_dfs, on=['SOPInstanceUID', 'SeriesInstanceUID', 'StudyInstanceUID'], how='left')
 print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].head(5))
print(res_dfs[CFG['image_target_cols']+CFG['exam_target_cols']].tail(5))
assert res_dfs.shape[0] == df.shape[0]
check_label_consistency(res_dfs)
 return res_dfs
STAGE1_CFGS = [
{
'tag': 'efb0_stage1',
'model_constructor': RSNAImgClassifierSingle,
'dataset_constructor': RSNADatasetStage1,
'output_len': 1
},
{
'tag': 'efb0_stage1_multilabel',
'model_constructor': RSNAImgClassifier,
'dataset_constructor': RSNADatasetStage1,
'output_len': 9
},
]
STAGE1_CFGS_TAG = 'efb0-stage1-single-multi-label'
def get_stage1_columns() :
new_feats = []
for cfg in STAGE1_CFGS:
for i in range(cfg['output_len']):
f = cfg['tag']+'_'+str(i)
 new_feats += [f]
return new_feats
def update_stage1_oof_preds(df, cv_df):
res_file_name = STAGE1_CFGS_TAG+"-train.csv"
new_feats = get_stage1_columns()
for f in new_feats:
df[f] = 0
if os.path.isfile(res_file_name):
df = pd.read_csv(res_file_name)
 print('img acc:',(( df[new_feats[0]]>0)==df[CFG['image_target_cols'][0]] ).mean())
 return df
for fold,(train_fold, valid_fold)in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])) :
if fold < 0:
continue
valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
filt = df.StudyInstanceUID.isin(valid_patients)
 valid_ = df.loc[filt,:].reset_index(drop=True)
 image_preds_all_list = []
for cfg in STAGE1_CFGS:
valid_ds = cfg['dataset_constructor'](valid_, 0.0, CFG['train_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=True)
 val_loader = torch.utils.data.DataLoader(
 valid_ds,
batch_size=256,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False,
sampler=SequentialSampler(valid_ds)
)
 device = torch.device(CFG['device'])
 model = cfg['model_constructor']().to(device)
 model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, cfg['tag'])))
 model.eval()
image_preds_all = []
correct_count = 0
count = 0
for step,(imgs, target)in enumerate(val_loader):
imgs = imgs.to(device ).float()
target = target.to(device ).float()
image_preds = model(imgs)
 if len(image_preds.shape)== 1:
image_preds = image_preds.view(-1, 1)
 correct_count +=(( image_preds[:,0]>0)== target[:,0] ).sum().detach().item()
count += imgs.shape[0]
image_preds_all += [image_preds.cpu().detach().numpy() ]
print('acc: {:.4f}, {}, {}, {}/{}'.format(correct_count/count, correct_count, count, step+1, len(val_loader)) , end='\r')
 print()
image_preds_all = np.concatenate(image_preds_all, axis=0)
 image_preds_all_list += [image_preds_all]
del model, val_loader
torch.cuda.empty_cache()
image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)
 df.loc[filt, new_feats] = image_preds_all_list
df.to_csv(res_file_name, index=False)
 return df
def update_stage1_test_preds(df):
new_feats = get_stage1_columns()
for f in new_feats:
df[f] = 0
image_preds_all_list = []
for cfg in STAGE1_CFGS:
test_ds = cfg['dataset_constructor'](df, 0.0, CFG['test_img_path'], image_subsampling=False, transforms=get_valid_transforms() , output_label=False)
 test_loader = torch.utils.data.DataLoader(
 test_ds,
batch_size=256,
num_workers=CFG['num_workers'],
shuffle=False,
pin_memory=False,
sampler=SequentialSampler(test_ds)
)
 device = torch.device(CFG['device'])
 model = cfg['model_constructor']().to(device)
 model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], cfg['tag'])))
 model.eval()
image_preds_all = []
for step, imgs in enumerate(tqdm(test_loader)) :
imgs = imgs.to(device ).float()
image_preds = model(imgs)
 image_preds_all += [image_preds.cpu().detach().numpy() ]
image_preds_all = np.concatenate(image_preds_all, axis=0)
 image_preds_all_list += [image_preds_all]
del model, test_loader
torch.cuda.empty_cache()
image_preds_all_list = np.concatenate(image_preds_all_list, axis=1)
 df.loc[:,new_feats] = image_preds_all_list
return df
if __name__ == '__main__':
if CFG['train']:
seed_everything(SEED)
 if CFG['train']:
train_df = pd.read_csv(CFG['train_path'])
 cv_df = pd.read_csv(CFG['cv_fold_path'])
 with torch.no_grad() :
train_df = update_stage1_oof_preds(train_df,cv_df)
 else:
do_full=False
if path.exists('.. /input/rsna-str-pulmonary-embolism-detection/train')and not do_full:
test_df=pd.read_csv(CFG['test_path'] ).head(50)
 else:
test_df=pd.read_csv(CFG['test_path'])
 with torch.no_grad() :
test_df = update_stage1_test_preds(test_df)
 if CFG['train']:
for fold,(train_fold, valid_fold)in enumerate(zip(CFG['train_folds'], CFG['valid_folds'])) :
train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, train_fold, valid_fold)
 device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 scaler = GradScaler()
optimizer = torch.optim.Adam(model.parameters() , lr=CFG['lr'])
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False
for epoch in range(CFG['epochs']):
train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)
 with torch.no_grad() :
valid_one_epoch(epoch, model, device, scheduler, val_loader, schd_loss_update=schd_loss_update)
 torch.save(model.state_dict() ,'{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag']))
model.load_state_dict(torch.load('{}/model_fold_{}_{}'.format(CFG['model_path'], fold, CFG['tag'])))
 valid_patients = cv_df.loc[cv_df.fold.isin(valid_fold), 'StudyInstanceUID'].unique()
valid_ = train_df.loc[train_df.StudyInstanceUID.isin(valid_patients),:].reset_index(drop=True)
 with torch.no_grad() :
val_pred_df = inference(model, device, valid_, CFG['train_img_path'])
 target = valid_[CFG['image_target_cols']].values
pred =(val_pred_df[CFG['image_target_cols']].values > 0.5 ).astype(int)
 print('Image PE Accuracy: {:.3f}'.format(( target==pred ).mean() *100))
loss = rsna_wloss_inference(valid_[CFG['image_target_cols']].values, valid_[CFG['exam_target_cols']].values,
val_pred_df[CFG['image_target_cols']].values, val_pred_df[CFG['exam_target_cols']].values,
list(valid_.groupby('StudyInstanceUID', sort=False)['SOPInstanceUID'].count()))
print('Validation loss = {:.4f}'.format(loss.detach().item()))
del model, optimizer, train_loader, val_loader, scaler, scheduler
torch.cuda.empty_cache()
train_loader, val_loader = prepare_train_dataloader(train_df, cv_df, np.arange(0, 20), np.array([]))
device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 scaler = GradScaler()
optimizer = torch.optim.Adam(model.parameters() , lr=CFG['lr'])
 scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=1); schd_loss_update=False
for epoch in range(CFG['epochs']):
train_one_epoch(epoch, model, device, scaler, optimizer, train_loader)
 torch.save(model.state_dict() ,'{}/model_{}'.format(CFG['model_path'], CFG['tag']))
else:
device = torch.device(CFG['device'])
 model = RSNAClassifier().to(device)
 model.load_state_dict(torch.load('{}/model_{}'.format(CFG['model_path'], CFG['tag'])))
 test_pred_df = inference(model, device, test_df, CFG['test_img_path'])
 test_pred_df.to_csv('kh_submission_raw.csv')
 ids = []
labels = []
gp_mean = test_pred_df.loc[:, ['StudyInstanceUID']+CFG['exam_target_cols']].groupby('StudyInstanceUID', sort=False ).mean()
for col in CFG['exam_target_cols']:
ids += [[patient+'_'+col for patient in gp_mean.index]]
labels += [gp_mean[col].values]
ids += [test_pred_df.SOPInstanceUID.values]
labels += [test_pred_df[CFG['image_target_cols']].values[:,0]]
ids = np.concatenate(ids)
 labels = np.concatenate(labels)
 assert len(ids)== len(labels)
 submission = pd.DataFrame()
submission['id'] = ids
submission['label'] = labels
print(submission.head(3))
print(submission.tail(3))
print(submission.shape)
 submission.to_csv('submission.csv', index=False )<load_from_csv>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class TestDatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
train_transaction = reduce_mem_usage(pd.read_csv('.. /input/train_transaction.csv', index_col='TransactionID'))
test_transaction = reduce_mem_usage(pd.read_csv('.. /input/test_transaction.csv', index_col='TransactionID'))
train_identity = reduce_mem_usage(pd.read_csv('.. /input/train_identity.csv', index_col='TransactionID'))
test_identity = reduce_mem_usage(pd.read_csv('.. /input/test_identity.csv', index_col='TransactionID'))
sample_submission = pd.read_csv('.. /input/sample_submission.csv', index_col='TransactionID' )<merge>
dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 test_data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)
train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True )<feature_engineering>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net('./effdet5-cutmix-augmix/best-checkpoint.bin' )
def corret_card_id(x):
x=x.replace('.0','')
 x=x.replace('-999','nan')
 return x
def define_indexes(df):
START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, '%Y-%m-%d')
 df['TransactionDT'] = df['TransactionDT'].apply(lambda x:(startdate + datetime.timedelta(seconds = x)))
 df['year'] = df['TransactionDT'].dt.year
df['month'] = df['TransactionDT'].dt.month
df['dow'] = df['TransactionDT'].dt.dayofweek
df['hour'] = df['TransactionDT'].dt.hour
df['day'] = df['TransactionDT'].dt.day
cards_cols= ['card1', 'card2', 'card3', 'card5']
for card in cards_cols:
if '1' in card:
df['card_id']= df[card].map(str)
 else :
df['card_id']+= ' '+df[card].map(str)
 df['card_id']=df['card_id'].apply(corret_card_id)
 return df<categorify>
class BaseWheatTTA:
image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):
def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):
def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):
def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):
def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
train['TransactionAmt_to_mean_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_mean_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('mean')
 train['TransactionAmt_to_std_card1'] = train['TransactionAmt'] / train.groupby(['card1'])['TransactionAmt'].transform('std')
 train['TransactionAmt_to_std_card4'] = train['TransactionAmt'] / train.groupby(['card4'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_mean_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_mean_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('mean')
 test['TransactionAmt_to_std_card1'] = test['TransactionAmt'] / test.groupby(['card1'])['TransactionAmt'].transform('std')
 test['TransactionAmt_to_std_card4'] = test['TransactionAmt'] / test.groupby(['card4'])['TransactionAmt'].transform('std')
 train['id_02_to_mean_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('mean')
 train['id_02_to_mean_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('mean')
 train['id_02_to_std_card1'] = train['id_02'] / train.groupby(['card1'])['id_02'].transform('std')
 train['id_02_to_std_card4'] = train['id_02'] / train.groupby(['card4'])['id_02'].transform('std')
 test['id_02_to_mean_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('mean')
 test['id_02_to_mean_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('mean')
 test['id_02_to_std_card1'] = test['id_02'] / test.groupby(['card1'])['id_02'].transform('std')
 test['id_02_to_std_card4'] = test['id_02'] / test.groupby(['card4'])['id_02'].transform('std')
 train['D15_to_mean_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_card1'] = train['D15'] / train.groupby(['card1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_card1'] = test['D15'] / test.groupby(['card1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std')
 train['D15_to_mean_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('mean')
 train['D15_to_mean_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('mean')
 train['D15_to_std_addr1'] = train['D15'] / train.groupby(['addr1'])['D15'].transform('std')
 train['D15_to_std_card4'] = train['D15'] / train.groupby(['card4'])['D15'].transform('std')
 test['D15_to_mean_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('mean')
 test['D15_to_mean_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('mean')
 test['D15_to_std_addr1'] = test['D15'] / test.groupby(['addr1'])['D15'].transform('std')
 test['D15_to_std_card4'] = test['D15'] / test.groupby(['card4'])['D15'].transform('std' )<count_unique_values>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
one_value_cols = [col for col in train.columns if train[col].nunique() <= 1]
one_value_cols_test = [col for col in test.columns if test[col].nunique() <= 1]
many_null_cols = [col for col in train.columns if train[col].isnull().sum() / train.shape[0] > 0.9]
many_null_cols_test = [col for col in test.columns if test[col].isnull().sum() / test.shape[0] > 0.9]
big_top_value_cols = [col for col in train.columns if train[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
big_top_value_cols_test = [col for col in test.columns if test[col].value_counts(dropna=False, normalize=True ).values[0] > 0.9]
cols_to_drop = list(set(many_null_cols + many_null_cols_test + big_top_value_cols + big_top_value_cols_test + one_value_cols+ one_value_cols_test))
cols_to_drop.remove('isFraud')
 train.drop(cols_to_drop, axis=1, inplace=True)
 test.drop(cols_to_drop, axis=1, inplace=True )<prepare_x_and_y>
def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
del train_transaction, train_identity, test_transaction, test_identity
target = train['isFraud'].copy()
X_train = train.drop('isFraud', axis=1)
 X_train.drop('TransactionDT', axis=1, inplace=True)
 X_test = test.drop('TransactionDT', axis=1)
 del train, test<set_options>
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.44, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
gc.collect()<categorify>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
for f in X_train.select_dtypes(include='category' ).columns.tolist() + X_train.select_dtypes(include='object' ).columns.tolist() :
lbl = LabelEncoder()
lbl.fit(list(X_train[f].values)+ list(X_test[f].values))
X_train[f] = lbl.transform(list(X_train[f].values))
X_test[f] = lbl.transform(list(X_test[f].values))<init_hyperparams>
results_plabel = []
for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
image_id = image_ids[i]
image_ = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 h,w,_ = np.shape(image_)
 boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result_p = {
'image_id': image_id,
'width':w,
'height':h,
'source':'usask_1',
'x':box[0],
'y':box[1],
'w':box[2],
'h':box[3],
}
results_plabel.append(result_p )
params = {'num_leaves': 491,
'min_child_weight': 0.03454472573214212,
'feature_fraction': 0.3797454081646243,
'bagging_fraction': 0.4181193142567742,
'min_data_in_leaf': 106,
'objective': 'binary',
'max_depth': -1,
'learning_rate': 0.006883242363721497,
"boosting_type": "gbdt",
"bagging_seed": 11,
"metric": 'auc',
"verbosity": -1,
'reg_alpha': 0.3899927210061127,
'reg_lambda': 0.6485237330340494,
'random_state': 47
}<define_variables>
results_df = pd.DataFrame(results_plabel, columns=['image_id', 'width','height','source','x','y','w','h'])
 results_df.head()
splits = 5
folds = KFold(n_splits = splits)
 oof = np.zeros(len(X_train))
predictions = np.zeros(len(X_test))<train_model>
results = []
for images, image_ids in test_data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )
<save_to_csv><EOS> 
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False )