Kaggle competition: VinBigData Chest X-ray Abnormalities Detection
<SOS> metric: OpenImagesObjectDetectionAP Kaggle data source: vinbigdata-chest-x-ray-abnormalities-detection<normalization>
warnings.filterwarnings("ignore" )
def TTAImage(image, index):
image1 = image.copy()
if index==0:
rotated_image = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image
elif index==1:
rotated_image2 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image2 = cv2.rotate(rotated_image2, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image2
elif index==2:
rotated_image3 = cv2.rotate(image1, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 rotated_image3 = cv2.rotate(rotated_image3, cv2.ROTATE_90_CLOCKWISE)
 return rotated_image3
elif index == 3:
return image1
def rotBoxes90(boxes, im_w, im_h):
ret_boxes =[]
for box in boxes:
x1, y1, x2, y2 = box
x1, y1, x2, y2 = x1-im_w//2, im_h//2 - y1, x2-im_w//2, im_h//2 - y2
x1, y1, x2, y2 = y1, -x1, y2, -x2
x1, y1, x2, y2 = int(x1+im_w//2), int(im_h//2 - y1), int(x2+im_w//2), int(im_h//2 - y2)
 x1a, y1a, x2a, y2a = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
 ret_boxes.append([x1a, y1a, x2a, y2a])
 return np.array(ret_boxes)
 def detect1Image(img, img0, model, device, aug):
img = img.transpose(2,0,1)
 img = torch.from_numpy(img ).to(device)
 img = img.half() if half else img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 pred = model(img, augment=aug)[0]
pred = non_max_suppression(pred, score_thr, iou_thr, merge=True, classes=None, agnostic=False)
 boxes = []
scores = []
for i, det in enumerate(pred):
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], img0.shape ).round()
for *xyxy, conf, cls in det:
boxes.append([int(xyxy[0]), int(xyxy[1]), int(xyxy[2]), int(xyxy[3])])
 scores.append(conf)
 return np.array(boxes), np.array(scores )<categorify>
train_data = pd.read_csv(path+'train.csv')
 samp_subm = pd.read_csv(path+'sample_submission.csv' )
def detect() :
models = []
for w in weights if isinstance(weights, list)else [weights]:
models.append(torch.load(w, map_location=device)['model'].to(device ).float().eval())
 dataset = LoadImages(source, img_size=imgsz)
 img = torch.zeros(( 1, 3, imgsz, imgsz), device=device)
 for model in models:
if half:
model.half()
_ = model(img.half() if half else img)if device.type != 'cpu' else None
all_paths, all_bboxes, all_confs = [], [], []
for p, img, img0, _ in dataset:
print()
img = img.transpose(1,2,0)
 all_paths.append(p)
 m_box, m_score = [], []
for model in models:
enboxes = []
enscores = []
for i in range(4):
img1 = TTAImage(img, i)
 boxes, scores = detect1Image(img1, img0, model, device, aug=False)
 for _ in range(3-i):
boxes = rotBoxes90(boxes, *img.shape[:2])
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = detect1Image(img, img0, model, device, aug=True)
 enboxes.append(boxes)
 enscores.append(scores)
 boxes, scores = run_wbf(enboxes, enscores, image_size=1024, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes.astype(np.int32 ).clip(min=0, max=1024)
 indices = scores >= score_thr
boxes = boxes[indices]
scores = scores[indices]
m_box.append(boxes)
 m_score.append(scores)
 all_bboxes.append(m_box)
 all_confs.append(m_score)
 return all_paths, all_bboxes, all_confs<find_best_params>
print('Number train samples:', len(train_data.index))
print('Number test samples:', len(samp_subm.index))
with torch.no_grad() :
res = detect()
paths,all_boxes,all_confs = res<define_variables>
idnum = 2
image_id = train_data.loc[idnum, 'image_id']
data_file = dicom.dcmread(path+'train/'+image_id+'.dicom')
 img = data_file.pixel_array
results =[]
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 bts = []
sts = []
for row in range(len(paths)) :
image_id = paths[row].split("/")[-1].split(".")[0]
boxes, scores = run_wbf(all_boxes[row],all_confs[row], skip_box_thr = skip_box_thr, iou_thr = iou_thr)
 indices = scores >= score_thr
boxes = boxes[indices]
scores = scores[indices]
boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {'image_id': image_id,'PredictionString': format_prediction_string(boxes, scores)}
results.append(result)
 bts.append(boxes)
 sts.append(scores)
 !rm -rf *
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
samp_subm.to_csv('submission1.csv', index=False )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<set_options>
pred_2class = pd.read_csv(".. /input/vinbigdata-2class-prediction/2-cls test pred.csv")
 low_threshold = 0.001
high_threshold = 0.87
pred_2class
<load_from_csv><EOS> 
NORMAL = "14 1 0 0 1 1"
pred_det_df = pd.read_csv(".. /input/vinbigdatastack/submission_postprocessed.csv")
 n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str("submission2.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )