Kaggle competition: VinBigData Chest X-ray Abnormalities Detection
<SOS> metric: OpenImagesObjectDetectionAP Kaggle data source: vinbigdata-chest-x-ray-abnormalities-detection<train_model>
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
pd.set_option('max_columns', 50)
if len(os.listdir('.. /input/global-wheat-detection/test/')) <11:
pass
else:
run_training()<load_pretrained>
!pip install detectron2 -f \
https://dl.fbaipublicfiles.com/detectron2/wheels/cu102/torch1.7/index.html
!pip install pytorch-pfn-extras timm
weights = f'plabel_model/last-checkpoint1.bin'
if not os.path.exists(weights):
weights = '.. /input/test11/weights/best-checkpoint-005epoch.bin'
net = load_net(weights )<categorify>
def save_yaml(filepath: str, content: Any, width: int = 120):
with open(filepath, "w")as f:
yaml.dump(content, f, width=width )
results = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
@dataclass
class Flags:
debug: bool = True
outdir: str = "results/det"
device: str = "cuda:0"
imgdir_name: str = "vinbigdata-chest-xray-resized-png-256x256"
seed: int = 111
target_fold: int = 0
label_smoothing: float = 0.0
model_name: str = "resnet18"
model_mode: str = "normal"
epoch: int = 20
batchsize: int = 8
valid_batchsize: int = 16
num_workers: int = 4
snapshot_freq: int = 5
ema_decay: float = 0.999
scheduler_type: str = ""
scheduler_kwargs: Dict[str, Any] = field(default_factory=lambda: {})
 scheduler_trigger: List[Union[int, str]] = field(default_factory=lambda: [1, "iteration"])
 aug_kwargs: Dict[str, Dict[str, Any]] = field(default_factory=lambda: {})
 mixup_prob: float = -1.0
def update(self, param_dict: Dict)-> "Flags":
for key, value in param_dict.items() :
if not hasattr(self, key):
raise ValueError(f"[ERROR] Unexpected key for flag = {key}")
 setattr(self, key, value)
 return self
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
flags_dict = {
"debug": False,
"outdir": "results/tmp_debug",
"imgdir_name": "vinbigdata-chest-xray-resized-png-256x256",
"model_name": "resnet18",
"num_workers": 4,
"epoch": 15,
"batchsize": 8,
"scheduler_type": "CosineAnnealingWarmRestarts",
"scheduler_kwargs": {"T_0": 28125},
"scheduler_trigger": [1, "iteration"],
"aug_kwargs": {
"HorizontalFlip": {"p": 0.5},
"ShiftScaleRotate": {"scale_limit": 0.15, "rotate_limit": 10, "p": 0.5},
"RandomBrightnessContrast": {"p": 0.5},
"CoarseDropout": {"max_holes": 8, "max_height": 25, "max_width": 25, "p": 0.5},
"Blur": {"blur_limit": [3, 7], "p": 0.5},
"Downscale": {"scale_min": 0.25, "scale_max": 0.9, "p": 0.3},
"RandomGamma": {"gamma_limit": [80, 120], "p": 0.6},
}
}
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<define_variables>
print("torch", torch.__version__)
 flags = Flags().update(flags_dict)
 print("flags", flags)
 debug = flags.debug
outdir = Path(flags.outdir)
 os.makedirs(str(outdir), exist_ok=True)
 flags_dict = dataclasses.asdict(flags)
 save_yaml(str(outdir / "flags.yaml"), flags_dict)
 inputdir = Path("/kaggle/input")
 datadir = inputdir / "vinbigdata-chest-xray-abnormalities-detection"
imgdir = inputdir / flags.imgdir_name
train = pd.read_csv(datadir / "train.csv")
DIR_INPUT = '/kaggle/input/global-wheat-detection'
DIR_TRAIN = f'{DIR_INPUT}/train'
DIR_TEST = f'{DIR_INPUT}/test'
DIR_WEIGHTS = '/kaggle/input/efficientdet2'
WEIGHTS_FILE = f'{DIR_WEIGHTS}/effdet5-cutmix-augmix/last-checkpoint.bin'
N_CLASSES=2
DEVICE = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu' )<load_from_csv>
is_normal_df = train.groupby("image_id")["class_id"].agg(lambda s:(s == 14 ).sum() ).reset_index().rename({"class_id": "num_normal_annotations"}, axis=1)
 is_normal_df.head()
test_df = pd.read_csv(f'{DIR_WEIGHTS}/submission.csv')
 test_df<load_from_csv>
num_normal_anno_counts_df = num_normal_anno_counts.reset_index()
num_normal_anno_counts_df["name"] = num_normal_anno_counts_df["index"].map({0: "Abnormal", 3: "Normal"})
 num_normal_anno_counts_df
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True)
 marking<data_type_conversions>
def get_vinbigdata_dicts(
 imgdir: Path,
train_df: pd.DataFrame,
train_data_type: str = "original",
use_cache: bool = True,
debug: bool = True,
target_indices: Optional[np.ndarray] = None, 
):
debug_str = f"_debug{int(debug)}"
train_data_type_str = f"_{train_data_type}"
cache_path = Path(".")/ f"dataset_dicts_cache{train_data_type_str}{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 train_meta = pd.read_csv(imgdir / "train_meta.csv")
 if debug:
train_meta = train_meta.iloc[:500]
image_id = train_meta.loc[0, "image_id"]
image_path = str(imgdir / "train" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, train_meta_row in tqdm(train_meta.iterrows() , total=len(train_meta)) :
record = {}
image_id, height, width = train_meta_row.values
filename = str(imgdir / "train" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
objs = []
for index2, row in train_df.query("image_id == @image_id" ).iterrows() :
class_id = row["class_id"]
if class_id == 14:
pass
else:
h_ratio = resized_height / height
w_ratio = resized_width / width
bbox_resized = [
int(row["x_min"])* w_ratio,
int(row["y_min"])* h_ratio,
int(row["x_max"])* w_ratio,
int(row["y_max"])* h_ratio,
]
obj = {
"bbox": bbox_resized,
"bbox_mode": BoxMode.XYXY_ABS,
"category_id": class_id,
}
objs.append(obj)
 record["annotations"] = objs
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 if target_indices is not None:
dataset_dicts = [dataset_dicts[i] for i in target_indices]
return dataset_dicts
def get_vinbigdata_dicts_test(
 imgdir: Path, test_meta: pd.DataFrame, use_cache: bool = True, debug: bool = True, 
):
debug_str = f"_debug{int(debug)}"
cache_path = Path(".")/ f"dataset_dicts_cache_test{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 if debug:
test_meta = test_meta.iloc[:500]
image_id = test_meta.loc[0, "image_id"]
image_path = str(imgdir / "test" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, test_meta_row in tqdm(test_meta.iterrows() , total=len(test_meta)) :
record = {}
image_id, height, width = test_meta_row.values
filename = str(imgdir / "test" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 return dataset_dicts
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<predict_on_test>

 class DatasetMixin(Dataset):
def __init__(self, transform=None):
self.transform = transform
def __getitem__(self, index):

 if torch.is_tensor(index):
index = index.tolist()
if isinstance(index, slice):
current, stop, step = index.indices(len(self))
return [self.get_example_wrapper(i)for i in
six.moves.range(current, stop, step)]
elif isinstance(index, list)or isinstance(index, numpy.ndarray):
return [self.get_example_wrapper(i)for i in index]
else:
return self.get_example_wrapper(index)
 def __len__(self):

 raise NotImplementedError
def get_example_wrapper(self, i):

 example = self.get_example(i)
 if self.transform:
example = self.transform(example)
 return example
def get_example(self, i):

 raise NotImplementedError
def make_predictions(images, score_threshold=0.22):
images = torch.stack(images ).cuda().float()
predictions = []
with torch.no_grad() :
outputs = model(images, torch.tensor([1]*images.shape[0] ).float().cuda())
 gc.collect()
torch.cuda.empty_cache()
for i in range(images.shape[0]):
boxes = outputs[i].detach().cpu().numpy() [:,:4]
scores = outputs[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
predictions.append({
'boxes': boxes[indexes],
'scores': scores[indexes],
})
 return [predictions]
def run_wbf(predictions, image_index, image_size=512, iou_thr=0.43, skip_box_thr=0.43, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<choose_model_class>
class VinbigdataTwoClassDataset(DatasetMixin):
def __init__(self, dataset_dicts, image_transform=None, transform=None, train: bool = True,
mixup_prob: float = -1.0, label_smoothing: float = 0.0):
super(VinbigdataTwoClassDataset, self ).__init__(transform=transform)
 self.dataset_dicts = dataset_dicts
self.image_transform = image_transform
self.train = train
self.mixup_prob = mixup_prob
self.label_smoothing = label_smoothing
def _get_single_example(self, i):
d = self.dataset_dicts[i]
filename = d["file_name"]
img = cv2.imread(filename)
 if self.image_transform:
img = self.image_transform(img)
 img = torch.tensor(np.transpose(img,(2, 0, 1)).astype(np.float32))
if self.train:
label = int(len(d["annotations"])> 0)
 if self.label_smoothing > 0:
if label == 0:
return img, float(label)+ self.label_smoothing
else:
return img, float(label)- self.label_smoothing
else:
return img, float(label)
 else:
return img, None
def get_example(self, i):
img, label = self._get_single_example(i)
 if self.mixup_prob > 0.and np.random.uniform() < self.mixup_prob:
j = np.random.randint(0, len(self.dataset_dicts))
p = np.random.uniform()
img2, label2 = self._get_single_example(j)
 img = img * p + img2 *(1 - p)
 if self.train:
label = label * p + label2 *(1 - p)
 if self.train:
label_logit = torch.tensor([1 - label, label], dtype=torch.float32)
 return img, label_logit
else:
return img
def __len__(self):
return len(self.dataset_dicts )
def load_net_eval(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path, map_location=DEVICE)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval()
return net.cuda()
model = load_net_eval(WEIGHTS_FILE )<load_pretrained>
dataset_dicts = get_vinbigdata_dicts(imgdir, train, debug=debug)
 dataset = VinbigdataTwoClassDataset(dataset_dicts )
def collate_fn(batch):
return tuple(zip(*batch))
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 test_data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<define_variables>
class Transform:
def __init__(
 self, hflip_prob: float = 0.5, ssr_prob: float = 0.5, random_bc_prob: float = 0.5 
):
self.transform = A.Compose(
 [
A.HorizontalFlip(p=hflip_prob),
A.ShiftScaleRotate(
 shift_limit=0.0625, scale_limit=0.1, rotate_limit=10, p=ssr_prob 
),
A.RandomBrightnessContrast(p=random_bc_prob),
] 
)
 def __call__(self, image):
image = self.transform(image=image)["image"]
return image
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<categorify>
aug_dataset = VinbigdataTwoClassDataset(dataset_dicts, image_transform=Transform() )
detection_threshold = 0.4
results = []
testdf_psuedo = []
for images, image_ids in test_data_loader:
predictions = make_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes = boxes[scores >= detection_threshold].astype(np.int32)
 scores = scores[scores >= detection_threshold]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': 'nvnn'+image_id,
'width': 1024,
'height': 1024,
'source': 'nvnn',
'x': float(box[0]),
'y': float(box[1]),
'w': float(box[2]),
'h': float(box[3])
 }
testdf_psuedo.append(result )<prepare_output>
class Transform:
def __init__(self, aug_kwargs: Dict):
self.transform = A.Compose(
 [getattr(A, name )(**kwargs)for name, kwargs in aug_kwargs.items() ] 
)
 def __call__(self, image):
image = self.transform(image=image)["image"]
return image
test_df_pseudo = pd.DataFrame(testdf_psuedo, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h'])
 test_df_pseudo.head()<feature_engineering>
class CNNFixedPredictor(nn.Module):
def __init__(self, cnn: nn.Module, num_classes: int = 2):
super(CNNFixedPredictor, self ).__init__()
self.cnn = cnn
self.lin = Linear(cnn.num_features, num_classes)
 print("cnn.num_features", cnn.num_features)
 for param in self.cnn.parameters() :
param.requires_grad = False
def forward(self, x):
feat = self.cnn(x)
 return self.lin(feat)
train_df = pd.read_csv(f'{DIR_INPUT}/train.csv')
 train_df['x'] = -1
train_df['y'] = -1
train_df['w'] = -1
train_df['h'] = -1
def expand_bbox(x):
r = np.array(re.findall("([0-9]+[.]?[0-9]*)", x))
if len(r)== 0:
r = [-1, -1, -1, -1]
return r
train_df[['x', 'y', 'w', 'h']] = np.stack(train_df['bbox'].apply(lambda x: expand_bbox(x)))
 train_df.drop(columns=['bbox'], inplace=True)
 train_df['x'] = train_df['x'].astype(np.float)
 train_df['y'] = train_df['y'].astype(np.float)
 train_df['w'] = train_df['w'].astype(np.float)
 train_df['h'] = train_df['h'].astype(np.float)
 train_df.head()<choose_model_class>
def build_predictor(model_name: str, model_mode: str = "normal"):
if model_mode == "normal":
return timm.create_model(model_name, pretrained=True, num_classes=2, in_chans=3)
 elif model_mode == "cnn_fixed":
timm_model = timm.create_model(model_name, pretrained=True, num_classes=0, in_chans=3)
 return CNNFixedPredictor(timm_model, num_classes=2)
 else:
raise ValueError(f"[ERROR] Unexpected value model_mode={model_mode}")
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path, map_location=DEVICE)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchTrain(net, config)
 return net.cuda()
model = load_net(WEIGHTS_FILE )<import_modules>
def accuracy(y: torch.Tensor, t: torch.Tensor)-> torch.Tensor:

 assert y.shape[:-1] == t.shape, f"y {y.shape}, t {t.shape} is inconsistent."
pred_label = torch.max(y.detach() , dim=-1)[1]
count = t.nelement()
correct =(pred_label == t ).sum().float()
acc = correct / count
return acc
def accuracy_with_logits(y: torch.Tensor, t: torch.Tensor)-> torch.Tensor:

 assert y.shape == t.shape
gt_label = torch.max(t.detach() , dim=-1)[1]
return accuracy(y, gt_label )
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
def __getitem__(self, index: int):
image_id = self.image_ids[index]
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
if 'nvnn' in image_id:
image = cv2.imread(f'{DIR_TEST}/{image_id[4:]}.jpg', cv2.IMREAD_COLOR)
 else:
image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize // 2
xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize, imsize, 3), 1, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1])> 0)]
return result_image, result_boxes<categorify>
def cross_entropy_with_logits(input, target, dim=-1):
loss = torch.sum(- target * F.log_softmax(input, dim), dim)
 return loss.mean()
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count
def collate_fn(batch):
return tuple(zip(*batch))
train_dataset = DatasetRetriever(
 image_ids=np.array(list(set(train_df['image_id'].values)) ).astype('O'),
marking=train_df,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=np.array(list(set(train_df['image_id'].values)) ).astype('O'),
marking=train_df,
transforms=get_valid_transforms() ,
test=False, 
)
 indices = torch.randperm(len(train_dataset)).tolist()
train_dataset = torch.utils.data.Subset(train_dataset, indices[:-50])
 validation_dataset = torch.utils.data.Subset(validation_dataset, indices[-50:])
 device = torch.device('cuda')if torch.cuda.is_available() else torch.device('cpu' )<init_hyperparams>
class Classifier(nn.Module):

 def __init__(self, predictor, lossfun=cross_entropy_with_logits):
super().__init__()
self.predictor = predictor
self.lossfun = lossfun
self.prefix = ""
def forward(self, image, targets):
outputs = self.predictor(image)
 loss = self.lossfun(outputs, targets)
 metrics = {
f"{self.prefix}loss": loss.item() ,
f"{self.prefix}acc": accuracy_with_logits(outputs, targets ).item()
}
ppe.reporting.report(metrics, self)
 return loss, metrics
def predict(self, data_loader):
pred = self.predict_proba(data_loader)
 label = torch.argmax(pred, dim=1)
 return label
def predict_proba(self, data_loader):
device: torch.device = next(self.parameters() ).device
y_list = []
self.eval()
with torch.no_grad() :
for batch in data_loader:
if isinstance(batch,(tuple, list)) :
batch = batch[0].to(device)
 else:
batch = batch.to(device)
 y = self.predictor(batch)
 y = torch.softmax(y, dim=-1)
 y_list.append(y)
 pred = torch.cat(y_list)
 return pred
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
summary_loss = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
return self.model
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
self.optimizer.zero_grad()
loss, _, _ = self.model(images, boxes, labels)
 loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<init_hyperparams>
supported_models = timm.list_models()
print(f"{len(supported_models)} models are supported in timm.")
 print(supported_models )
class TrainGlobalConfig:
num_workers = 2
batch_size = 4
n_epochs = 10
lr = 0.0002
folder = 'full-effdet5-cutmix-augmix'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)<load_pretrained>

 class EMA(object):

 def __init__(
 self,
model: nn.Module,
decay: float,
strict: bool = True,
use_dynamic_decay: bool = True, 
):
self.decay = decay
self.model = model
self.strict = strict
self.use_dynamic_decay = use_dynamic_decay
self.logger = getLogger(__name__)
 self.n_step = 0
self.shadow = {}
self.original = {}
self._assigned = False
for name, param in model.named_parameters() :
if param.requires_grad:
self.shadow[name] = param.data.clone()
def step(self):
self.n_step += 1
if self.use_dynamic_decay:
_n_step = float(self.n_step)
 decay = min(self.decay,(1.0 + _n_step)/(10.0 + _n_step))
else:
decay = self.decay
for name, param in self.model.named_parameters() :
if param.requires_grad:
assert name in self.shadow
new_average =(1.0 - decay)* param.data + decay * self.shadow[name]
self.shadow[name] = new_average.clone()
__call__ = step
def assign(self):

 if self._assigned:
if self.strict:
raise ValueError("[ERROR] `assign` is called again before `resume`.")
 else:
self.logger.warning(
 "`assign` is called again before `resume`."
"shadow parameter is already assigned, skip." 
)
 return
for name, param in self.model.named_parameters() :
if param.requires_grad:
assert name in self.shadow
self.original[name] = param.data.clone()
param.data = self.shadow[name]
self._assigned = True
def resume(self):

 if not self._assigned:
if self.strict:
raise ValueError("[ERROR] `resume` is called before `assign`.")
 else:
self.logger.warning("`resume` is called before `assign`, skip.")
 return
for name, param in self.model.named_parameters() :
if param.requires_grad:
assert name in self.shadow
param.data = self.original[name]
self._assigned = False
def collate_fn(batch):
return tuple(zip(*batch))
def run_training() :
device = torch.device('cuda:0')
 model.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=model, device=device, config=TrainGlobalConfig)
 return fitter.fit(train_loader, val_loader )<train_model>

 class LRScheduler(Extension):

 trigger = 1, 'iteration'
priority = PRIORITY_READER
name = None
def __init__(self, optimizer: optim.Optimizer, scheduler_type: str, scheduler_kwargs: Mapping[str, Any])-> None:
super().__init__()
self.scheduler = getattr(optim.lr_scheduler, scheduler_type )(optimizer, **scheduler_kwargs)
 def __call__(self, manager: ExtensionsManager)-> None:
self.scheduler.step()
def state_dict(self)-> None:
return self.scheduler.state_dict()
def load_state_dict(self, to_load)-> None:
self.scheduler.load_state_dict(to_load)
if len(os.listdir(DIR_TEST)) > 10:
model = run_training()<data_type_conversions>
def create_trainer(model, optimizer, device)-> Engine:
model.to(device)
 def update_fn(engine, batch):
model.train()
optimizer.zero_grad()
loss, metrics = model(*[elem.to(device)for elem in batch])
 loss.backward()
optimizer.step()
return metrics
trainer = Engine(update_fn)
 return trainer
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<define_variables>
import dataclasses
import os
import sys
from pathlib import Path
import numpy as np
import pandas as pd
import pytorch_pfn_extras.training.extensions as E
import torch
from ignite.engine import Events
from pytorch_pfn_extras.training import IgniteExtensionsManager
from sklearn.model_selection import StratifiedKFold
from torch import nn, optim
from torch.utils.data.dataloader import DataLoader
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 data_loader = DataLoader(
 dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<categorify>
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=flags.seed)
 y = np.array([int(len(d["annotations"])> 0)for d in dataset_dicts])
 split_inds = list(skf.split(dataset_dicts, y))
train_inds, valid_inds = split_inds[flags.target_fold]
train_dataset = VinbigdataTwoClassDataset(
 [dataset_dicts[i] for i in train_inds],
image_transform=Transform(flags.aug_kwargs),
mixup_prob=flags.mixup_prob,
label_smoothing=flags.label_smoothing, 
)
 valid_dataset = VinbigdataTwoClassDataset([dataset_dicts[i] for i in valid_inds])
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTARotate180(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 2,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 2,(2, 3))
def deaugment_boxes(self, boxes):
boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]
return boxes
class TTARotate270(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 3,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 3,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = boxes[:, [1,3]]
res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
train_loader = DataLoader(
 train_dataset,
batch_size=flags.batchsize,
num_workers=flags.num_workers,
shuffle=True,
pin_memory=True, 
)
 valid_loader = DataLoader(
 valid_dataset,
batch_size=flags.valid_batchsize,
num_workers=flags.num_workers,
shuffle=False,
pin_memory=True, 
)
 device = torch.device(flags.device)
 predictor = build_predictor(model_name=flags.model_name, model_mode=flags.model_mode)
 classifier = Classifier(predictor)
 model = classifier
optimizer = optim.Adam([param for param in model.parameters() if param.requires_grad], lr=1e-3)
 trainer = create_trainer(model, optimizer, device)
 ema = EMA(predictor, decay=flags.ema_decay)
 def eval_func(*batch):
loss, metrics = model(*[elem.to(device)for elem in batch])
 if flags.ema_decay > 0:
classifier.prefix = "ema_"
ema.assign()
loss, metrics = model(*[elem.to(device)for elem in batch])
 ema.resume()
classifier.prefix = ""
valid_evaluator = E.Evaluator(
 valid_loader, model, progress_bar=False, eval_func=eval_func, device=device 
)
 log_trigger =(1, "epoch")
 log_report = E.LogReport(trigger=log_trigger)
 extensions = [
log_report,
E.ProgressBarNotebook(update_interval=10 if debug else 100),
E.PrintReportNotebook() ,
E.FailOnNonNumber() ,
]
epoch = flags.epoch
models = {"main": model}
optimizers = {"main": optimizer}
manager = IgniteExtensionsManager(
 trainer, models, optimizers, epoch, extensions=extensions, out_dir=str(outdir), 
)
 manager.extend(valid_evaluator)
 manager.extend(
 E.snapshot_object(predictor, "predictor.pt"), trigger=(flags.snapshot_freq, "epoch")
)
 if flags.scheduler_type != "":
scheduler_type = flags.scheduler_type
print(f"using {scheduler_type} scheduler with kwargs {flags.scheduler_kwargs}")
 manager.extend(
 LRScheduler(optimizer, scheduler_type, flags.scheduler_kwargs),
trigger=flags.scheduler_trigger, 
)
 manager.extend(E.observe_lr(optimizer=optimizer), trigger=log_trigger)
 if flags.ema_decay > 0:
manager.extend(lambda manager: ema() , trigger=(1, "iteration"))
def save_ema_model(manager):
ema.assign()
torch.save(predictor.state_dict() , outdir / "predictor_ema.pt")
 ema.resume()
manager.extend(save_ema_model, trigger=(flags.snapshot_freq, "epoch"))
_ = trainer.run(train_loader, max_epochs=epoch )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , TTARotate180() , TTARotate270() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))<choose_model_class>
torch.save(predictor.state_dict() , outdir / "predictor_last.pt")
 df = log_report.to_dataframe()
df.to_csv(outdir / "log.csv", index=False)
 df
def load_net_eval(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path, map_location=DEVICE)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval()
return net.cuda()
model = load_net_eval('/kaggle/working/efficientdet_pseudolabeling.bin' )<categorify>
print("Training done! Start prediction...")
 valid_pred = classifier.predict_proba(valid_loader ).cpu().numpy()
valid_pred_df = pd.DataFrame({
"image_id": [dataset_dicts[i]["image_id"] for i in valid_inds],
"class0": valid_pred[:, 0],
"class1": valid_pred[:, 1]
})
 valid_pred_df.to_csv(outdir/"valid_pred.csv", index=False)
 test_meta = pd.read_csv(inputdir / "vinbigdata-testmeta" / "test_meta.csv")
 dataset_dicts_test = get_vinbigdata_dicts_test(imgdir, test_meta, debug=debug)
 test_dataset = VinbigdataTwoClassDataset(dataset_dicts_test, train=False)
 test_loader = DataLoader(
 test_dataset,
batch_size=flags.valid_batchsize,
num_workers=flags.num_workers,
shuffle=False,
pin_memory=True, 
)
 test_pred = classifier.predict_proba(test_loader ).cpu().numpy()
test_pred_df = pd.DataFrame({
"image_id": [d["image_id"] for d in dataset_dicts_test],
"class0": test_pred[:, 0],
"class1": test_pred[:, 1]
})
 test_pred_df.to_csv(outdir/"test_pred.csv", index=False )
def make_tta_predictions(images, score_threshold=0.25):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = model(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions<categorify>
pred_2class = pd.read_csv(inputdir/"vinbigdata2classpred/test_pred.csv")
 low_threshold = 0.0
high_threshold = 0.976
pred_2class
<save_to_csv><EOS> 
NORMAL = "14 1 0 0 1 1"
pred_det_df = pd.read_csv(inputdir/"vinbigdata-detectron2-prediction/results/20210125_all_alb_aug_512_cos/submission.csv")
 n_normal_before = len(pred_det_df.query("PredictionString == @NORMAL"))
merged_df = pd.merge(pred_det_df, pred_2class, on="image_id", how="left")
 if "target" in merged_df.columns:
merged_df["class0"] = 1 - merged_df["target"]
c0, c1, c2 = 0, 0, 0
for i in range(len(merged_df)) :
p0 = merged_df.loc[i, "class0"]
if p0 < low_threshold:
c0 += 1
elif low_threshold <= p0 and p0 < high_threshold:
merged_df.loc[i, "PredictionString"] += f" 14 {p0} 0 0 1 1"
c1 += 1
else:
merged_df.loc[i, "PredictionString"] = NORMAL
c2 += 1
n_normal_after = len(merged_df.query("PredictionString == @NORMAL"))
print(
 f"n_normal: {n_normal_before} -> {n_normal_after} with threshold {low_threshold} & {high_threshold}" 
)
 print(f"Keep {c0} Add {c1} Replace {c2}")
 submission_filepath = str(outdir / "submission.csv")
 submission_df = merged_df[["image_id", "PredictionString"]]
submission_df.to_csv(submission_filepath, index=False)
 print(f"Saved to {submission_filepath}" )