Kaggle competition: Global Wheat Detection
<SOS> metric: custom metric Kaggle data source: global-wheat-detection<sort_values>
sys.path.insert(0, ".. /input/weightedboxesfusion")
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device)
df_test = df_test.sort_values([col_groupby, col_index] ).reset_index(drop=True)
 df_test.head()<feature_engineering>
def load_resnet101_model(checkpoint_path):
num_classes = 2
backbone = resnet_fpn_backbone('resnet101', pretrained= False)
 model_faster = FasterRCNN(backbone, num_classes)
 in_features = model_faster.roi_heads.box_predictor.cls_score.in_features
model_faster.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
 checkpoint = torch.load(checkpoint_path)
 model_faster.load_state_dict(checkpoint['model_state_dict'])
 model_faster.to(device);
model_faster.eval()
return model_faster
def load_resnet50_model(checkpoint_path):
num_classes = 2
backbone = resnet_fpn_backbone('resnet50', pretrained= False)
 model_faster = FasterRCNN(backbone, num_classes)
 in_features = model_faster.roi_heads.box_predictor.cls_score.in_features
model_faster.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
 model_faster.to(device);
checkpoint = torch.load(checkpoint_path)
 model_faster.load_state_dict(checkpoint['model_state_dict'])
 model_faster.to(device);
model_faster.eval()
return model_faster
def load_resnet152_model(checkpoint_path):
num_classes = 2
backbone = resnet_fpn_backbone('resnet152', pretrained= False)
 model_faster = FasterRCNN(backbone, num_classes)
 in_features = model_faster.roi_heads.box_predictor.cls_score.in_features
model_faster.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
 model_faster.to(device);
checkpoint = torch.load(checkpoint_path)
 model_faster.load_state_dict(checkpoint['model_state_dict'])
 model_faster.to(device);
model_faster.eval()
return model_faster
df_test['dicom_path'] = test_image_path + "/" +\
df_test[col_groupby].values + "/" + \
df_test['SeriesInstanceUID'].values + "/" + \
df_test[col_index].values + ".dcm"
print(df_test['dicom_path'][0])
 df_test.head()<merge>
resnet101_33e = ".. /input/resnet10133e-1024/cp33.pt"
resnet50 = ".. /input/resnet50-40e/resnet50_40e.pt"
resnet101_25e = ".. /input/resnet101mymodel/cp25.pt"
resnet152_20e = ".. /input/resnet152-20e/cp20.pt"
resnet152_19e =".. /input/resnet152-19e/cp19(1 ).pt"
df_test_study = df_test[df_test[col_groupby].duplicated() ==False]
df_test_study['start_index'] = df_test_study.index.values
df_tmp = df_test.groupby(col_groupby)[col_index].agg(len ).reset_index()
df_tmp.columns = [col_groupby, 'num_images']
df_test_study = pd.merge(df_test_study, df_tmp, on=col_groupby)
 print(df_test_study.shape)
 df_test_study.head()<create_dataframe>
models = [
load_resnet50_model(resnet50),
load_resnet152_model(resnet152_20e),
load_resnet152_model(resnet152_19e)
 ]
class DicomDataset(Dataset):
def __init__(self, X_study, X_image, transform=None, meta=False, verbose=False):
self.X_study = X_study
self.X_image = X_image
self.transform = transform
self.verbose = verbose
def __getitem__(self, index):
study = self.X_study[col_groupby][index]
start_index = self.X_study['start_index'][index]
end_index = self.X_study['start_index'][index] + self.X_study['num_images'][index]
df_study = self.X_image.iloc[start_index:end_index].reset_index(drop=True)
 images_study = []
z_pos = []
for i in range(len(df_study)) :
tmp_path = df_study['dicom_path'][i]
try:
tmp_dcm = pydicom.dcmread(tmp_path)
 tmp_npy = np.asarray(tmp_dcm.pixel_array)
 images_study.append(tmp_npy)
 if i==0:
RescaleSlope = tmp_dcm['RescaleSlope'].value
RescaleIntercept = tmp_dcm['RescaleIntercept'].value
PatientPosition = tmp_dcm['PatientPosition'].value
z_pos.append(tmp_dcm['ImagePositionPatient'].value[-1])
 except:
print("loading error!!!, study: {}, index: {}".format(study, i))
tmp_npy = np.zeros([512, 512], np.int16)
 images_study.append(tmp_npy)
 if i==0:
RescaleSlope = 1
RescaleIntercept = -1024
PatientPosition = 'HFS'
z_pos.append(-10000-i)
 images_study = np.array(images_study)
 z_pos = np.array(z_pos)
 images_study = images_study[np.argsort(z_pos)]
df_study['z_pos'] = z_pos
df_study = df_study.sort_values('z_pos' ).reset_index(drop=True)
 df_study['series_index'] = np.arange(len(df_study))
if self.verbose: print(images_study.shape)
 if self.verbose: print(z_pos)
 if self.verbose: print(RescaleIntercept, RescaleSlope, PatientPosition)
 images_study_processed =(images_study.astype(np.float32)* RescaleSlope + RescaleIntercept)/1000
if PatientPosition=='FFP':
images_study_processed = images_study_processed[:, ::-1, ::-1]
images_study_processed = images_study_processed.reshape([-1, 1, 512, 512] ).astype(np.float16)
 return images_study_processed, df_study
def __len__(self):
return len(self.X_study )<concatenate>
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test/'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def my_collate(batch):
return torch.Tensor(batch[0][0]), batch[0][1]<define_search_space>
def get_valid_transforms() :
return A.Compose([
ToTensorV2(p=1.0),
], p=1.0 )
class nnWindow(nn.Module):
def __init__(self):
super(nnWindow, self ).__init__()
wso = np.array(((40,80),(80,200),(40,400)))/1000
conv_ = nn.Conv2d(1,3, kernel_size=(1, 1))
conv_.weight.data.copy_(torch.tensor([[[[1./wso[0][1]]]],[[[1./wso[1][1]]]],[[[1./wso[2][1]]]]]))
conv_.bias.data.copy_(torch.tensor([0.5 - wso[0][0]/wso[0][1],
0.5 - wso[1][0]/wso[1][1],
0.5 -wso[2][0]/wso[2][1]]))
self.window = nn.Sequential(
 conv_,
nn.Sigmoid() ,
nn.InstanceNorm2d(3)
)
 def forward(self, input1):
return self.window(input1)
 class MyEffNet_b0(nn.Module):
def __init__(self, num_classes=10, base_model='tf_efficientnet_b0_ns'):
super(MyEffNet_b0, self ).__init__()
self.num_classes = num_classes
self.mode = 'train'
self.window = nnWindow()
self.base_model = timm.create_model(base_model, pretrained=False, num_classes=10 ).to(device, non_blocking=True)
 self.avgpool = nn.AdaptiveAvgPool2d(1)
 self.last_linear = nn.Linear(self.base_model.num_features, num_classes)
 def forward(self, input1):
bs, ch, h, w = input1.size()
x = self.window(input1)
 x = self.base_model.forward_features(x)
 feature = self.avgpool(x ).view(bs, -1)
 y = self.last_linear(feature)
 return y
def feature(self, input1):
bs, ch, h, w = input1.size()
x = self.window(input1)
 x = self.base_model.forward_features(x)
 feature = self.avgpool(x ).view(bs, -1)
 y = self.last_linear(feature)
 return y, feature<define_search_model>
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool1d(1)
 self.fc1 = nn.Conv1d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv1d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class CNN1D(nn.Module):
def __init__(self, num_classes=400, input_ch=1, verbose=False):
super(CNN1D, self ).__init__()
pool = 4
drop = 0.1
self.verbose = verbose
self.layer1 = nn.Sequential(
 nn.Conv1d(input_ch//pool, 64, kernel_size=7, stride=1, padding=3, bias=False),
nn.BatchNorm1d(64),
nn.ReLU(inplace=True),
SEModule(64, 16), 
)
 self.fpool = nn.MaxPool1d(kernel_size=pool, stride=pool, padding=0)
 self.maxpool = nn.MaxPool1d(kernel_size=3, stride=2, padding=1)
 self.layer2 = nn.Sequential(
 nn.Conv1d(64, 128, kernel_size=3, stride=1, padding=1, bias=False),
nn.BatchNorm1d(128),
nn.ReLU(inplace=True),
SEModule(128, 16), 
)
 self.layer3 = nn.Sequential(
 nn.Conv1d(128, 256, kernel_size=3, stride=1, padding=1, bias=False),
nn.BatchNorm1d(256),
nn.ReLU(inplace=True),
SEModule(256, 16), 
)
 self.layer4 = nn.Sequential(
 nn.Conv1d(256, 512, kernel_size=3, stride=1, padding=1, bias=False),
nn.BatchNorm1d(512),
nn.ReLU(inplace=True),
SEModule(512, 16), 
)
 self.avgpool = nn.AdaptiveAvgPool1d(1)
 self.fc2 = nn.Conv1d(
 input_ch//pool+64+128+256+512,
2, kernel_size=1)
 self.fc = nn.Sequential(
 nn.Linear(512, 512),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(512, 512),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(512, 9), 
)
 def forward(self, x_input):
bs, ch, d = x_input.size()
x0 = torch.transpose(x_input, 1, 2)
 x0 = self.fpool(x0)
 x0 = torch.transpose(x0, 1, 2)
 x1 = self.layer1(x0)
 x1 = self.maxpool(x1)
 x2 = self.layer2(x1)
 x2 = self.maxpool(x2)
 x3 = self.layer3(x2)
 x3 = self.maxpool(x3)
 x4 = self.layer4(x3)
 x5 = torch.cat([
x0,
F.adaptive_avg_pool1d(x1, d),
F.adaptive_avg_pool1d(x2, d),
F.adaptive_avg_pool1d(x3, d),
F.adaptive_avg_pool1d(x4, d),
], axis=1)
 y2 = self.fc2(x5)
 b, ch, d = x_input.size()
y = self.avgpool(x4)
 y = y.view(b, -1)
 y = self.fc(y)
 return y, y2<create_dataframe>
dataset = DatasetRetriever(np.array([path.split('/')[-1][:-4] for path in glob.glob(f'{DATA_ROOT_PATH}/*.jpg')]),get_valid_transforms())
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)
dataset_test = DicomDataset(df_test_study, df_test)
 test_loader = DataLoader(
 dataset_test,
batch_size=1,
shuffle=False,
num_workers=num_cpu,
pin_memory=True,
collate_fn=my_collate
)<train_on_grid>
class BaseWheatTTA:

 image_size = 1024
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTARotate180(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 2,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 2,(2, 3))
def deaugment_boxes(self, boxes):
boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]
return boxes
class TTARotate270(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 3,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 3,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = boxes[:, [1,3]]
res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )
model_b0 = MyEffNet_b0(base_model='tf_efficientnet_b2_ns' ).to(device, non_blocking=True)
 model_1dcnn = CNN1D(input_ch=num_features1_1 ).to(device, non_blocking=True)
 model_b0.eval()
model_1dcnn.eval()
lastfunc = nn.Sigmoid().to(device, non_blocking=True )<categorify>
def make_tta_predictions(model,images, score_threshold=0.35):
model.eval()
model.to(device)
 with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for tta_transform in tta_transforms:
result = []
det = model(tta_transform.batch_augment(images.clone()))
for i in range(images.shape[0]):
boxes = det[i]['boxes'].detach().cpu().numpy() [:,:4]
scores = det[i]['scores'].detach().cpu().numpy() [:]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def make_ensemble_predictions(images):
images = list(image.to(device)for image in images)
 result = []
for model in models:
predictions = make_tta_predictions(model, images)
 with torch.no_grad() :
for i in range(len(images)) :
boxes, scores, labels = run_wbf(predictions, image_index=i)
 outputs = {'boxes': boxes,'labels': labels, 'scores':scores }
result.append([outputs])
 torch.cuda.empty_cache()
return result
def run_wbf(predictions, image_index, image_size=1024, iou_thr=0.45, skip_box_thr=0.35,score_threshold = 0.35, weights=None,conf_type='max'):
boxes = [prediction[image_index]['boxes']/(image_size)for prediction in predictions]
scores = [prediction[image_index]['scores'] for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0])for prediction in predictions]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr, conf_type=conf_type)
 indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
scores = scores[indexes]
boxes = boxes*(image_size-1)
 return boxes, scores, labels
def batch_padding(batch):
bs, ch, d = batch.shape
d_new = int(np.ceil(d/64)*64)
 batch_new = torch.from_numpy(np.zeros([bs, ch, d_new], np.float32)).to(device, non_blocking=True)
 batch_new[:, :, :d] = batch
return batch_new<define_variables>
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , TTARotate180() , TTARotate270() , None]):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform]))
df_pred_image = []
df_pred_study = []
starttime = time.time()
verbose = False
for study_index,(images, df_study)in enumerate(test_loader):
if verbose: print("load index {} done".format(study_index), time.time() -starttime)
 if(study_index+1)%10==0:
print("{}/{}, sec: {:.1f}".format(study_index+1, len(df_test_study), time.time() -starttime))
num_batches = int(np.ceil(images.shape[0]/BATCH_SIZE))
num_images = len(df_study)
 df_study_image = df_study[[col_groupby, col_index, 'SeriesInstanceUID']]
df_study_study = df_study[[col_groupby]].iloc[:1]
for fold in range(NUM_FOLD):
model_b0.load_state_dict(torch.load("{}/weight_epoch_16_fold{}.pth".format(weight_dir1_1, fold+1)))
 model_1dcnn.load_state_dict(torch.load("{}/cnn_weight_best_fold{}.pth".format(weight_dir2_1, fold+1)))
 features = []
for batch_index in range(num_batches):
with torch.no_grad() :
with torch.cuda.amp.autocast() :
batch = images[batch_index*BATCH_SIZE:(batch_index+1)*BATCH_SIZE].to(device, non_blocking=True)
 _, feature = model_b0.feature(batch)
 features.append(feature)
 features = torch.cat(features, axis=0)
 features = torch.transpose(features, 0,1 ).reshape([1, num_features1_1, -1])
 features = batch_padding(features)
 with torch.no_grad() :
output1, output2 = model_1dcnn(features)
 output2 = output2[:,-1:]
output1 = lastfunc(output1)
 output2 = lastfunc(output2)[:,:,:num_images]
for i, col in enumerate(col_targets[:-1]):
df_study_study["{}_pred_fold{}".format(col, fold+1)] = output1[0, i].data.cpu().numpy()
df_study_image["{}_pred_fold{}".format(col_targets[-1], fold+1)] = output2[0, 0].data.cpu().numpy()
df_pred_study.append(df_study_study)
 df_pred_image.append(df_study_image)
<concatenate>
validation_image_precisions = []
iou_thresholds = [x for x in np.arange(0.5, 0.76, 0.05)]
results = []
ts = 0.147
for images, image_ids in data_loader:
predictions = make_ensemble_predictions(images)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i, iou_thr=0.35, skip_box_thr=0.35, score_threshold = 0.35, weights=None,conf_type='avg')
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=1023)
 image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result)
<feature_engineering><EOS> 
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head(10 )