Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<choose_model_class>
DEVICE = torch.device("cuda:0")
 DATA_SOURCE = os.path.join(".. ","input","aptos2019-blindness-detection")
 MODEL_SOURCE = os.path.join(".. ","input","densenet161-1-18-v2-pth")
 MODEL_SIZE = 224
ds_train = get_dataset(files_train, CFG, augment=True, shuffle=True, repeat=True)
 ds_train = ds_train.map(lambda img, label:(img, tuple([label])))
 steps_train = count_data_items(files_train)/(CFG['batch_size'] * REPLICAS)
 model_B0 = compile_new_model(CFG, 'EfficientNetB0')
 model_B3 = compile_new_model(CFG, 'EfficientNetB3')
 model_B5 = compile_new_model(CFG, 'EfficientNetB5')
 model_B6 = compile_new_model(CFG, 'EfficientNetB6')
 print("
Begin Training Models B0")
 history_B0 = model_B0.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B0
")
 Begin Training Models B1")
 Done Training model_B1
")
 Begin Training Models B2")
 Done Training model_B2
")
 print("
Begin Training Models B3")
 history_B3 = model_B3.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B3
")
 Begin Training Models B4")
 Done Training model_B4
")
 print("
Begin Training Models B5")
 history_B5 = model_B5.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B5
")
 print("
Begin Training Models B6")
 history_B6 = model_B6.fit(ds_train,
verbose = 1,
steps_per_epoch = steps_train,
epochs = CFG['epochs'],
callbacks = [get_lr_callback(CFG)])
 print("
Done Training model_B6
")
 Begin Training Models B7")
 Done Training model_B7
" )<predict_on_test>
def crop_image(img,tol=7):
w, h = img.shape[1],img.shape[0]
gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
 gray_img = cv2.blur(gray_img,(5,5))
shape = gray_img.shape
gray_img = gray_img.reshape(-1,1)
 quant = quantile_transform(gray_img, n_quantiles=256, random_state=0, copy=True)
 quant =(quant*256 ).astype(int)
 gray_img = quant.reshape(shape)
 xp =(gray_img.mean(axis=0)>tol)
 yp =(gray_img.mean(axis=1)>tol)
 x1, x2 = np.argmax(xp), w-np.argmax(np.flip(xp))
y1, y2 = np.argmax(yp), h-np.argmax(np.flip(yp))
if x1 >= x2 or y1 >= y2 :
return img
else:
img1=img[y1:y2,x1:x2,0]
img2=img[y1:y2,x1:x2,1]
img3=img[y1:y2,x1:x2,2]
img = np.stack([img1,img2,img3],axis=-1)
 return img
def process_image(image, size=512):
image = cv2.resize(image,(size,int(size*image.shape[0]/image.shape[1])))
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 try:
image = crop_image(image, tol=15)
 except Exception as e:
image = image
print(str(e))
return image
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 preds_B0 = model_B0.predict(ds_testAug, verbose=1, steps=steps )<create_dataframe>
class RetinopathyDataset(Dataset):
def __init__(self, transform, is_test=False):
self.transform = transform
self.base_transform = transforms.Resize(( MODEL_SIZE, MODEL_SIZE))
self.is_test = is_test
if not os.path.exists("cache"): os.mkdir("cache")
 if is_test : file = "test.csv"
else : file = "train.csv"
csv_file = os.path.join(DATA_SOURCE, file)
 df = pd.read_csv(csv_file)
 self.data = df.reset_index(drop=True)
 def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
if self.is_test : archive = "test_images"
else : archive = "train_images"
folder = os.path.join(DATA_SOURCE, archive)
 code = str(self.data.loc[idx, 'id_code'])
 file = code + ".png"
cache_path = os.path.join("cache",code+".png")
 cached = os.path.exists(cache_path)
 if not cached :
path = os.path.join(folder, file)
 image = cv2.imread(path)
 image = process_image(image)
 imgpil = Image.fromarray(image)
 imgpil = self.base_transform(imgpil)
 imgpil.save(cache_path,"PNG")
 imgpil = Image.open(cache_path)
 img_tensor = self.transform(imgpil)
 if self.is_test : return {'image': img_tensor}
else :
label = self.data.loc[idx, "diagnosis"]
return {'image': img_tensor, 'label': label}
def get_df(self):
return self.data
<create_dataframe>
NUM_FOLDS = 5
data_augmentation = transforms.Compose([
transforms.RandomRotation(( -15, 15)) ,
transforms.Resize(224),
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
 ])
 DATA = RetinopathyDataset(data_augmentation)
 df = DATA.get_df()
skf = StratifiedKFold(n_splits=NUM_FOLDS)
 folds_generator = skf.split(df.index.values, df.diagnosis.values)
 data_train, data_eval = [], []
for t, e in folds_generator:
data_train.append(t)
 data_eval.append(e )
<predict_on_test>
def get_dataloader_for_fold(n, data, train_data, eval_data, batch_size):

 train_sampler = SubsetRandomSampler(train_data[n])
 valid_sampler = SubsetRandomSampler(eval_data[n])
 data_loader_train = torch.utils.data.DataLoader(data,
batch_size=batch_size, drop_last=False,
sampler=train_sampler)
 data_loader_eval = torch.utils.data.DataLoader(data,
batch_size=batch_size, drop_last=False,
sampler=valid_sampler)
 return data_loader_train, data_loader_eval
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 preds_B3 = model_B3.predict(ds_testAug, verbose=1, steps=steps )<predict_on_test>
class Classificator0(nn.Module):

 def __init__(self, size=128):
super(Classificator0, self ).__init__()
self.size = size
self.network = nn.Sequential(
 nn.BatchNorm1d(size),
nn.Dropout(p=0.3),
nn.Linear(in_features=size, out_features=5, bias=True), 
)
 def forward(self, x):
return self.network(x)
 class Classificator(nn.Module):

 def __init__(self, size=128):
super(Classificator, self ).__init__()
self.size = size
self.network = nn.Sequential(
 nn.BatchNorm1d(size),
nn.Dropout(p=0.25),
nn.Linear(in_features=size, out_features=size, bias=True),
nn.ReLU() ,
nn.BatchNorm1d(size),
nn.Dropout(p=0.5),
nn.Linear(in_features=size, out_features=5, bias=True), 
)
 def forward(self, x):
return self.network(x )
<predict_on_test>
def get_base_model() :

 model = torchvision.models.densenet161(pretrained=False)
 in_features = model.classifier.in_features
model.classifier = Classificator0(in_features)
 model_path = os.path.join(MODEL_SOURCE, "densenet161.1.18.v2.pth")
 model.load_state_dict(torch.load(model_path))
model.classifier = Classificator(in_features)
 model = model.to(DEVICE)
 model.eval()
return model
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 preds_B5 = model_B5.predict(ds_testAug, verbose=1, steps=steps )<predict_on_test>
def train_model(model, optimizer, scheduler, train_data_loader, eval_data_loader,
file_name, num_epochs = 50, patience = 7, prev_loss = 1000.00):

 criterion = nn.CrossEntropyLoss()
countdown = patience
best_loss = 1000.00
since = time.time()
for epoch in range(num_epochs):
running_loss = 0.0
counter = 0
for bi, d in enumerate(train_data_loader):
inputs = d["image"].to(DEVICE, dtype=torch.float)
 labels = d["label"].to(DEVICE, dtype=torch.long)
 if inputs.shape[0] > 1 :
counter += inputs.size(0)
 model.to(DEVICE)
 model.train()
optimizer.zero_grad()
outputs = model(inputs)
 loss = criterion(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 loss_val = running_loss / counter
print("{:7} {:.4f} {:.4f}".format(counter, loss.item() *1, loss_val), end="\r")
 epoch_loss = running_loss /(len(train_data_loader)* train_data_loader.batch_size)
 time_elapsed = time.time() - since
print(" T{:3}/{:3} loss: {:.4f}({:3.0f}m {:2.0f}s)".format(
 epoch, num_epochs - 1, epoch_loss,time_elapsed // 60, time_elapsed % 60))
running_loss = 0.0
counter = 0
for bi, d in enumerate(eval_data_loader):
inputs = d["image"].to(DEVICE, dtype=torch.float)
 counter += inputs.size(0)
 labels = d["label"].to(DEVICE, dtype=torch.long)
 model.to(DEVICE)
 model.eval()
with torch.no_grad() :
outputs = model(inputs)
 loss = criterion(outputs, labels)
 running_loss += loss.item() * inputs.size(0)
 loss_val = running_loss / counter
print("{:7} {:.4f} {:.4f}".format(counter, loss.item() *1, loss_val), end="\r")
 epoch_loss = running_loss /(len(eval_data_loader)* eval_data_loader.batch_size)
 if epoch_loss < best_loss :
best_loss = epoch_loss
if epoch_loss < prev_loss:
torch.save(model.state_dict() , file_name)
 prev_loss = epoch_loss
print("*", end="")
 else:
print(".", end="")
 countdown = patience
else:
print("{:1}".format(countdown), end="")
 countdown -= 1
time_elapsed = time.time() - since
print("E{:3}/{:3} loss: {:.4f}({:3.0f}m {:2.0f}s)".format(
 epoch, num_epochs - 1, epoch_loss,time_elapsed // 60, time_elapsed % 60))
scheduler.step()
if countdown <= 0 : break
return prev_loss
print("done.")
CFG['batch_size'] = 256
cnt_test = count_data_items(files_test)
 steps = cnt_test /(CFG['batch_size'] * REPLICAS)* CFG['tta_steps']
ds_testAug = get_dataset(files_test, CFG, augment=True, repeat=True,
labeled=False, return_image_names=False)
 preds_B6 = model_B6.predict(ds_testAug, verbose=1, steps=steps )<create_dataframe>
batch_size = 56
num_round_per_fold = 2
for no in range(NUM_FOLDS):
print("-"*22, "fold",no)
 bst_loss = 10000.00
for r in range(num_round_per_fold):
print("-"*11,"round",r)
 data_loader_train, data_loader_eval = get_dataloader_for_fold(no,
DATA, data_train, data_eval, batch_size)
 model = get_base_model()
plist = [{"params": model.features.denseblock4.parameters() , "lr":0.0001},
{"params": model.classifier.parameters() }]
optimizer = optim.Adam(plist, lr=0.001, amsgrad=True)
 scheduler = optim.lr_scheduler.MultiStepLR(optimizer, [1,5], gamma=0.1, last_epoch=-1)
 bst_loss = train_model(model, optimizer, scheduler,
data_loader_train, data_loader_eval,
"tmp"+str(no)+".pth", prev_loss=bst_loss,
num_epochs=13, patience=3)
 print("-"*22, "best loss", bst_loss)
 print("")
<split>
def get_trained_model(no):

 extractor = torchvision.models.densenet161(pretrained=False)
 in_features = extractor.classifier.in_features
extractor.classifier = Classificator(in_features)
 model_path = os.path.join("tmp"+str(no)+".pth")
 extractor.load_state_dict(torch.load(model_path))
extractor = extractor.to(DEVICE)
 extractor.eval()
return extractor
preds = np.stack(preds_B0)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B0 = preds.reshape(-1 )<split>
def get_extractor_model(no):

 extractor = get_trained_model(no)
 extractor.classifier = nn.Identity()
extractor = extractor.to(DEVICE)
 extractor.eval()
return extractor
preds = np.stack(preds_B1)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B1 = preds.reshape(-1 )<define_variables>
def get_train_features(data_loader, extractor):

 for bi, d in enumerate(data_loader):
print(".", end="")
 img_tensor = d["image"].to(DEVICE)
 target = d["label"].numpy()
with torch.no_grad() : feature = extractor(img_tensor)
 feature = feature.cpu().detach().squeeze(0 ).numpy()
if bi == 0 :
features = feature
targets = target
else :
features = np.concatenate([features, feature], axis=0)
 targets = np.concatenate([targets, target], axis=0)
 return features, targets
<split>
XGBOOST_PARAM = {
"random_state" : 42,
"n_estimators" : 200,
"objective" : "multi:softmax",
"num_class" : 5,
"eval_metric" : "mlogloss",
}
preds = np.stack(preds_B3)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B3 = preds.reshape(-1 )<define_variables>
batch_size = 64
eval_set = []
for no in range(NUM_FOLDS):
print("-"*22, "fold",no)
 data_loader_train, data_loader_eval = get_dataloader_for_fold(no,
DATA, data_train, data_eval, batch_size)
 extractor = get_extractor_model(no)
 print("...........|.............................................|")
 features_eval, targets_eval = get_train_features(data_loader_eval,
extractor)
 features_train, targets_train = get_train_features(data_loader_train,
extractor)
 print("")
 xgb_model = xgb.XGBClassifier(**XGBOOST_PARAM)
 xgb_model = xgb_model.fit(features_train,targets_train.reshape(-1),
eval_set=[(features_eval, targets_eval.reshape(-1)) ],
early_stopping_rounds=20,
verbose=False)
 print("score",xgb_model.evals_result() ["validation_0"]["mlogloss"][-1])
 pickle.dump(xgb_model, open("xgb_model_"+str(no), "wb"))
<split>
base_transform = transforms.Compose([
transforms.Resize(224),
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 DATA.transform = base_transform
preds = np.stack(preds_B5)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B5 = preds.reshape(-1 )<split>
predictions, targets = np.zeros(len(DATA)) , np.zeros(len(DATA))
batch_slice =(0, 0)
 softmax = nn.Softmax(dim=1)
 for no in range(NUM_FOLDS):
_, data_loader_eval = get_dataloader_for_fold(no,
DATA, data_train, data_eval, 64)
 model = get_trained_model(no)
 for bi, d in enumerate(data_loader_eval):
inputs = d["image"].to(DEVICE, dtype=torch.float)
 batch_slice =(batch_slice[1], batch_slice[1]+inputs.size(0))
with torch.no_grad() :
outputs = model(inputs)
 outputs = softmax(outputs)
 predictions[batch_slice[0]:batch_slice[1]] = \
outputs.cpu().detach().squeeze(0 ).numpy().argmax(axis=1)
 targets[batch_slice[0]:batch_slice[1]] = d["label"]
print("Cohen Kappa quadratic score",
cohen_kappa_score(targets, predictions, weights="quadratic"))
preds = np.stack(preds_B6)
 preds = preds[:,:cnt_test* CFG['tta_steps']]
preds = preds[:df_test.shape[0]*CFG['tta_steps']]
preds = np.stack(np.split(preds, CFG['tta_steps']),axis=1)
 preds = np.mean(preds, axis=1)
 preds_B6 = preds.reshape(-1 )<define_variables>
predictions, targets = np.zeros(0), np.zeros(0)
 for no in range(NUM_FOLDS):
_, data_loader_eval = get_dataloader_for_fold(no,
DATA, data_train, data_eval, batch_size)
 features_eval, targets_eval = get_train_features(data_loader_eval,
extractor)
 print("")
 xgb_model = xgb.XGBClassifier()
model_path = os.path.join("xgb_model_"+str(no))
xgb_model = pickle.load(open(model_path, "rb"))
prediction = xgb_model.predict(features_eval)
 predictions = np.concatenate([predictions, prediction], axis=0)
 targets = np.concatenate([targets, targets_eval], axis=0)
 print("Cohen Kappa quadratic score",
cohen_kappa_score(targets, predictions, weights="quadratic"))
<prepare_output>
base_transform = transforms.Compose([
transforms.Resize(224),
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 data_test = RetinopathyDataset(base_transform, is_test=True)
 data_loader = torch.utils.data.DataLoader(data_test,
batch_size=16, shuffle=False,
num_workers=0, drop_last=False)
 def get_test_features(data_loader, extractor):

 for bi, d in enumerate(data_loader):
if bi % 4 == 0 : print(".", end="")
 img_tensor = d["image"].to(DEVICE)
 with torch.no_grad() : feature = extractor(img_tensor)
 feature = feature.cpu().detach().numpy()
if bi == 0 :
features = feature
else :
features = np.concatenate([features, feature], axis=0)
 return features
preds =preds_B0*0.2 + preds_B3*0.3 + preds_B5*0.2 + preds_B6*0.3<create_dataframe>
print("................................ v")
 predictions = np.zeros(( len(data_test),5))
for tta in range(1):
print("............ tta"+str(tta)+"................ ")
 for no in range(NUM_FOLDS):
extractor = get_extractor_model(no)
 features = get_test_features(data_loader, extractor)
 print("",no)
 xgb_model = xgb.XGBClassifier()
model_path = os.path.join("xgb_model_"+str(no))
xgb_model = pickle.load(open(model_path, "rb"))
prediction = xgb_model.predict_proba(features)
 predictions = predictions + prediction
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
batch_size = 8
data_augmentation = transforms.Compose([
transforms.Resize(( MODEL_SIZE, MODEL_SIZE)) ,
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.ToTensor() ,
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ])
 data_test = RetinopathyDataset(data_augmentation, is_test=True)
 data_loader = torch.utils.data.DataLoader(data_test,
batch_size=batch_size, shuffle=False,
num_workers=0, drop_last=False )
submission = pd.DataFrame(dict(
 image_name = image_names,
target = preds))
submission = submission.sort_values('image_name')
 submission.to_csv(f'submission.csv', index=False)
 submission.head()<load_from_csv>
softmax = nn.Softmax(dim=1)
 for tta in range(4):
print("............ tta"+str(tta)+"...............")
 for no in range(NUM_FOLDS):
model = get_trained_model(no)
 batch_slice =(0, 0)
 for bi, d in enumerate(data_loader):
if bi %(64//batch_size)== 0 : print(".", end="")
 img_tensor = d["image"].to(DEVICE, dtype=torch.float)
 batch_slice =(batch_slice[1], batch_slice[1]+img_tensor.size(0))
with torch.no_grad() :
feature = model(img_tensor)
 feature = softmax(feature)
 predictions[batch_slice[0]:batch_slice[1],:] += \
feature.cpu().detach().squeeze(0 ).numpy()
print("" )
<install_modules><EOS> 
prediction_final = predictions.argmax(axis=1)
 csv_file = os.path.join(DATA_SOURCE, "sample_submission.csv")
 df = pd.read_csv(csv_file)
 df["diagnosis"] = prediction_final
df.to_csv('submission.csv',index=False )