Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<init_hyperparams>
t_start = time.time()
warnings.filterwarnings('always')
 warnings.filterwarnings('ignore')
 %matplotlib inline
style.use('fivethirtyeight')
 sns.set(style='whitegrid', color_codes=True)
 !ls.. /input/*
num_epochs = 25
num_classes = 2
batch_size = 128
learning_rate = 0.002<set_options>
fastai.__version__
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu' )<split>
def is_interactive() :
return 'runtime' in get_ipython().config.IPKernelApp.connection_file
print('Interactive?', is_interactive() )
train, val = train_test_split(labels,
stratify=labels.has_cactus,
test_size=0.3)
 train.shape, val.shape<categorify>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(42 )
class Cactus(Dataset):
def __init__(self, df_data, data_dir = './', transform=None):
super().__init__()
self.df = df_data.values
self.data_dir = data_dir
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
img_name,label = self.df[index]
img_path = os.path.join(self.data_dir, img_name)
 image = cv2.imread(img_path)
 if self.transform is not None:
image = self.transform(image)
 return image, label<categorify>
Path('/tmp/.cache/torch/checkpoints/' ).mkdir(exist_ok=True, parents=True)
 model_path = '/tmp/.cache/torch/checkpoints/efficientNet.pth'
!cp.. /input/efficientnet*/efficientNet_*.pth {model_path}
trans_train = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(32, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])])
 trans_valid = transforms.Compose([transforms.ToPILImage() ,
transforms.Pad(32, padding_mode='reflect'),
transforms.ToTensor() ,
transforms.Normalize(mean=[0.5, 0.5, 0.5],std=[0.5, 0.5, 0.5])] )<create_dataframe>
PATH = Path('.. /input/aptos2019-blindness-detection')
 PATH_train = Path('.. /input/drd-newold/drd')
 df_train = pd.read_csv('.. /input/oldandnew/new_train_data.csv')
 df_test = pd.read_csv(PATH/'test.csv')
 _ = df_train.hist()
dataset_train = Cactus(df_data=train,
data_dir=train_path,
transform=trans_train)
 dataset_valid = Cactus(df_data=val,
data_dir=train_path,
transform=trans_valid)
 loader_train = DataLoader(dataset = dataset_train,
batch_size=batch_size,
shuffle=True,
num_workers=0)
 loader_valid = DataLoader(dataset = dataset_valid,
batch_size=batch_size//2,
shuffle=False,
num_workers=0 )<choose_model_class>
aptos19_stats =([0.42, 0.22, 0.075], [0.27, 0.15, 0.081])
 data = ImageDataBunch.from_df(df=df_train,
path=PATH_train, folder='aptos_drd_jpeg', suffix='.jpeg',
valid_pct=0.1,
ds_tfms=get_transforms(flip_vert=True, max_warp=0.05, max_rotate=20.) ,
bs=4,
num_workers=os.cpu_count()
).normalize(aptos19_stats )
class SimpleCNN(nn.Module):
def __init__(self):
super(SimpleCNN, self ).__init__()
self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=2)
 self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=2)
 self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, padding=2)
 self.conv4 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, padding=2)
 self.conv5 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, padding=2)
 self.bn1 = nn.BatchNorm2d(32)
 self.bn2 = nn.BatchNorm2d(64)
 self.bn3 = nn.BatchNorm2d(128)
 self.bn4 = nn.BatchNorm2d(256)
 self.bn5 = nn.BatchNorm2d(512)
 self.pool = nn.MaxPool2d(kernel_size=2, stride=2)
 self.avg = nn.AvgPool2d(4)
 self.fc = nn.Linear(512 * 1 * 1, 2)
 def forward(self, x):
x = self.pool(F.leaky_relu(self.bn1(self.conv1(x))))
x = self.pool(F.leaky_relu(self.bn2(self.conv2(x))))
x = self.pool(F.leaky_relu(self.bn3(self.conv3(x))))
x = self.pool(F.leaky_relu(self.bn4(self.conv4(x))))
x = self.pool(F.leaky_relu(self.bn5(self.conv5(x))))
x = self.avg(x)
 x = x.view(-1, 512 * 1 * 1)
 x = self.fc(x)
 return x<choose_model_class>
print(f'Classes:
{data.classes}' )
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adamax(model.parameters() , lr=learning_rate )<choose_model_class>
data.show_batch(rows=3, figsize=(7,6))
total_step = len(loader_train)
 for epoch in range(num_epochs):
for i,(images, labels)in enumerate(loader_train):
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 loss = criterion(outputs, labels)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
if(i+1)% 100 == 0:
print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}' 
.format(epoch+1, num_epochs, i+1, total_step, loss.item()))<find_best_params>
package_path = '.. /input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master'
sys.path.append(package_path)
model.eval()
with torch.no_grad() :
correct = 0
total = 0
for images, labels in loader_valid:
images = images.to(device)
 labels = labels.to(device)
 outputs = model(images)
 _, predicted = torch.max(outputs.data, 1)
 total += labels.size(0)
 correct +=(predicted == labels ).sum().item()
print('Test Accuracy of the model on the 1750 validation images: {} %'.format(100 * correct / total))
torch.save(model.state_dict() , 'model.ckpt' )<load_pretrained>
def EfficientNetB4(pretrained=True):

 model = EfficientNet.from_name('efficientnet-b4', override_params={'num_classes': 5 })
 if pretrained:
model_state = torch.load(model_path)
 if '_fc.weight' in model_state.keys() :
model_state.pop('_fc.weight')
 model_state.pop('_fc.bias')
 res = model.load_state_dict(model_state, strict=False)
 assert str(res.missing_keys)== str(['_fc.weight', '_fc.bias']), 'issue loading pretrained weights'
else:
mapping = { i:o for i,o in zip(model_state.keys() , model.state_dict().keys())}
mapped_model_state = OrderedDict([ 
(mapping[k], v)for k,v in model_state.items() if not mapping[k].startswith('_fc')
 ])
 res = model.load_state_dict(mapped_model_state, strict=False)
 print(res)
 return model
dataset_valid = Cactus(df_data=sub, data_dir=test_path, transform=trans_valid)
 loader_test = DataLoader(dataset = dataset_valid, batch_size=32, shuffle=False, num_workers=0 )<save_to_csv>
model = EfficientNetB4(pretrained=True)
model.eval()
preds = []
for batch_i,(data, target)in enumerate(loader_test):
data, target = data.cuda() , target.cuda()
output = model(data)
 pr = output[:,1].detach().cpu().numpy()
for i in pr:
preds.append(i)
 sub['has_cactus'] = preds
sub.to_csv('sub.csv', index=False )<load_from_csv>
class FocalLoss(nn.Module):
def __init__(self, gamma=3., reduction='mean'):
super().__init__()
self.gamma = gamma
self.reduction = reduction
def forward(self, inputs, targets):
CE_loss = nn.CrossEntropyLoss(reduction='none' )(inputs, targets)
 pt = torch.exp(-CE_loss)
 F_loss =(( 1 - pt)**self.gamma)* CE_loss
if self.reduction == 'sum':
return F_loss.sum()
elif self.reduction == 'mean':
return F_loss.mean()
df = pd.DataFrame.from_csv('.. /input/aerial-cactus-identification/train.csv', index_col=None)
 df.head()<define_variables>
class OverSamplingCallback(LearnerCallback):
def __init__(self,learn:Learner, weights:torch.Tensor=None):
super().__init__(learn)
 self.labels = self.learn.data.train_dl.dataset.y.items
_, counts = np.unique(self.labels, return_counts=True)
 self.weights =(weights if weights is not None else
torch.DoubleTensor(( 1/counts)[self.labels]))
def on_train_begin(self, **kwargs):
self.learn.data.train_dl.dl.batch_sampler = BatchSampler(
 WeightedRandomSampler(self.weights, len(self.learn.data.train_dl.dataset)) ,
self.learn.data.train_dl.batch_size,False )
makedirs('/tmp/.torch/models/' )<load_pretrained>
learn = Learner(data, model,
loss_func=FocalLoss() ,
metrics=[accuracy, KappaScore(weights="quadratic")],
callback_fns=[BnFreeze,
partial(SaveModelCallback, monitor='kappa_score', name='best_kappa')] 
)
 learn.split(lambda m:(model._conv_head,))
learn.freeze()
learn.model_dir = '/tmp/'
test_data = ImageList.from_folder('.. /input/aerial-cactus-identification/test/test')
 trfm = get_transforms(do_flip=True, flip_vert=True, max_rotate=10.0, max_zoom=1.1, max_lighting=0.2, max_warp=0.2, p_affine=0.75, p_lighting=0.75)
 data =(ImageList.from_df(df, path='.. /input/aerial-cactus-identification/train/train')
.split_by_rand_pct(0.1)
.label_from_df() 
.add_test(test_data)
.transform(trfm, size=128)
.databunch(path='.', bs=64, device= torch.device('cuda:0')) 
.normalize(imagenet_stats)
 )<choose_model_class>
learn.freeze()
learn.lr_find(start_lr=1e-6, end_lr=1e1, wd=5e-3)
 learn.recorder.plot(suggestion=True )
data.path = pathlib.Path('.')
 learn = cnn_learner(data, models.resnet18, metrics=accuracy)
 learn.lr_find()<train_model>
learn.fit_one_cycle(2, max_lr=3e-2, div_factor=10, final_div=100, wd=5e-3)
 learn.save('stage-1')
 learn.recorder.plot_losses()
learn.fit_one_cycle(5, 1e-02 )<predict_on_test>
learn.unfreeze()
preds = learn.get_preds(ds_type=DatasetType.Test)
 preds = [x[1] for x in preds[0].tolist() ]
preds<define_variables>
learn.fit_one_cycle(cyc_len=5, max_lr=slice(5e-5, 5e-4), pct_start=0, wd=1e-3)
 learn.save('stage-3')
pred_df = []
for item in zip(data.test_ds.items, preds):
pred_df.append([item[0].stem + '.jpg', item[1]] )<create_dataframe>
tta_params = {'beta':0.12, 'scale':1.0}
pred_df = pd.DataFrame(pred_df, columns=['id', 'has_cactus'] )<save_to_csv>
sample_df = pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv')
 sample_df.head()
pred_df.to_csv('submission.csv', index=False )<set_options>
learn.data.add_test(ImageList.from_df(
 sample_df, PATH,
folder='test_images',
suffix='.png' 
))
plt.style.use('ggplot' )<load_from_csv>
preds,y = learn.TTA(ds_type=DatasetType.Test, **tta_params )
train = pd.read_csv('.. /input/train.csv', nrows = 5000000 )<filter>
sample_df.diagnosis = preds.argmax(1)
 sample_df.head()
train = train[train.fare_amount > 0]<load_from_csv>
sample_df.to_csv('submission.csv',index=False)
 _ = sample_df.hist()
<filter><EOS> 
sample_df['diagnosis'].value_counts()