Kaggle competition: Ultrasound Nerve Segmentation
<SOS> metric: Dice Kaggle data source: ultrasound-nerve-segmentation<train_model>
model = KerasRegressor(build_fn=get_model, size1=20, size2=0, size3=0, batch_size=16, verbose=2)
 estimators = fit(X_train.values, y_train, folds=10)
 print('RMSE:', np.sqrt(mean_score()))<load_from_csv>
!pip install segmentation_models_pytorch
! git clone https://github.com/Bjarten/early-stopping-pytorch.git
! mv./early-stopping-pytorch./lib
test_df = pd.read_csv('.. /input/test.csv')
 X_test = prepare_X(np.log1p(test_df.drop(['ID'], axis=1)))
 print(X_test.shape )<save_to_csv>
import torch
from torch import nn
from torch.utils.data import Dataset, DataLoader, sampler
from pathlib import Path
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import time
import torchvision
import cv2
import re
import segmentation_models_pytorch as smp
from lib.pytorchtools import *
sub = pd.read_csv('.. /input/sample_submission.csv')
 sub["target"] = np.expm1(predict(X_test.values))
print(sub.head())
 sub.to_csv('sub_keras_{:.2f}.csv'.format(np.sqrt(mean_score())) , index=False )<set_options>
class NerveDataset(Dataset):
def __init__(self, directory, pytorch=True, is_test=False):
super().__init__()
self.files = []
for file_name in directory.iterdir() :
if 'mask' in str(file_name):
continue
self.files.append(self.combine_files(file_name))
self.files = sorted(self.files, key=lambda file: int(re.search(r'\d+', str(file['image'])).group(0)))
 self.pytorch = pytorch
self.resize = torchvision.transforms.Resize(( 224,224),interpolation=Image.NEAREST)
 self.is_test = is_test
def combine_files(self, file_name: Path):
files = {
'image': file_name,
'mask': '.. ' + str(file_name ).split('.')[2] + '_mask.tif',
}
return files
def __len__(self):
return len(self.files)
 def open_as_array(self, idx, invert=False):
raw_image = self.resize(Image.open(self.files[idx]['image']))
raw_image = raw_image = np.stack([ np.array(raw_image)], axis=2)
 if invert:
raw_image = raw_image.transpose(( 2,0,1))
return(raw_image / np.iinfo(raw_image.dtype ).max)
 def open_mask(self, idx, add_dims=False):
raw_mask = self.resize(Image.open(self.files[idx]['mask']))
raw_mask = np.array(raw_mask)
 raw_mask = np.where(raw_mask==255, 1, 0)
 return np.expand_dims(raw_mask, 0)if add_dims else raw_mask
def __getitem__(self, idx):
x = torch.tensor(self.open_as_array(idx, invert=self.pytorch), dtype=torch.float32)
 if not self.is_test:
y = torch.tensor(self.open_mask(idx, add_dims=True), dtype=torch.torch.float32)
 return x, y
return x
def open_as_pil(self, idx):
arr = 256*self.open_as_array(idx)
 return Image.fromarray(arr.astype(np.uint8), 'L')
 def __repr__(self):
s = 'Dataset class with {} files'.format(self.__len__())
 return s
%matplotlib inline 
<load_from_csv>
images_path = Path('.. /input/ultrasound-nerve-segmentation/train')
 data = NerveDataset(images_path)
 len(data )
train=pd.read_csv('.. /input/train.csv')
 test=pd.read_csv('.. /input/test.csv' )<count_missing_values>
split_rate = 0.7
train_ds_len = int(len(data)* split_rate)
 valid_ds_len = len(data)- train_ds_len
train_ds, valid_ds = torch.utils.data.random_split(data,(train_ds_len, valid_ds_len))
print(f'Train dataset length: {len(train_ds)}
')
 print(f'Validation dataset length: {len(valid_ds)}
')
 print(f'All data length: {len(data)}
' )
train.isnull().sum().sort_values(ascending=False )<groupby>
unet = model = smp.Unet(
 encoder_name="resnet34",
encoder_weights="imagenet",
in_channels=1,
classes=1,
activation = "sigmoid" 
)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 learning_rate = 0.001
epochs = 50
metrics = [smp.utils.metrics.IoU() ]
loss_function = smp.utils.losses.DiceLoss()
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate)
 scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, epochs)
 stopper = EarlyStopping(patience=3)
 train_epoch = smp.utils.train.TrainEpoch(model,
loss=loss_function,
optimizer=optimizer,
metrics=metrics,
device=device,
verbose=True)
 val_epoch = smp.utils.train.ValidEpoch(model,
loss=loss_function,
metrics=metrics,
device=device,
verbose=True)
 train_dl = DataLoader(train_ds, batch_size=16, shuffle=True)
 valid_dl = DataLoader(valid_ds, batch_size=16, shuffle=True )
dtype_df = train.dtypes.reset_index()
dtype_df.columns = ["Count", "Column Type"]
dtype_df.groupby("Column Type" ).aggregate('count' ).reset_index()<count_unique_values>
test_images_path = Path('.. /input/ultrasound-nerve-segmentation/test')
 test_data = NerveDataset(test_images_path, is_test=True)
 test_dl = DataLoader(test_data, batch_size=1, shuffle=False)
 len(test_data )
unique_df = train.nunique().reset_index()
unique_df.columns = ["col_name", "unique_count"] 
<drop_column>
def rle_encoding(x):
dots = np.where(x.T.flatten() ==1)[0]
run_lengths = []
prev = -2
for b in dots:
if(b>prev+1): run_lengths.extend(( b+1, 0))
run_lengths[-1] += 1
prev = b
return run_lengths
train.drop(columns=constant_df['col_name'],inplace=True)
 test.drop(columns=constant_df['col_name'],inplace=True )<prepare_x_and_y>
encodings = []
counter = 0
for image in test_dl:
if torch.cuda.is_available() :
image = image.cuda()
pr_mask = unet(image)
 pr_mask = pr_mask[0]
pr_mask = pr_mask.squeeze().cpu().detach().numpy().round().astype(np.uint8)
 pr_mask = cv2.resize(pr_mask,(580,420), interpolation=cv2.INTER_CUBIC)
 encodings.append(rle_encoding(pr_mask))
print(f'Image: {counter} ')
 counter += 1
<prepare_output><EOS> 
df_submission = pd.DataFrame(columns=["img", "pixels"])
 for i, encoding in enumerate(encodings):
pixels = ' '.join(map(str, encoding))
df_submission.loc[i] = [str(i+1), pixels]
df_submission.to_csv('./submission.csv', index=False)
 print('Done!' )