Kaggle competition: Ultrasound Nerve Segmentation
<SOS> metric: Dice Kaggle data source: ultrasound-nerve-segmentation<split>
! pip install segmentation_models_pytorch albumentations
! pip install -U git+https://github.com/albu/albumentations --no-cache-dir
trainSVD,testSVD = maxabs(trainSVD,testSVD)
 X_train, X_test, Y_train, Y_test = train_test_split(trainSVD, y_train, test_size=0.1, random_state=0)
 lgb_predSVD, model, evals_resultRF = run_lgb(X_train, Y_train, X_test, Y_test, testSVD )<split>
import os
import csv
import re
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import itertools
import cv2
from tqdm.notebook import tqdm
from glob import glob
from PIL import Image
from skimage.transform import resize
from sklearn.model_selection import train_test_split, KFold
import shutil
import torch
from torch.utils.data import DataLoader,Dataset
import torch.nn as nn
import albumentations
import torchvision
from torchvision import transforms, models
import random
import segmentation_models_pytorch as smp
from PIL import Image
trainRF = train.copy()
testRF = test.copy()
trainRF,testRF = rowagg(trainRF,testRF )<train_model>
ROOT = "/kaggle/input/ultrasound-nerve-segmentation/"
trainpath = "/kaggle/input/ultrasound-nerve-segmentation/train/"
testpath = "/kaggle/input/ultrasound-nerve-segmentation/test/"
masks = [os.path.join(trainpath,i)for i in os.listdir(trainpath)if "mask" in i]
imgs = [i.replace("_mask","")for i in masks]
df = pd.DataFrame({"image":imgs,"mask":masks})
 df_train, df_val = train_test_split(df,test_size = 0.15)
 print(df_train.values.shape)
 print(df_val.values.shape )
rf_clf=RandomForestRegressor(random_state=42,n_jobs=-1)
 rf_clf.fit(trainRF,y_train)
 rank = pd.DataFrame()
rank["importance"] = np.array(rf_clf.feature_importances_)
 rank["feature"] = np.array(trainRF.columns ).T
rank = rank.sort_values(by=['importance'], ascending=False)
 col = rank[:2000]<split>
def rle_encoding(x):
dots = np.where(x.T.flatten() ==1)[0]
run_lengths = []
prev = -2
for b in dots:
if(b>prev+1): run_lengths.extend(( b+1, 0))
run_lengths[-1] += 1
prev = b
return run_lengths
def convert_to_tensor(x,**kwargs):
return x.transpose(2,0,1 ).astype("float32")
 def func_for_preprocessing(preprocessing_fn=None):
transform = []
if preprocessing_fn:
transform.append(albumentations.Lambda(image=preprocessing_fn))
transform.append(albumentations.Lambda(image=convert_to_tensor))
return albumentations.Compose(transform)
 def trainaugs() :
transform = [
albumentations.Resize(height=224,width=224,interpolation=Image.BILINEAR),
albumentations.RandomBrightnessContrast(brightness_limit=0.15, contrast_limit=0),
albumentations.ShiftScaleRotate(rotate_limit=15, shift_limit=0.15, scale_limit=0.2),
albumentations.HorizontalFlip() ,
]
return albumentations.Compose(transform)
 def valaugs() :
transform = [
albumentations.Resize(height=224,width=224,interpolation=Image.BILINEAR),
]
return albumentations.Compose(transform)
 class GetDataset(Dataset):
def __init__(self,imagespath,maskspath,augment=None,preprocess=None):
self.imagespath = imagespath
self.maskspath = maskspath
self.augment = augment
self.preprocess = preprocess
def __len__(self):
return len(self.imagespath)
 def __getitem__(self,idx):
image = cv2.cvtColor(cv2.imread(self.imagespath[idx]),cv2.COLOR_BGR2RGB)
 mask = cv2.imread(self.maskspath[idx], cv2.IMREAD_GRAYSCALE)
 if self.augment:
sample = self.augment(image=image, mask=mask)
 image,mask = sample['image'],sample['mask']
if self.preprocess:
sample = self.preprocess(image=image,mask=mask)
 image,mask = sample['image'],sample['mask']
mask =(mask / 255 ).astype(np.float32)
 mask = np.expand_dims(mask, axis=0)
 return image,mask
trainRF=trainRF[col.feature]
testRF=testRF[col.feature]
trainRF,testRF = maxabs(trainRF,testRF)
 X_train, X_test, Y_train, Y_test = train_test_split(trainRF, y_train, test_size=0.1, random_state=0)
 lgb_predRF, model, evals_resultRF = run_lgb(X_train, Y_train, X_test, Y_test, testRF )<save_to_csv>
encoder = "resnet34"
encoder_wts = "imagenet"
activation = "sigmoid"
model = smp.Unet(encoder_name=encoder,activation=activation,encoder_weights=encoder_wts)
 preprocess_func = smp.encoders.get_preprocessing_fn(encoder,encoder_wts)
 traindata = GetDataset(imagespath = df_train['image'].tolist() ,
maskspath = df_train['mask'].tolist() ,
augment = trainaugs() ,
preprocess = func_for_preprocessing(preprocess_func))
validationdata = GetDataset(imagespath = df_val['image'].tolist() ,
maskspath = df_val['mask'].tolist() ,
augment = valaugs() ,
preprocess = func_for_preprocessing(preprocess_func))
batch_size = 16
trainloader = DataLoader(traindata,batch_size = batch_size,shuffle=True)
 valloader = DataLoader(validationdata,batch_size=batch_size,shuffle=False)
sub=pd.read_csv('.. /input/sample_submission.csv')
 sub["target"] = lgb_predRF
sub.to_csv('sub.csv', index=False )<import_modules>
trainmodel = True
epochs = 20
device = "cuda"
loss = smp.utils.losses.DiceLoss()
metrics = [ smp.utils.metrics.IoU(threshold=0.5)]
optimizer = torch.optim.Adam([dict(params=model.parameters() , lr=0.001)])
 lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, epochs)
 trainepoch = smp.utils.train.TrainEpoch(model,loss=loss,optimizer=optimizer,metrics=metrics,device=device,verbose=True)
 validepoch = smp.utils.train.ValidEpoch(model,loss=loss,metrics=metrics,device=device,verbose=True)
import random
import numpy as np
import pandas as pd
from sklearn.base import clone
from keras.models import Sequential, load_model
from keras.layers import Dense, BatchNormalization, Dropout
from keras.callbacks import EarlyStopping, TensorBoard
from keras.wrappers.scikit_learn import KerasRegressor
from sklearn.model_selection import cross_val_score, check_cv, KFold, train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from subprocess import check_output
from IPython.display import display<set_options>
best_iou_score = 0.0
train_logs_list, valid_logs_list = [], []
for i in range(0,epochs):
print('
Epoch: {}'.format(i))
trainlogs = trainepoch.run(trainloader)
 validlogs = validepoch.run(valloader)
 lr_scheduler.step()
train_logs_list.append(trainlogs)
 valid_logs_list.append(validlogs)
 if best_iou_score < validlogs['iou_score']:
best_iou_score = validlogs['iou_score']
torch.save(model, './best_model.pth' )
warnings.filterwarnings('ignore')
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
random_seed = 42
random.seed = random_seed
np.random.seed = random_seed
tf.set_random_seed(random_seed )<load_from_csv>
train_logs_df = pd.DataFrame(train_logs_list)
 valid_logs_df = pd.DataFrame(valid_logs_list)
 train_logs_df.T
train_df = pd.read_csv('.. /input/train.csv' )<define_variables>
best_model = torch.load('./best_model.pth')
 test_dataset = GetDataset(imagespath = df_val['image'].tolist() ,
maskspath = df_val['mask'].tolist() ,
augment = valaugs() ,
preprocess = func_for_preprocessing(preprocess_func))
test_dataset_vis = GetDataset(imagespath = df_val['image'].tolist() ,
maskspath = df_val['mask'].tolist())
 def visualize(**images):
n = len(images)
 plt.figure(figsize=(16, 5))
for i,(name, image)in enumerate(images.items()):
plt.subplot(1, n, i + 1)
 plt.xticks([])
 plt.yticks([])
 plt.title(' '.join(name.split('_')).title())
 if image.shape[0] == 3:
image = image.transpose([1, 2, 0])
 plt.imshow(image)
 plt.show()
for i in range(25):
n = np.random.choice(len(test_dataset))
image_vis = test_dataset_vis[n][0].astype('uint8')
 mask_vis = test_dataset_vis[n][1].astype('uint8')
 image, gt_mask = test_dataset[n]
gt_mask = gt_mask.squeeze()
x_tensor = torch.from_numpy(image ).to(device ).unsqueeze(0)
 pr_mask = best_model.predict(x_tensor)
 pr_mask = pr_mask.squeeze().cpu().numpy().round()
kernel = np.ones(( 5,5),np.uint8)
 pr_mask_er = cv2.erode(pr_mask,kernel,iterations = 4)
 pr_mask_er = cv2.dilate(pr_mask_er,kernel,iterations = 4)
 pr_mask = cv2.resize(pr_mask,(580, 420))
pr_mask_er = cv2.resize(pr_mask_er,(580, 420))
mask_vis = mask_vis.squeeze()
visualize(
 image=image_vis,
ground_truth_mask=mask_vis,
predicted_mask=pr_mask,
predicted_mask_erosion_with_dilation=pr_mask_er
)
cols = ['f190486d6', '58e2e02e6', 'eeb9cd3aa', '9fd594eec', '6eef030c1',
'15ace8c9f', 'fb0f5dbfe', '58e056e12', '20aa07010', '024c577b9',
'd6bb78916', 'b43a7cfd5', '58232a6fb', '1702b5bf0', '324921c7b',
'62e59a501', '2ec5b290f', '241f0f867', 'fb49e4212', '66ace2992',
'f74e8f13d', '5c6487af1', '963a49cdc', '26fc93eb7', '1931ccfdd',
'703885424', '70feb1494', '491b9ee45', '23310aa6f', 'e176a204a',
'6619d81fc', '1db387535', 'fc99f9426', '91f701ba2', '0572565c2',
'190db8488', 'adb64ff71', 'c47340d97', 'c5a231d81', '0ff32eb98']<prepare_x_and_y>
imgs = [f for f in os.listdir(testpath)]
imgs = sorted(imgs, key=lambda s: int(s.split('.')[0]))
encodings = []
for m in tqdm(imgs):
x = cv2.imread(os.path.join(testpath, m))
x = valaugs()(image=x)['image']
x = func_for_preprocessing(preprocess_func )(image=x)['image']
x_tensor = torch.from_numpy(x ).to(device ).unsqueeze(0)
 pr_mask = best_model.predict(x_tensor)
 pr_mask = pr_mask.squeeze().cpu().numpy().round().astype(np.uint8)
 pr_mask = albumentations.Resize(height=420,width=580,interpolation=Image.NEAREST )(image=pr_mask)['image']
encodings.append(rle_encoding(pr_mask))
<choose_model_class><EOS> 
df_submission = pd.DataFrame(columns=["img", "pixels"])
 for i, encoding in enumerate(encodings):
pixels = ' '.join(map(str, encoding))
df_submission.loc[i] = [str(i+1), pixels]
df_submission.to_csv('submission.csv', index=False )