Kaggle competition: SIIM-ACR Pneumothorax Segmentation
<SOS> metric: SIIMDice Kaggle data source: siim-acr-pneumothorax-segmentation<choose_model_class>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
sys.path.insert(0, '.. /input/siim-acr-pneumothorax-segmentation')
def get_bn_model() :
model = Sequential([
Lambda(standardize, input_shape=(28, 28, 1)) ,
Convolution2D(32,(3, 3), activation='relu'),
BatchNormalization(axis=3),
Convolution2D(32,(3, 3), activation='relu'),
MaxPooling2D() ,
BatchNormalization(axis=3),
Convolution2D(64,(3, 3), activation='relu'),
BatchNormalization(axis=3),
Convolution2D(64,(3, 3), activation='relu'),
MaxPooling2D() ,
Flatten() ,
BatchNormalization() ,
Dense(512, activation='relu'),
BatchNormalization() ,
Dense(10, activation='softmax')
 ])
 model.compile(Adam() ,
loss='categorical_crossentropy',
metrics=['accuracy'])
 return model<train_model>
sz = 256
noise_th = 75.0*255*(sz/128.0)**2
nfolds = 4
MASKS = '.. /input/siimacr-pneumothorax-segmentation-data-256/masks'
DATA = '.. /input/hypercolumns-pneumothorax-fastai-0-819-lb'
preds0, items = [], []
for fold in range(nfolds):
preds0.append(torch.load(os.path.join(DATA,'preds_fold'+str(fold)+'.pt')))
 items.append(np.load(os.path.join(DATA,'items_fold'+str(fold)+'.npy'),
allow_pickle=True))
preds0 = torch.cat(preds0)
 items = np.concatenate(items)
 ys = []
for item in items:
img = cv2.imread(str(item ).replace('train', 'masks'),0)
 img = torch.Tensor(img ).byte().unsqueeze(0)
 img =(img > 127)
 ys.append(img)
 ys = torch.cat(ys )
model = get_bn_model()
history = model.fit_generator(generator=batches,
steps_per_epoch=len(batches),
epochs=30,
validation_data=val_batches,
validation_steps=len(val_batches))<save_to_csv>
def dice_overall(preds, targs):
n = preds.shape[0]
preds = preds.view(n, -1)
 targs = targs.view(n, -1)
 intersect =(preds * targs ).float().sum(-1)
 union =(preds+targs ).float().sum(-1)
 u0 = union==0
intersect[u0] = 1
union[u0] = 2
return(2.* intersect / union )
predictions = model.predict_classes(X_test, verbose=0)
 submissions = pd.DataFrame({'ImageID': list(range(1, len(predictions)+ 1)) ,
'Label': predictions})
 submissions.to_csv('cnn_batchnorm.csv', index=False, header=True )<load_from_csv>
TEST = '.. /input/siimacr-pneumothorax-segmentation-data-256/test/'
SAMPLE = '.. /input/siim-acr-pneumothorax-segmentation/sample_submission.csv'
items_test = Path(TEST ).ls()
ids_test = [p.stem for p in items_test]
preds_test = torch.load(os.path.join(DATA,'preds_test.pt'))
preds_test0 = preds_test.clone()
preds_test[preds_test.view(preds_test.shape[0],-1 ).float().sum(-1)< noise_th,...] = 0
def load_data() :
data_train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 X_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' ).values.reshape(-1,28,28,1)
 sample_submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
 X = data_train.iloc[:,1:].values.reshape(-1,28,28,1)
 y = data_train.iloc[:,0].values
np.random.seed(2)
 np.random.shuffle(X)
 np.random.seed(2)
 np.random.shuffle(y)
 number_train = int(0.9*(X.shape[0]))
X_train = X[:number_train]
y_train = y[:number_train]
X_dev = X[number_train:]
y_dev=y[number_train:]
mean = X_train.mean()
std = X_train.std()
X_train =(X_train-mean)/std
X_dev =(X_dev-mean)/std
X_test =(X_test-mean)/std
return X_train, y_train, X_dev,y_dev, X_test
def submit(model):
pred=model.predict(X_test ).argmax(axis=1)
 df = pd.DataFrame({"ImageId": list(range(1,pred.shape[0]+1)) ,
"Label": pred})
 df.to_csv("submission_neuralnet.csv",index=False,header=True)
 X_train, y_train, X_dev, y_dev, X_test = load_data()<choose_model_class>
preds_t =(preds_test>best_thr ).long().numpy()
rles = []
for p in progress_bar(preds_t):
if(p.sum() > 0):
im = PIL.Image.fromarray(( p.T*255 ).astype(np.uint8)).resize(( 1024,1024))
im = np.asarray(im)
 rles.append(mask2rle(im, 1024, 1024))
else: rles.append('-1')
 sub_df = pd.DataFrame({'ImageId': ids_test, 'EncodedPixels': rles})
 sub_df.to_csv('submission.csv', index=False)
 print(len(sub_df))
sub_df.head()
model = Sequential([Conv2D(128,3, activation='relu',input_shape=(28,28,1)) ,
Conv2D(128,3, activation='relu'),
MaxPool2D() ,
Conv2D(128,3, activation='relu'),
BatchNormalization() ,
Dropout(0.25),
Conv2D(128,3, activation='relu'),
MaxPool2D() ,
Flatten() ,
Dense(512,activation='relu'),
BatchNormalization() ,
Dropout(0.25),
Dense(256,activation='relu'),
Dense(10,activation='softmax')
 ])
 model.compile(RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0),
loss=tf.keras.losses.SparseCategoricalCrossentropy() ,metrics=['sparse_categorical_accuracy'])
 lr_reducer = ReduceLROnPlateau(monitor='val_sparse_categorical_accuracy', patience=3,verbose=1,factor=0.5,min_lr=0.00001)
 plot_model(model, show_shapes=True, show_layer_names=True )<train_model>
def split_mask(mask):
threshold_obj = 30
labled,n_objs = ndimage.label(mask > best_thr)
 result = []
n_pixels = []
for i in range(n_objs):
obj =(labled == i + 1 ).astype(int)
 if(obj.sum() > threshold_obj):
result.append(obj)
 n_pixels.append(obj.sum())
 result = [x for _,x in sorted(zip(n_pixels,result),reverse=True,key=lambda x:x[0])]
return result
data_gen = ImageDataGenerator(rotation_range=15,zoom_range = 0.1,width_shift_range=0.1,height_shift_range=0.1)
 data_gen.fit(X_train)
 fig, axs = plt.subplots(3,5)
 axs[0,0].imshow(X_train[0].squeeze())
 axs[0,1].imshow(X_train[1].squeeze())
 result = data_gen.flow(X_train[[0,1]],y_train[[0,1]],batch_size=2)
 i=0
for x,y in result:
axs[1,i].imshow(x[0].squeeze())
 axs[2,i].imshow(x[1].squeeze())
 i=i+1
if i == 5:
break<train_model>
rles = []
ids_c = []
for p,idx in progress_bar(zip(preds_test,ids_test), total=len(preds_test)) :
if(p.sum() > 0):
masks = split_mask(to_np(p))
for mask in masks:
ids_c.append(idx)
 im = PIL.Image.fromarray(( mask.T*255 ).astype(np.uint8)).resize(( 1024,1024))
im = np.asarray(im)
 rles.append(mask2rle(im, 1024, 1024))
if len(masks)== 0:
rles.append('-1')
 ids_c.append(idx)
 else:
rles.append('-1')
 ids_c.append(idx)
 sub_df = pd.DataFrame({'ImageId': ids_c, 'EncodedPixels': rles})
 sub_df.to_csv('submission_split.csv', index=False)
 print(len(sub_df))
sub_df.head()