Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<define_search_model>
!pip install.. /input/keras-applications/Keras_Applications-1.0.8/ -f./ --no-index
!pip install.. /input/image-classifiers/image_classifiers-1.0.0/ -f./ --no-index
!pip install.. /input/efficientnet-1-0-0/efficientnet-1.0.0/ -f./ --no-index
!pip install.. /input/segmentation-models/segmentation_models-1.0.1/ -f./ --no-index
init_lr = 0.002
power = 2.0
adam_opt = Adam(
 lr=init_lr,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0 
)
 def polynomial_decay(epoch):
max_epochs = num_epochs
return init_lr *(1 -(epoch / float(max_epochs)))** power<train_model>
%env SM_FRAMEWORK=tf.keras
batch_size = 128
print('
model.compile(
 loss='categorical_crossentropy',
optimizer=adam_opt,
metrics=['accuracy'] 
)
 print('
h = model.fit_generator(
 daug.flow(X_train, y_train, batch_size=batch_size),
validation_data=(X_ttest, y_ttest),
epochs=num_epochs,
steps_per_epoch=len(X_train)// batch_size,
callbacks=callbacks,
verbose=1
)<predict_on_test>
warnings.filterwarnings('ignore')
 print('tensorflow version:', tf.__version__)
 os.environ['CUDA_VISIBLE_DEVICES'] = '0'
gpu_devices = tf.config.experimental.list_physical_devices('GPU')
 if gpu_devices:
for gpu_device in gpu_devices:
print('device available:', gpu_device)
 pd.set_option('display.max_columns', None )
label_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
print('Confusion matrix:')
 preds = model.predict(X_ttest, batch_size=batch_size)
 print(classification_report(y_ttest.argmax(axis=1),
preds.argmax(axis=1), target_names=label_names))<load_from_csv>
TEST = True
KAGGLE = True
MDLS_FOLDS = {'v39': [0, 2, 3, 4]}
if KAGGLE:
DATA_PATH = '.. /input/hubmap-kidney-segmentation'
MDLS_PATHS = {ver: f'.. /input/kidney-models-{ver}'
for ver, _ in MDLS_FOLDS.items() }
else:
DATA_PATH = './data2'
MDLS_PATHS = {ver: f'./models_{ver}'
for ver, _ in MDLS_FOLDS.items() }
THRESHOLD =.35
VOTERS = 1
TTAS = [0, 1]
EXPAND = 4
MIN_OVERLAP = 1 / 16
IDNT = rasterio.Affine(1, 0, 0, 0, 1, 0)
 STRATEGY = tf.distribute.get_strategy()
SUB_PATH = f'{DATA_PATH}/test' if TEST else f'{DATA_PATH}/train'
start_time = time.time()
sub_X_test = pd.read_csv(os.path.join(base_dir, 'test.csv'))
sub_X_test = sub_X_test.drop('id', axis=1)
 sub_X_test = sub_X_test.iloc[:,:].values
sub_X_test = sub_X_test.reshape(sub_X_test.shape[0], S, S, C)
 sub_X_test = sub_X_test / 255.0
preds = model.predict_classes(sub_X_test, batch_size=batch_size)
 id_col = np.arange(preds.shape[0])
 submission = pd.DataFrame({'id': id_col, 'label': preds})
 print(pd.DataFrame(submission))
submission.to_csv('submission.csv', index = False )<import_modules>
params_dict = {}
for ver, _ in MDLS_FOLDS.items() :
with open(f'{MDLS_PATHS[ver]}/params.json')as file:
params_dict[ver] = json.load(file)
 for ver, params in params_dict.items() :
print('version:', ver, '| loaded params:', params, '
' )
train = pd.read_csv(".. /input/Kannada-MNIST/train.csv")
 test = pd.read_csv(".. /input/Kannada-MNIST/test.csv")
 dig_test = pd.read_csv(".. /input/Kannada-MNIST/Dig-MNIST.csv")
 Y_train = train["label"]
Y_dig = dig_test["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 test = test.drop(labels=['id'], axis=1)
 dig_test = dig_test.drop(labels = ["label"],axis = 1)
 del train
print(Y_train.value_counts())
 X_train = X_train / 255.0
test = test / 255.0
dig_test = dig_test / 255.0
X_train = X_train.values.reshape(-1,28,28,1)
 test = test.values.reshape(-1,28,28,1)
 dig_test = dig_test.values.reshape(-1,28,28,1)
 Y_train = to_categorical(Y_train, num_classes = 10)
 Y_dig = to_categorical(Y_dig, num_classes=10)
 X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=2)
<choose_model_class>
def enc2mask(encs, shape):
img = np.zeros(shape[0] * shape[1], dtype=np.uint8)
 for m, enc in enumerate(encs):
if isinstance(enc, np.float)and np.isnan(enc): continue
s = enc.split()
for i in range(len(s)// 2):
start = int(s[2 * i])- 1
length = int(s[2 * i + 1])
 img[start : start + length] = 1 + m
return img.reshape(shape ).T
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 64, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(3,3), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(3,3), strides=(2,2)))
 model.add(Conv2D(filters = 128, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(64, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(32, activation = "relu"))
model.add(Dense(10, activation = "softmax"))
model.compile(optimizer = 'Adam' , loss = "categorical_crossentropy", metrics=["accuracy"])
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001)
 epochs = 30
batch_size = 124
TensorBoardcallback = TensorBoard(
 log_dir='./logs',
histogram_freq=1, batch_size=32,
write_graph=True, write_grads=False, write_images=True,
embeddings_freq=0, embeddings_layer_names=None,
embeddings_metadata=None, embeddings_data=None, update_freq=epochs 
)
 datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1, 
)
 datagen.fit(X_train)
 history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction, TensorBoardcallback],use_multiprocessing=False)
 fig, ax = plt.subplots(2,1)
 ax[0].plot(history.history['loss'], color='b', label="Training loss")
 ax[0].plot(history.history['val_loss'], color='r', label="validation loss",axes =ax[0])
 ax[0].legend(loc='best', shadow=False)
 ax[1].plot(history.history['accuracy'], color='b', label="Training accuracy")
 ax[1].plot(history.history['val_accuracy'], color='r',label="Validation accuracy")
 ax[1].legend(loc='best', shadow=True)
 plt.show()<save_to_csv>
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2 * intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred, smooth=1):
return(1 - dice_coef(y_true, y_pred, smooth))
def bce_dice_loss(y_true, y_pred):
return params['bce_weight'] * binary_crossentropy(y_true, y_pred)+ \ 
(1 - params['bce_weight'])* dice_loss(y_true, y_pred)
 def get_model(backbone, input_shape, path,
loss_type='bce_dice', umodel='unet',
classes=1, lr=.001):
if backbone == 'efficientnetb0':
weights = f'{path}/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
elif backbone == 'efficientnetb1':
weights = f'{path}/efficientnet-b1_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
elif backbone == 'efficientnetb2':
weights = f'{path}/efficientnet-b2_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
elif backbone == 'efficientnetb3':
weights = f'{path}/efficientnet-b3_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
elif backbone == 'resnet34':
weights = f'{path}/resnet34_imagenet_1000_no_top.h5'
else:
raise AttributeError('mode parameter error')
 with STRATEGY.scope() :
if loss_type == 'bce_dice':
loss = bce_dice_loss
elif loss_type == 'bce_jaccard_loss':
loss = bce_jaccard_loss
else:
raise AttributeError('loss mode parameter error')
 if umodel == 'unet':
model = Unet(backbone_name=backbone, encoder_weights=weights,
input_shape=input_shape,
classes=classes, activation='sigmoid')
 elif umodel == 'fpn':
model = FPN(backbone_name=backbone, encoder_weights=weights,
input_shape=input_shape,
classes=classes, activation='sigmoid')
 elif umodel == 'link':
model = Linknet(backbone_name=backbone, encoder_weights=weights,
input_shape=input_shape,
classes=classes, activation='sigmoid')
 else:
raise AttributeError('umodel mode parameter error')
 model.compile(
 optimizer=tfa.optimizers.Lookahead(
 tf.keras.optimizers.Adam(learning_rate=lr)
),
loss=loss,
metrics=[dice_coef] 
)
 return model
dig_result = model.evaluate(dig_test, Y_dig, verbose=0)
 print('loss of the model on Dig-MNIST:', dig_result[0])
 print('Accuracy of the model on Dig-MNIST:', dig_result[1])
 results = model.predict(test)
 results = np.argmax(results,axis = 1)
 submission = pd.read_csv('.. /input/Kannada-MNIST/sample_submission.csv')
 submission['label'] = results
submission.to_csv("submission.csv", index=False )<import_modules>
def make_grid(shape, window=256, min_overlap=32):
x, y = shape
nx = x //(window - min_overlap)+ 1
x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)
 x1[-1] = x - window
x2 =(x1 + window ).clip(0, x)
 ny = y //(window - min_overlap)+ 1
y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)
 y1[-1] = y - window
y2 =(y1 + window ).clip(0, y)
 slices = np.zeros(( nx, ny, 4), dtype=np.int64)
 for i in range(nx):
for j in range(ny):
slices[i, j] = x1[i], x2[i], y1[j], y2[j]
return slices.reshape(nx * ny, 4)
 def flip(img, axis=0):
if axis == 1:
return img[::-1, :, ]
elif axis == 2:
return img[:, ::-1, ]
elif axis == 3:
return img[::-1, ::-1, ]
else:
return img
import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import time
from sklearn.model_selection import train_test_split
from torch.utils.data import DataLoader, Dataset<load_from_csv>
img_files = [x for x in os.listdir(SUB_PATH)if '.tiff' in x]
print('images idxs:', img_files )
data1 = pd.read_csv('.. /input/Kannada-MNIST/train.csv')
 data2 = pd.read_csv('.. /input/Kannada-MNIST/Dig-MNIST.csv' )<concatenate>
df_sub = pd.DataFrame(subm ).T
df_sub
<drop_column><EOS> 
df_sub.to_csv('submission.csv', index=False )