Kaggle competition: HuBMAP - Hacking the Kidney
<SOS> metric: Dice Kaggle data source: hubmap-hacking-the-kidney<set_options>
mod_path = '.. /input/notebook43a3eaff7d-1/'
with open(mod_path+'params.yaml')as file:
P = yaml.load(file, Loader=yaml.FullLoader)
 pprint.pprint(P)
 THRESHOLD = 0.38
WINDOW = 1024
MIN_OVERLAP = 300
NEW_SIZE = P['DIM']
SUBMISSION_MODE = 'FULL'
CHECKSUM = True
warnings.filterwarnings('ignore')
<define_variables>
with open(mod_path + 'metrics.json')as json_file:
M = json.load(json_file)
 print('Model run datetime: '+M['datetime'])
 print('OOF val_dice_coe: ' + str(M['oof_dice_coe']))
MAIN_DIR = ".. /input/Kannada-MNIST/"
DATA_TRAIN = MAIN_DIR + "train.csv"
DATA_TEST = MAIN_DIR + "test.csv"
IMG_HEIGHT = 28
IMG_WIDTH = 28
CHANNELS = 1
IMG_SHAPE =(IMG_WIDTH, IMG_HEIGHT, CHANNELS)
 sns.set(font_scale=1.3 )<load_from_csv>
! pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index -q
! pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index -q
def load_data() :
df_train = pd.read_csv(DATA_TRAIN)
 df_test = pd.read_csv(DATA_TEST)
 return df_train, df_test<split>
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def make_grid(shape, window=256, min_overlap=32):

 x, y = shape
nx = x //(window - min_overlap)+ 1
x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)
 x1[-1] = x - window
x2 =(x1 + window ).clip(0, x)
 ny = y //(window - min_overlap)+ 1
y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)
 y1[-1] = y - window
y2 =(y1 + window ).clip(0, y)
 slices = np.zeros(( nx,ny, 4), dtype=np.int64)
 for i in range(nx):
for j in range(ny):
slices[i,j] = x1[i], x2[i], y1[j], y2[j]
return slices.reshape(nx*ny,4 )
data_train, data_test = load_data()<count_missing_values>
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 fold_models = []
for fold_model_path in glob.glob(mod_path+'*.h5'):
fold_models.append(tf.keras.models.load_model(fold_model_path,compile = False))
print(len(fold_models))
print(f"Data train has {data_train.isna().sum().sum(axis=0)} Nan values")
 print(f"Data test has {data_test.isna().sum().sum(axis=0)} Nan values" )<train_model>
AUTO = tf.data.experimental.AUTOTUNE
image_feature = {
'image': tf.io.FixedLenFeature([], tf.string),
'x1': tf.io.FixedLenFeature([], tf.int64),
'y1': tf.io.FixedLenFeature([], tf.int64)
 }
def _parse_image(example_proto):
example = tf.io.parse_single_example(example_proto, image_feature)
 image = tf.reshape(tf.io.decode_raw(example['image'],out_type=np.dtype('uint8')) ,(P['DIM'],P['DIM'], 3))
return image, example['x1'], example['y1']
def load_dataset(filenames, ordered=True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(_parse_image)
 return dataset
def get_dataset(FILENAME):
dataset = load_dataset(FILENAME)
 dataset = dataset.batch(64)
 dataset = dataset.prefetch(AUTO)
 return dataset
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self,
data,
list_IDs,
labels=None,
batch_size=32,
dim=IMG_SHAPE,
n_channels=1,
augment=False,
n_classes=10,
mode='fit',
shuffle=True,
random_state=42):
'Initialization'
self.data = data
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.augment = augment
self.mode = mode
self.random_state = random_state
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
if(self.mode == 'fit'):
return int(np.floor(len(self.list_IDs)/ self.batch_size))
else:
return int(np.ceil(len(self.list_IDs)/ self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
augment_batch = random.choice([True, False])
 indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
list_IDs_batch = [self.list_IDs[k] for k in indexes]
X = self.__generate_X(list_IDs_batch)
 if(self.mode == 'fit'):
y = self.__generate_y(list_IDs_batch)
 if(self.augment and augment_batch):
X = self.__augment_batch(X)
 return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('The mode parameter should be set to "fit" or "predict".')
 def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.seed(self.random_state)
 np.random.shuffle(self.indexes)
 def __generate_X(self, list_IDs_batch):
'Generates data containing batch_size samples'
X = np.empty(( len(list_IDs_batch), *self.dim))
for i, ID in enumerate(list_IDs_batch):
X[i,] = np.reshape(self.data[ID], self.dim)
 return X
def __generate_y(self, list_IDs_batch):
y = np.empty(( self.batch_size), dtype=int)
 for i, ID in enumerate(list_IDs_batch):
y[i, ] = self.labels[ID]
return keras.utils.to_categorical(y, num_classes=self.n_classes)
 def __random_transform(self, img):
composition = albu.Compose([
albu.ShiftScaleRotate(rotate_limit=10, shift_limit=0.15, scale_limit=0.1)
 ])
 composed = composition(image=img)
 aug_img = composed['image']
return aug_img
def __augment_batch(self, img_batch):
for i in range(img_batch.shape[0]):
img_batch[i, ] = self.__random_transform(img_batch[i, ])
 return img_batch<choose_model_class>
p = pathlib.Path('.. /input/hubmap-kidney-segmentation')
 subm = {}
for i, filename in tqdm(enumerate(p.glob('test/*.tiff')) ,
total = len(list(p.glob('test/*.tiff')))) :
print(f'{i+1} Predicting {filename.stem}')
 dataset = rasterio.open(filename.as_posix() , transform = identity)
 preds = np.zeros(dataset.shape, dtype=np.uint8)
 if SUBMISSION_MODE == 'PUBLIC_TFREC' and MIN_OVERLAP == 300 and WINDOW == 1024 and NEW_SIZE == 512:
print('SUBMISSION_MODE: PUBLIC_TFREC')
 fnames = glob.glob('/kaggle/input/hubmap-tfrecords-1024-512-test/test/'+filename.stem+'*.tfrec')
 if len(fnames)>0:
for FILENAME in fnames:
pred = None
for fold_model in fold_models:
tmp = fold_model.predict(get_dataset(FILENAME)) /len(fold_models)
 if pred is None:
pred = tmp
else:
pred += tmp
del tmp
gc.collect()
pred = tf.cast(( tf.image.resize(pred,(WINDOW,WINDOW)) > THRESHOLD),tf.bool ).numpy().squeeze()
idx = 0
for img, X1, Y1 in get_dataset(FILENAME):
for fi in range(X1.shape[0]):
x1 = X1[fi].numpy()
y1 = Y1[fi].numpy()
preds[x1:(x1+WINDOW),y1:(y1+WINDOW)] += pred[idx]
idx += 1
else:
pass
else:
print('SUBMISSION_MODE: FULL')
 slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)
 if dataset.count != 3:
print('Image file with subdatasets as channels')
 layers = [rasterio.open(subd)for subd in dataset.subdatasets]
for(x1,x2,y1,y2)in slices:
if dataset.count == 3:
image = dataset.read([1,2,3],
window=Window.from_slices(( x1,x2),(y1,y2)))
 image = np.moveaxis(image, 0, -1)
 else:
image = np.zeros(( WINDOW, WINDOW, 3), dtype=np.uint8)
 for fl in range(3):
image[:,:,fl] = layers[fl].read(window=Window.from_slices(( x1,x2),(y1,y2)))
 image = cv2.resize(image,(NEW_SIZE, NEW_SIZE),interpolation = cv2.INTER_AREA)
 image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
 image = np.expand_dims(image, 0)
 pred = None
for fold_model in fold_models:
if pred is None:
pred = np.squeeze(fold_model.predict(image))
else:
pred += np.squeeze(fold_model.predict(image))
pred = pred/len(fold_models)
 pred = cv2.resize(pred,(WINDOW, WINDOW))
preds[x1:x2,y1:y2] +=(pred > THRESHOLD ).astype(np.uint8)
 preds =(preds > 0.5 ).astype(np.uint8)
 subm[i] = {'id':filename.stem, 'predicted': rle_encode_less_memory(preds)}
if CHECKSUM:
print('Checksum: '+ str(np.sum(preds)))
 del preds
gc.collect() ;
<split><EOS> 
submission = pd.DataFrame.from_dict(subm, orient='index')
 submission.to_csv('submission.csv', index=False)
 submission.head()