Kaggle competition: Digit Recognizer
p = pathlib.Path('.. /input/hubmap-kidney-segmentation')
 subm = {}
for i, filename in tqdm(enumerate(p.glob('test/*.tiff')) ,
total = len(list(p.glob('test/*.tiff')))) :
print(f'{i+1} Predicting {filename.stem}')
 dataset = rasterio.open(filename.as_posix() , transform = identity)
 preds = np.zeros(dataset.shape, dtype=np.uint8)
 if SUBMISSION_MODE == 'PUBLIC_TFREC' and MIN_OVERLAP == 300 and WINDOW == 1024 and NEW_SIZE == 512:
print('SUBMISSION_MODE: PUBLIC_TFREC')
 fnames = glob.glob('/kaggle/input/hubmap-tfrecords-1024-512-test/test/'+filename.stem+'*.tfrec')
 if len(fnames)>0:
for FILENAME in fnames:
pred = None
for fold_model in fold_models:
tmp = fold_model.predict(get_dataset(FILENAME)) /len(fold_models)
 if pred is None:
pred = tmp
else:
pred += tmp
del tmp
gc.collect()
pred = tf.cast(( tf.image.resize(pred,(WINDOW,WINDOW)) > THRESHOLD),tf.bool ).numpy().squeeze()
idx = 0
for img, X1, Y1 in get_dataset(FILENAME):
for fi in range(X1.shape[0]):
x1 = X1[fi].numpy()
y1 = Y1[fi].numpy()
preds[x1:(x1+WINDOW),y1:(y1+WINDOW)] += pred[idx]
idx += 1
else:
pass
else:
print('SUBMISSION_MODE: FULL')
 slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)
 if dataset.count != 3:
print('Image file with subdatasets as channels')
 layers = [rasterio.open(subd)for subd in dataset.subdatasets]
for(x1,x2,y1,y2)in slices:
if dataset.count == 3:
image = dataset.read([1,2,3],
window=Window.from_slices(( x1,x2),(y1,y2)))
 image = np.moveaxis(image, 0, -1)
 else:
image = np.zeros(( WINDOW, WINDOW, 3), dtype=np.uint8)
 for fl in range(3):
image[:,:,fl] = layers[fl].read(window=Window.from_slices(( x1,x2),(y1,y2)))
 image = cv2.resize(image,(NEW_SIZE, NEW_SIZE),interpolation = cv2.INTER_AREA)
 image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
 image = np.expand_dims(image, 0)
 pred = None
for fold_model in fold_models:
if pred is None:
pred = np.squeeze(fold_model.predict(image))
else:
pred += np.squeeze(fold_model.predict(image))
pred = pred/len(fold_models)
 pred = cv2.resize(pred,(WINDOW, WINDOW))
preds[x1:x2,y1:y2] +=(pred > THRESHOLD ).astype(np.uint8)
 preds =(preds > 0.5 ).astype(np.uint8)
 subm[i] = {'id':filename.stem, 'predicted': rle_encode_less_memory(preds)}
if CHECKSUM:
print('Checksum: '+ str(np.sum(preds)))
 del preds
gc.collect() ;<save_to_csv>
from tensorflow.python import keras
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense, Flatten, Conv2D, Dropout, BatchNormalization, Activation, MaxPooling2D
from sklearn.model_selection import train_test_split
submission = pd.DataFrame.from_dict(subm, orient='index')
 submission.to_csv('submission.csv', index=False)
 submission.head()<install_modules>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
! pip install.. /input/kerasapplications/keras-team-keras-applications-3b180cb -f./ --no-index -q
! pip install.. /input/efficientnet/efficientnet-1.1.0/ -f./ --no-index -q
sys.path.append(".. /input/segmentation-models-pytorch-install")
 !pip install -q --no-deps.. /input/deepflash2-lfs
warnings.filterwarnings("ignore" )<categorify>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def load_model_weights(model, file, strict=True):
state = torch.load(file, map_location='cpu')
 stats = state['stats']
model_state = state['model']
model.load_state_dict(model_state, strict=strict)
 return model, stats
def _get_gaussian(patch_size, sigma_scale=1./ 8)-> np.ndarray:
tmp = np.zeros(patch_size)
 center_coords = [i // 2 for i in patch_size]
sigmas = [i * sigma_scale for i in patch_size]
tmp[tuple(center_coords)] = 1
gaussian_importance_map = gaussian_filter(tmp, sigmas, 0, mode='constant', cval=0)
 gaussian_importance_map = gaussian_importance_map / np.max(gaussian_importance_map)* 1
gaussian_importance_map = gaussian_importance_map.astype(np.float32)
 gaussian_importance_map[gaussian_importance_map == 0] = np.min(
 gaussian_importance_map[gaussian_importance_map != 0])
 return gaussian_importance_map<train_model>
img_rows, img_cols = 28, 28
num_classes = 10
train_x = train.values[:, 1:].astype('float32')
 train_y = keras.utils.to_categorical(train.label, num_classes)
 print(train_x.shape)
 print(train_y.shape)
 test_x = test.values.astype('float32')
 train_x /= 255
test_x /= 255
train_x = train_x.reshape(train_x.shape[0], img_rows, img_cols, 1)
 test_x = test_x.reshape(test_x.shape[0], img_rows, img_cols, 1)
 print(train_x.shape)
 print(train_y.shape )
class HubmapDataset(Dataset):
'HubmapDataset class that does not load the full tiff files.'
def __init__(self, file, stats, scale=3, shift=.8, output_shape=(512,512), s_th = 40):
self.mean, self.std = stats
self.scale = scale
self.shift = shift
self.output_shape = output_shape
self.input_shape = tuple(int(t*scale)for t in self.output_shape)
 self.s_th = s_th
self.p_th = 1000*(self.output_shape[0]//256)**2
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 self.data = rasterio.open(file, transform = identity, num_threads='all_cpus')
 if self.data.count != 3:
subdatasets = self.data.subdatasets
self.layers = []
if len(subdatasets)> 0:
for i, subdataset in enumerate(subdatasets, 0):
self.layers.append(rasterio.open(subdataset))
self.slices = []
self.out_slices = []
self.out_data_shape = tuple(int(x//self.scale)for x in self.data.shape)
 start_points = [o//2 for o in self.output_shape]
end_points = [(s - st)for s, st in zip(self.out_data_shape, start_points)]
n_points = [int(s//(o*self.shift)) +1 for s, o in zip(self.out_data_shape, self.output_shape)]
center_points = [np.linspace(st, e, num=n, endpoint=True, dtype=np.int64)for st, e, n in zip(start_points, end_points, n_points)]
for cx in center_points[1]:
for cy in center_points[0]:
slices = tuple(slice(int(( c*self.scale - o/2 ).clip(0, s)) , int(( c*self.scale + o/2 ).clip(max=s)))
 for(c, o, s)in zip(( cy, cx), self.input_shape, self.data.shape))
self.slices.append(slices)
 out_slices = tuple(slice(int(( c - o/2 ).clip(0, s)) , int(( c + o/2 ).clip(max=s)))
 for(c, o, s)in zip(( cy, cx), self.output_shape, self.out_data_shape))
self.out_slices.append(out_slices)
 def __len__(self):
return len(self.slices)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
slices = self.slices[idx]
if self.data.count == 3:
img = self.data.read([1, 2, 3],
window=Window.from_slices(*slices)
)
 img = np.moveaxis(img, 0, -1)
 else:
img = np.zeros(( *self.input_shape, 3), dtype=np.uint8)
 for fl in range(3):
img[:, :, fl] = self.layers[fl].read(
 window=Window.from_slices(*slices)
)
 if self.scale!=1:
img = cv2.resize(img, self.output_shape, interpolation = cv2.INTER_AREA)
 hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
 h,s,v = cv2.split(hsv)
 if(s>self.s_th ).sum() <= self.p_th or img.sum() <= self.p_th:
idx = -1
img =(img/255.0 - self.mean)/self.std
img = img.transpose(2, 0, 1 ).astype('float32')
 return torch.from_numpy(img), idx
class Model_pred:
'Class for prediction with multiple models'
def __init__(self, models, use_tta=True, batch_size=32):
self.models = models
self.bs = batch_size
self.tfms = [tta.HorizontalFlip() ] if use_tta else []
def predict(self, ds):
dl = DataLoader(ds, self.bs, num_workers=0, shuffle=False, pin_memory=True)
 pred = np.zeros(ds.out_data_shape, dtype='float32')
 merge_map = np.zeros(ds.out_data_shape, dtype='float32')
 gw_numpy = _get_gaussian(ds.output_shape)
 gw = torch.from_numpy(gw_numpy ).to(device)
 with torch.no_grad() :
for images, idxs in tqdm(iter(dl), total=len(dl)) :
if(( idxs>=0 ).sum() > 0):
images = images[idxs>=0].to(device)
 idxs = idxs[idxs>=0]
merger = tta.Merger()
for t in tta.Compose(self.tfms):
aug_images = t.augment_image(images)
 model_merger = tta.Merger()
for model in tqdm(self.models):
out = model(aug_images)
 out = F.softmax(out, dim=1)
 model_merger.append(out)
 torch.cuda.empty_cache()
out = t.deaugment_mask(model_merger.result())
 merger.append(out)
 batch_smx = merger.result() *gw.view(1,1,*gw.shape)
 batch_smx = [x for x in batch_smx.permute(0,2,3,1 ).cpu().numpy() ]
for smx, idx in zip(batch_smx, idxs):
slcs = ds.out_slices[idx]
pred[slcs] += smx[...,1]
merge_map[slcs] += gw_numpy
pred /= merge_map
return pred<define_variables>
models = []
for idx in range(15):
model = Sequential()
model.add(Conv2D(32, kernel_size=3,
activation='relu',
input_shape=(img_rows, img_cols, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size=3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(32, kernel_size=5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 3, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model.add(BatchNormalization())
 model.add(Dropout(0.4))
model.add(Conv2D(128, kernel_size = 4, activation='relu'))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dropout(0.4))
model.add(Dense(10, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'])
 models.append(model )
class CONFIG() :
data_path = Path('.. /input/hubmap-kidney-segmentation')
 model_path = Path('.. /input/hubmap-single-fold-models-b1b5')
 scale = 3
shift = 0.8
tile_shape =(512, 512)
 encoder_name = "efficientnet-b4"
encoder_weights = None
in_channels = 3
classes = 2
batch_size = 4
tta = True
threshold = 0.30
cfg = CONFIG()<define_variables>
def shift(arr, num, axis, fill_value=0.) :
result = np.empty_like(arr)
 if axis == 0:
if num > 0:
result[:num, :] = fill_value
result[num:, :] = arr[:-num, :]
elif num < 0:
result[num:, :] = fill_value
result[:num, :] = arr[-num:, :]
else:
result = arr
elif axis == 1:
if num > 0:
result[:, :num] = fill_value
result[:, num:] = arr[:, :-num]
elif num < 0:
result[:, num:] = fill_value
result[:, :num] = arr[:, -num:]
else:
result = arr
return result
def gen_img(arr):
shif = 4 *(1 if random.random() < 0.5 else -1)
 axis = random.randint(0, 1)
 return shift(arr, shif, axis)
 def gen_data(x, y):
new_x = np.empty(x.shape)
 new_y = np.empty(y.shape)
 for idx in range(x.shape[0]):
new_x[idx] = gen_img(x[idx])
 new_y[idx] = y[idx]
return new_x, new_y
THRESHOLD = 0.3
VOTERS = 0.5
WINDOW = 1024
MIN_OVERLAP = 300
NEW_SIZE = 256
SUM_PRED = 128
CHECK=False<categorify>
es = keras.callbacks.EarlyStopping(monitor='val_acc', mode='max', verbose=1, patience=50, restore_best_weights=True)
 lrs = keras.callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 for i in range(0, 0):
extra_train_x, extra_train_y = gen_data(train_x, train_y)
 x = np.concatenate(( extra_train_x, train_x), axis=0)
 y = np.concatenate(( extra_train_y, train_y), axis=0)
 models[i].fit(x, y,
batch_size=128,
epochs=50,
validation_split=0.1,
callbacks=[es, lrs] )
def make_grid_tf(shape, window=1024, min_overlap=32):

 x, y = shape
nx = x //(window - min_overlap)+ 1
x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)
 x1[-1] = x - window
x2 =(x1 + window ).clip(0, x)
 ny = y //(window - min_overlap)+ 1
y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)
 y1[-1] = y - window
y2 =(y1 + window ).clip(0, y)
 slices = np.zeros(( nx,ny, 4), dtype=np.int64)
 for i in range(nx):
for j in range(ny):
slices[i,j] = x1[i], x2[i], y1[j], y2[j]
return slices.reshape(nx*ny,4 )<load_pretrained>
es = keras.callbacks.EarlyStopping(monitor='val_acc', mode='max', verbose=1, patience=50, restore_best_weights=True)
 lrs = keras.callbacks.LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 for i in range(0, 15):
datagen = keras.preprocessing.image.ImageDataGenerator(
 rotation_range=10,
zoom_range=0.10,
width_shift_range=0.10,
height_shift_range=0.10)
 gen_train_x, gen_val_x, gen_train_y, gen_val_y = train_test_split(train_x, train_y, test_size=0.1)
 models[i].fit_generator(datagen.flow(gen_train_x, gen_train_y, batch_size=64),
epochs=50,
steps_per_epoch=gen_train_x.shape[0]//64,
validation_data=(gen_val_x,gen_val_y),
callbacks=[lrs])
models_path_list = [
'.. /input/hubmap-efficientnet-b4/model-fold-4.h5',
'.. /input/hubmap-efficientnet-b4/model-fold-0.h5',
'.. /input/hubmap-efficientnet-b4/model-fold-3.h5',
'.. /input/hubmap-efficientnet-b6-pseudo/model-fold-2.h5',
'.. /input/hubmap-efficientnet-b6-pseudo/model-fold-1.h5',
]
fold_models = []
for fold_model_path in models_path_list:
fold_models.append(tf.keras.models.load_model(fold_model_path,compile = False))
print(len(fold_models)) 
<categorify>
results = np.zeros(( test_x.shape[0], 10))
for i in range(15):
results = results + models[i].predict(test_x )
deepflash_models=[]

 model = smp.Unet(encoder_name='efficientnet-b6',
encoder_weights=cfg.encoder_weights,
in_channels=cfg.in_channels,
classes=cfg.classes)
 model, stats = load_model_weights(model, '.. /input/hubmap-deepflash-efficientnetb6/'+f'unet_efficientnet-b6.pth')
 if torch.cuda.is_available() : model.cuda()
deepflash_models.append(model)
 del model
gc.collect() 
<feature_engineering>
predictions = np.argmax(results, axis=1 )
p = pathlib.Path('.. /input/hubmap-kidney-segmentation')
 ids=[]
for index, filename in tqdm(enumerate(p.glob('test/*.tiff')) ,
total = len(list(p.glob('test/*.tiff')))) :
ids.append(filename.stem)
 df_sample=pd.DataFrame()
df_sample['id']=ids
df_sample['predicted']=np.nan
df_sample = df_sample.set_index('id')
 if df_sample.shape[0]==5:
df_sample = df_sample.iloc[1:2, :]
else:
df_sample=df_sample
print(len(deepflash_models))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<save_to_csv>
submission = pd.DataFrame(data={'ImageId': pd.RangeIndex(start=1, stop=test_x.shape[0]+1), 'Label': predictions})
 submission = submission.set_index('ImageId' )
df = pd.DataFrame({'id':names,'predicted':predicts})
 df.to_csv('submission.csv',index=False)
 df.head()<set_options>
submission.to_csv('submission.csv' )