Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<prepare_x_and_y>
import numpy as np
import pandas as pd
import os
import time
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import seaborn as sb
import itertools
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from keras.layers.advanced_activations import LeakyReLU
from keras.optimizers import RMSprop, Adam
from keras.preprocessing.image import ImageDataGenerator
AUTO = tf.data.experimental.AUTOTUNE
image_feature = {
'image': tf.io.FixedLenFeature([], tf.string),
'x1': tf.io.FixedLenFeature([], tf.int64),
'y1': tf.io.FixedLenFeature([], tf.int64)
 }
def _parse_image(example_proto):
example = tf.io.parse_single_example(example_proto, image_feature)
 image = tf.reshape(tf.io.decode_raw(example['image'],out_type=np.dtype('uint8')) ,(P['DIM'],P['DIM'], 3))
return image, example['x1'], example['y1']
def load_dataset(filenames, ordered=True):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(_parse_image)
 return dataset
def get_dataset(FILENAME):
dataset = load_dataset(FILENAME)
 dataset = dataset.batch(64)
 dataset = dataset.prefetch(AUTO)
 return dataset<define_variables>
os.listdir('.. /input')
 train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv" )
p = pathlib.Path('.. /input/hubmap-kidney-segmentation')
 subm = {}
for i, filename in tqdm(enumerate(p.glob('test/*.tiff')) ,
total = len(list(p.glob('test/*.tiff')))) :
print(f'{i+1} Predicting {filename.stem}')
 dataset = rasterio.open(filename.as_posix() , transform = identity)
 preds = np.zeros(dataset.shape, dtype=np.uint8)
 if SUBMISSION_MODE == 'PUBLIC_TFREC' and MIN_OVERLAP == 300 and WINDOW == 1024 and NEW_SIZE == 512:
print('SUBMISSION_MODE: PUBLIC_TFREC')
 fnames = glob.glob('/kaggle/input/hubmap-tfrecords-1024-512-test/test/'+filename.stem+'*.tfrec')
 if len(fnames)>0:
for FILENAME in fnames:
pred = None
for fold_model in fold_models:
tmp = fold_model.predict(get_dataset(FILENAME)) /len(fold_models)
 if pred is None:
pred = tmp
else:
pred += tmp
del tmp
gc.collect()
pred = tf.cast(( tf.image.resize(pred,(WINDOW,WINDOW)) > THRESHOLD),tf.bool ).numpy().squeeze()
idx = 0
for img, X1, Y1 in get_dataset(FILENAME):
for fi in range(X1.shape[0]):
x1 = X1[fi].numpy()
y1 = Y1[fi].numpy()
preds[x1:(x1+WINDOW),y1:(y1+WINDOW)] += pred[idx]
idx += 1
else:
pass
else:
print('SUBMISSION_MODE: FULL')
 slices = make_grid(dataset.shape, window=WINDOW, min_overlap=MIN_OVERLAP)
 if dataset.count != 3:
print('Image file with subdatasets as channels')
 layers = [rasterio.open(subd)for subd in dataset.subdatasets]
for(x1,x2,y1,y2)in slices:
if dataset.count == 3:
image = dataset.read([1,2,3],
window=Window.from_slices(( x1,x2),(y1,y2)))
 image = np.moveaxis(image, 0, -1)
 else:
image = np.zeros(( WINDOW, WINDOW, 3), dtype=np.uint8)
 for fl in range(3):
image[:,:,fl] = layers[fl].read(window=Window.from_slices(( x1,x2),(y1,y2)))
 image = cv2.resize(image,(NEW_SIZE, NEW_SIZE),interpolation = cv2.INTER_AREA)
 image = np.expand_dims(image, 0)
 pred = None
for fold_model in fold_models:
if pred is None:
pred = np.squeeze(fold_model.predict(image))
else:
pred += np.squeeze(fold_model.predict(image))
pred = pred/len(fold_models)
 pred = cv2.resize(pred,(WINDOW, WINDOW))
preds[x1:x2,y1:y2] +=(pred > THRESHOLD ).astype(np.uint8)
 preds =(preds > 0.5 ).astype(np.uint8)
 subm[i] = {'id':filename.stem, 'predicted': rle_encode_less_memory(preds)}
if CHECKSUM:
print('Checksum: '+ str(np.sum(preds)))
 del preds
gc.collect() ;<save_to_csv>
print(X_train.shape)
 print(test.shape)
 sb.countplot(Y_train)
 Y_train.value_counts()
submission = pd.DataFrame.from_dict(subm, orient='index')
 submission.to_csv('submission.csv', index=False)
 submission.head()<set_options>
print("Previous shape of Y_train:")
 print(Y_train.shape)
 Y_train = to_categorical(Y_train, num_classes = 10)
 print("New shape of Y_train:")
 print(Y_train.shape )
sys.path.append(".. /input/segmentation-models-pytorch-install")
 !pip install -q --no-deps.. /input/deepflash2-lfs
warnings.filterwarnings("ignore" )<categorify>
X_train, X_dev, Y_train, Y_dev = train_test_split(X_train, Y_train, test_size = 0.1 )
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs)
 def load_model_weights(model, file, strict=True):
state = torch.load(file, map_location='cpu')
 stats = state['stats']
model_state = state['model']
model.load_state_dict(model_state, strict=strict)
 return model, stats
def _get_gaussian(patch_size, sigma_scale=1./ 8)-> np.ndarray:
tmp = np.zeros(patch_size)
 center_coords = [i // 2 for i in patch_size]
sigmas = [i * sigma_scale for i in patch_size]
tmp[tuple(center_coords)] = 1
gaussian_importance_map = gaussian_filter(tmp, sigmas, 0, mode='constant', cval=0)
 gaussian_importance_map = gaussian_importance_map / np.max(gaussian_importance_map)* 1
gaussian_importance_map = gaussian_importance_map.astype(np.float32)
 gaussian_importance_map[gaussian_importance_map == 0] = np.min(
 gaussian_importance_map[gaussian_importance_map != 0])
 return gaussian_importance_map<train_model>
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
class HubmapDataset(Dataset):
'HubmapDataset class that does not load the full tiff files.'
def __init__(self, file, stats, scale=3, shift=.8, output_shape=(512,512), s_th = 40):
self.mean, self.std = stats
self.scale = scale
self.shift = shift
self.output_shape = output_shape
self.input_shape = tuple(int(t*scale)for t in self.output_shape)
 self.s_th = s_th
self.p_th = 1000*(self.output_shape[0]//256)**2
identity = rasterio.Affine(1, 0, 0, 0, 1, 0)
 self.data = rasterio.open(file, transform = identity, num_threads='all_cpus')
 if self.data.count != 3:
subdatasets = self.data.subdatasets
self.layers = []
if len(subdatasets)> 0:
for i, subdataset in enumerate(subdatasets, 0):
self.layers.append(rasterio.open(subdataset))
self.slices = []
self.out_slices = []
self.out_data_shape = tuple(int(x//self.scale)for x in self.data.shape)
 start_points = [o//2 for o in self.output_shape]
end_points = [(s - st)for s, st in zip(self.out_data_shape, start_points)]
n_points = [int(s//(o*self.shift)) +1 for s, o in zip(self.out_data_shape, self.output_shape)]
center_points = [np.linspace(st, e, num=n, endpoint=True, dtype=np.int64)for st, e, n in zip(start_points, end_points, n_points)]
for cx in center_points[1]:
for cy in center_points[0]:
slices = tuple(slice(int(( c*self.scale - o/2 ).clip(0, s)) , int(( c*self.scale + o/2 ).clip(max=s)))
 for(c, o, s)in zip(( cy, cx), self.input_shape, self.data.shape))
self.slices.append(slices)
 out_slices = tuple(slice(int(( c - o/2 ).clip(0, s)) , int(( c + o/2 ).clip(max=s)))
 for(c, o, s)in zip(( cy, cx), self.output_shape, self.out_data_shape))
self.out_slices.append(out_slices)
 def __len__(self):
return len(self.slices)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
slices = self.slices[idx]
if self.data.count == 3:
img = self.data.read([1, 2, 3],
window=Window.from_slices(*slices)
)
 img = np.moveaxis(img, 0, -1)
 else:
img = np.zeros(( *self.input_shape, 3), dtype=np.uint8)
 for fl in range(3):
img[:, :, fl] = self.layers[fl].read(
 window=Window.from_slices(*slices)
)
 if self.scale!=1:
img = cv2.resize(img, self.output_shape, interpolation = cv2.INTER_AREA)
 hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
 h,s,v = cv2.split(hsv)
 if(s>self.s_th ).sum() <= self.p_th or img.sum() <= self.p_th:
idx = -1
img =(img/255.0 - self.mean)/self.std
img = img.transpose(2, 0, 1 ).astype('float32')
 return torch.from_numpy(img), idx
class Model_pred:
'Class for prediction with multiple models'
def __init__(self, models, use_tta=True, batch_size=32):
self.models = models
self.bs = batch_size
self.tfms = [tta.HorizontalFlip() , tta.VerticalFlip() ] if use_tta else []
def predict(self, ds):
dl = DataLoader(ds, self.bs, num_workers=0, shuffle=False, pin_memory=True)
 pred = np.zeros(ds.out_data_shape, dtype='float32')
 merge_map = np.zeros(ds.out_data_shape, dtype='float32')
 gw_numpy = _get_gaussian(ds.output_shape)
 gw = torch.from_numpy(gw_numpy ).to(device)
 with torch.no_grad() :
for images, idxs in tqdm(iter(dl), total=len(dl)) :
if(( idxs>=0 ).sum() > 0):
images = images[idxs>=0].to(device)
 idxs = idxs[idxs>=0]
merger = tta.Merger()
for t in tta.Compose(self.tfms):
aug_images = t.augment_image(images)
 model_merger = tta.Merger()
for model in self.models:
out = model(aug_images)
 out = F.softmax(out, dim=1)
 model_merger.append(out)
 out = t.deaugment_mask(model_merger.result())
 merger.append(out)
 batch_smx = merger.result() *gw.view(1,1,*gw.shape)
 batch_smx = [x for x in batch_smx.permute(0,2,3,1 ).cpu().numpy() ]
for smx, idx in zip(batch_smx, idxs):
slcs = ds.out_slices[idx]
pred[slcs] += smx[...,1]
merge_map[slcs] += gw_numpy
pred /= merge_map
return pred<define_variables>
model = Sequential()
model.add(Conv2D(filters=64, kernel_size=(3,3), padding="same", input_shape=(28, 28, 1)))
 model.add(LeakyReLU(0.1))
model.add(Conv2D(filters=64, kernel_size=(3,3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(MaxPool2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None))
model.add(Dropout(0.25))
model.add(Conv2D(filters=128, kernel_size=(3,3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(Conv2D(filters=128, kernel_size=(3,3), padding="same"))
model.add(LeakyReLU(0.1))
model.add(MaxPool2D(pool_size=(2, 2), strides=None, padding='valid', data_format=None))
model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256))
model.add(LeakyReLU(0.1))
model.add(Dropout(0.5))
model.add(Dense(10, activation="softmax"))
class CONFIG() :
data_path = Path('.. /input/hubmap-kidney-segmentation')
 model_path = Path('.. /input/hubmap-models-final/scale_models')
 scale = 3
shift = 0.8
tile_shape =(512, 512)
 encoder_name = "efficientnet-b2"
encoder_weights = None
in_channels = 3
classes = 2
batch_size = 32
tta = True
threshold = 0.5
cfg = CONFIG()<load_from_csv>
epochs = 30
batch_size = 64
start = time.time()
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
validation_data =(X_dev, Y_dev),
epochs = epochs,
steps_per_epoch=X_train.shape[0] // batch_size 
)
 end = time.time()
print("Total time training was:")
 print(end - start)
<save_to_csv><EOS> 
predictions = model.predict(test)
 results = np.argmax(predictions, axis = 1)
 print(results)
 results = pd.Series(results, name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("output.csv",index=False )