Kaggle competition: Digit Recognizer
TEST = True
KAGGLE = True
MDLS_FOLDS = {'v39': [0, 2, 3, 4]}
if KAGGLE:
DATA_PATH = '.. /input/hubmap-kidney-segmentation'
MDLS_PATHS = {ver: f'.. /input/kidney-models-{ver}'
for ver, _ in MDLS_FOLDS.items() }
else:
DATA_PATH = './data2'
MDLS_PATHS = {ver: f'./models_{ver}'
for ver, _ in MDLS_FOLDS.items() }
THRESHOLD =.35
VOTERS = 1
TTAS = [0, 1]
EXPAND = 4
MIN_OVERLAP = 1 / 16
IDNT = rasterio.Affine(1, 0, 0, 0, 1, 0)
 STRATEGY = tf.distribute.get_strategy()
SUB_PATH = f'{DATA_PATH}/test' if TEST else f'{DATA_PATH}/train'
start_time = time.time()<load_pretrained>
train, test = pd.read_csv('.. /input/digit-recognizer/train.csv'), \
pd.read_csv('.. /input/digit-recognizer/test.csv')
 train.head()
params_dict = {}
for ver, _ in MDLS_FOLDS.items() :
with open(f'{MDLS_PATHS[ver]}/params.json')as file:
params_dict[ver] = json.load(file)
 for ver, params in params_dict.items() :
print('version:', ver, '| loaded params:', params, '
' )<categorify>
y_train = train['label']
x_train, x_test = train.iloc[:,1:], test
def enc2mask(encs, shape):
img = np.zeros(shape[0] * shape[1], dtype=np.uint8)
 for m, enc in enumerate(encs):
if isinstance(enc, np.float)and np.isnan(enc): continue
s = enc.split()
for i in range(len(s)// 2):
start = int(s[2 * i])- 1
length = int(s[2 * i + 1])
 img[start : start + length] = 1 + m
return img.reshape(shape ).T
def rle_encode_less_memory(img):
pixels = img.T.flatten()
pixels[0] = 0
pixels[-1] = 0
runs = np.where(pixels[1:] != pixels[:-1])[0] + 2
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<compute_test_metric>
x_train, x_test = x_train / 255., x_test / 255.
 x_train, x_test = x_train.values.reshape(-1,28,28,1),\
x_test.values.reshape(-1,28,28,1 )
def dice_coef(y_true, y_pred, smooth=1):
y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2 * intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def dice_loss(y_true, y_pred, smooth=1):
return(1 - dice_coef(y_true, y_pred, smooth))
def bce_dice_loss(y_true, y_pred):
return params['bce_weight'] * binary_crossentropy(y_true, y_pred)+ \ 
(1 - params['bce_weight'])* dice_loss(y_true, y_pred)
 def get_model(backbone, input_shape, path,
loss_type='bce_dice', umodel='unet',
classes=1, lr=.001):
if backbone == 'efficientnetb0':
weights = f'{path}/efficientnet-b0_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
elif backbone == 'efficientnetb1':
weights = f'{path}/efficientnet-b1_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
elif backbone == 'efficientnetb2':
weights = f'{path}/efficientnet-b2_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
elif backbone == 'efficientnetb3':
weights = f'{path}/efficientnet-b3_weights_tf_dim_ordering_tf_kernels_autoaugment_notop.h5'
elif backbone == 'resnet34':
weights = f'{path}/resnet34_imagenet_1000_no_top.h5'
else:
raise AttributeError('mode parameter error')
 with STRATEGY.scope() :
if loss_type == 'bce_dice':
loss = bce_dice_loss
elif loss_type == 'bce_jaccard_loss':
loss = bce_jaccard_loss
else:
raise AttributeError('loss mode parameter error')
 if umodel == 'unet':
model = Unet(backbone_name=backbone, encoder_weights=weights,
input_shape=input_shape,
classes=classes, activation='sigmoid')
 elif umodel == 'fpn':
model = FPN(backbone_name=backbone, encoder_weights=weights,
input_shape=input_shape,
classes=classes, activation='sigmoid')
 elif umodel == 'link':
model = Linknet(backbone_name=backbone, encoder_weights=weights,
input_shape=input_shape,
classes=classes, activation='sigmoid')
 else:
raise AttributeError('umodel mode parameter error')
 model.compile(
 optimizer=tfa.optimizers.Lookahead(
 tf.keras.optimizers.Adam(learning_rate=lr)
),
loss=loss,
metrics=[dice_coef] 
)
 return model<categorify>
y_train = to_categorical(y_train, num_classes=10 )
def make_grid(shape, window=256, min_overlap=32):
x, y = shape
nx = x //(window - min_overlap)+ 1
x1 = np.linspace(0, x, num=nx, endpoint=False, dtype=np.int64)
 x1[-1] = x - window
x2 =(x1 + window ).clip(0, x)
 ny = y //(window - min_overlap)+ 1
y1 = np.linspace(0, y, num=ny, endpoint=False, dtype=np.int64)
 y1[-1] = y - window
y2 =(y1 + window ).clip(0, y)
 slices = np.zeros(( nx, ny, 4), dtype=np.int64)
 for i in range(nx):
for j in range(ny):
slices[i, j] = x1[i], x2[i], y1[j], y2[j]
return slices.reshape(nx * ny, 4)
 def flip(img, axis=0):
if axis == 1:
return img[::-1, :, ]
elif axis == 2:
return img[:, ::-1, ]
elif axis == 3:
return img[::-1, ::-1, ]
else:
return img<define_variables>

 model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(filters=32, kernel_size=(5,5), \
activation='relu', input_shape=(28,28,1)) ,
tf.keras.layers.MaxPooling2D(pool_size=(2,2)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), \
activation='relu', input_shape=(28,28,1)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Conv2D(filters=128, kernel_size=(3,3), \
activation='relu', input_shape=(28,28,1)) ,
tf.keras.layers.Dropout (.4),
tf.keras.layers.MaxPooling2D(pool_size=(2,2), strides=(2,2)) ,
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(256, activation='relu'),
tf.keras.layers.Dropout (.4),
tf.keras.layers.Dense(10, activation='softmax')
 ])
 model.compile(
 optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'] 
)
 model.summary()
img_files = [x for x in os.listdir(SUB_PATH)if '.tiff' in x]
print('images idxs:', img_files )<create_dataframe>

 num_epochs = 50
history = model.fit(x_train, y_train,
epochs=num_epochs,
callbacks=[
tf.keras.callbacks.EarlyStopping(monitor='loss', patience=6),
tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=4)
 ],
validation_split=0.2,
verbose=0 )
df_sub = pd.DataFrame(subm ).T
df_sub<save_to_csv>
scores = model.evaluate(x_train, y_train, batch_size=32)
 print(f'Loss: {scores[0]} Accuracy: {scores[1]}' )
df_sub.to_csv('submission.csv', index=False )<set_options>
y_predicted = model.predict(x_test)
 y_test_labels = np.argmax(y_predicted, axis=1 )
color = sns.color_palette()
%matplotlib inline
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 50 )<load_from_csv>
x_test_ids = [(i + 1)for i in range(x_test.shape[0])]
<feature_engineering><EOS> 
results = pd.DataFrame({
'ImageId': x_test_ids,
'Label': y_test_labels
})
 results.to_csv('submission.csv', index=False )