Kaggle competition: HuBMAP - Hacking the Kidney
predict_df = pd.read_csv(data_dir+"test.csv")
 predict_df.head(2 )<predict_on_test>
imshow_from_file('.. /input/pics-j/small_scale.png' )
predicted_labels = bes_model_so_far.predict_classes(X_predict )<save_to_csv>
imshow_from_file('.. /input/pics-j/big_scale.png' )
<install_modules><EOS> 

 IMAGES_DIR = '.. /input/hubmap-kidney-segmentation/test'
WEIGHTS_SMALL = '.. /input/weights-j/t198_best_model.ckpt'
WEIGHTS_BIG = '.. /input/weights-j/t246_best_model.ckpt'
OUTPUT_FILE = 'submission.csv'
THRESHOLD_SMALL = 0.6
RESIZE_FACTOR_SMALL = 0.25
ROUNDING_ORIG_SMALL = 64
OVERLAP_SMALL = 448
BASE_SIZE_CROP_SMALL = 4096
NET_SCALE_SMALL = 16
N_INPUT_CHANNELS_SMALL = 3
N_CLASSES_SMALL = 1
N_FEATUREMAPS_SMALL = [N_INPUT_CHANNELS_SMALL, 4, 4, 8, 16, 24, 32, 32, 48, N_CLASSES_SMALL]
KERNELS_X_SIZE_SMALL = 3
KERNELS_Y_SIZE_SMALL = 3
KERNELS_INCREASED_X_SIZE_SMALL = 7
KERNELS_INCREASED_Y_SIZE_SMALL = 7
BASE_SIZE_BIG = 900
BASE_SIZE_HALF_BIG = BASE_SIZE_BIG // 2
NET_SCALE_BIG = 8
CENTER =(BASE_SIZE_BIG - 1)/ 2
N_PHI = 384
N_PHI_TILED = 3 * N_PHI
N_PHI_OUT = N_PHI // NET_SCALE_BIG
N_PHI_OUT_TILLED = 3 * N_PHI_OUT
N_R = 128
N_R_OUT = N_R // NET_SCALE_BIG
R_MAX = BASE_SIZE_BIG / 2
r_tmp_index = np.arange(N_R ).astype(np.float32 ).reshape(N_R, 1)
 r_mean_max =(N_R - 1 + 0.5)/ NET_SCALE_BIG - 0.5
NET_SCALE_BIG = 8
N_INPUT_CHANNELS_BIG = 3
N_CLASSES_BIG = 1
N_FEATUREMAPS_BIG = [N_INPUT_CHANNELS_BIG, 4, 4, 8, 16, 16, 32, 32, 32, N_CLASSES_BIG]
KERNELS_X_SIZE_BIG = 3
KERNELS_Y_SIZE_BIG = 3
KERNELS_INCREASED_X_SIZE_BIG = 7
KERNELS_INCREASED_Y_SIZE_BIG = 7
KERNELS_INCREASED_X_SIZE_BIG_HALF =(KERNELS_INCREASED_X_SIZE_BIG - 1)/ 2
def int_round(inp):
return int(np.round(inp))
RESIZE_FACTOR_SMALL_SCALE = RESIZE_FACTOR_SMALL
def bool_to_u8(inp):
return 255 *(inp.astype(np.uint8))
def int_round(inp):
return int(np.round(inp))
def gauss_kernel(half_width):
x = np.arange(-half_width, half_width + 1, dtype=np.float32)
 pseudo_sigma = 0.9 * half_width
y = np.exp(-(x / pseudo_sigma)**2)
 y = y / np.sum(y)
 return y
def extend_contour(contour, n_extended):
contour_extended = np.zeros(( contour.shape[0] + 2 * n_extended, 2), np.float32)
 contour_extended[n_extended: -n_extended, :] = contour
contour_extended[0: n_extended, :] = contour[-n_extended:, :]
contour_extended[-n_extended:, :] = contour[0: n_extended, :]
return contour_extended
def smooth_contour(contour, kernel):
kernel_half_width =(kernel.size - 1)// 2
contour_extended = extend_contour(contour, kernel_half_width)
 contour_smoothed = np.zeros(( contour.shape[0], 2), np.float32)
 contour_smoothed[:, 0] = np.convolve(contour_extended[:, 0], kernel, mode='valid')
 contour_smoothed[:, 1] = np.convolve(contour_extended[:, 1], kernel, mode='valid')
 return contour_smoothed
def natural_parametrization(contour, points_per_pixel):
x = contour[:, 0]
y = contour[:, 1]
dx = np.diff(x)
 dy = np.diff(y)
 dd = np.sqrt(dx**2 + dy**2)
 dd = np.concatenate(( [0], dd))
d = np.cumsum(dd)
 x_extended = np.concatenate(( x, [x[0]]))
y_extended = np.concatenate(( y, [y[0]]))
dd_extention = np.sqrt(( x[0] - x[-1])**2 +(y[0] - y[-1])**2)
 d_extended = np.concatenate(( d, [d[-1] + dd_extention]))
new_size = int_round(points_per_pixel * d_extended[-1])
 d_uniform = np.linspace(0, d_extended[-1], new_size + 1)[0:-1]
x_iterpolated = np.interp(d_uniform, d_extended, x_extended)
 y_iterpolated = np.interp(d_uniform, d_extended, y_extended)
 contour_interpolated = np.zeros(( x_iterpolated.size, 2), np.float32)
 contour_interpolated[:, 0] = x_iterpolated
contour_interpolated[:, 1] = y_iterpolated
return contour_interpolated
def delete_files(dir_):
files = glob.glob(dir_ + '/*')
 for file in files:
os.remove(file)
 def prepare_dir(dir_):
if os.path.exists(dir_):
delete_files(dir_)
 else:
os.makedirs(dir_)
 def prepare_crop_indexes(x1, x2, size, size_crop):
if x1 < 0:
x1_src = 0
x2_src = x2
x1_dst = size_crop - x2
x2_dst = size_crop
elif x2 > size:
x1_src = x1
x2_src = size
x1_dst = 0
x2_dst = size_crop -(x2 - size)
 else:
x1_src = x1
x2_src = x2
x1_dst = 0
x2_dst = size_crop
return x1_src, x2_src, x1_dst, x2_dst
def round_with_mult_higher(inp, mult):
return(( np.ceil(inp / mult)) * mult ).astype(np.int64)
 def crop_with_overlap_1d(size, overlap, base_size, rounding):
n_crops_non_integer =(size - overlap)/(base_size - overlap)
 n_crops = int_round(n_crops_non_integer)
 delimeters = np.linspace(0, size, n_crops + 1)
 starts_0 = np.zeros(n_crops, np.float32)
 starts_0[0] = 0.0
ends_0 = np.zeros(n_crops, np.float32)
 ends_0[-1] = size
starts_0[1: ] = delimeters[1: -1] - overlap / 2
ends_0[0: -1] = delimeters[1: -1] + overlap / 2
centers =(starts_0 + ends_0)/ 2
widths_0 = ends_0 - starts_0
widths = round_with_mult_higher(widths_0, rounding)
 starts = np.zeros(n_crops, np.int64)
 starts[0] = 0
starts[1: -1] = np.round(centers[1: -1] - widths[1: -1] / 2 ).astype(np.int64)
 base_size_rounded = round_with_mult_higher(base_size, rounding)
 starts[-1] = size - base_size_rounded
ends = np.zeros(n_crops, np.int64)
 ends[-1] = size
ends[0] = base_size_rounded
ends[1: -1] = starts[1: -1] + widths[1: -1]
separators = np.zeros(n_crops + 1, np.float32)
 separators[0] = 0.0
separators[-1] = size - 1
separators[1:-1] =(ends[0: -1] - 1 + starts[1:])/ 2
return starts, ends, separators
def weight_variable(name, shape):
return tf.Variable(tf.initializers.GlorotUniform()(shape=shape), name=name)
 def weight_variable_for_skip_connection(name, shape):
return tf.Variable(tf.constant_initializer(1.0 / shape[2] )(shape=shape), name=name)
 def bias_variable(name, shape):
return tf.Variable(tf.zeros_initializer()(shape=shape), name=name)
 def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1,1,1,1], padding='SAME')
 def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')
 def avarage_pool_2x2(x):
return tf.nn.avg_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')
 def avarage_pool_4x4(x):
return tf.nn.avg_pool(x, ksize=[1,4,4,1], strides=[1,4,4,1], padding='SAME')
 class NetModuleSmallScale(tf.Module):
def __init__(self):
self.W_conv1 = weight_variable("W_conv1", [KERNELS_Y_SIZE_SMALL, KERNELS_X_SIZE_SMALL, N_FEATUREMAPS_SMALL[0], N_FEATUREMAPS_SMALL[1]])
 self.b_conv1 = bias_variable("b_conv1", [N_FEATUREMAPS_SMALL[1]])
 block_name = "block1"
n_featuremaps = N_FEATUREMAPS_SMALL[1: 3 + 1]
self.block1_W_conv1 = weight_variable(block_name + "_" + "W_conv1", [KERNELS_Y_SIZE_SMALL, KERNELS_X_SIZE_SMALL, n_featuremaps[0], n_featuremaps[1]])
 self.block1_b_conv1 = bias_variable(block_name + "_" + "b_conv1", [n_featuremaps[1]])
 self.block1_W_conv2 = weight_variable(block_name + "_" + "W_conv2", [KERNELS_Y_SIZE_SMALL, KERNELS_X_SIZE_SMALL, n_featuremaps[1], n_featuremaps[2]])
 self.block1_b_conv2 = bias_variable(block_name + "_" + "b_conv2", [n_featuremaps[2]])
 self.block1_W_conv3 = weight_variable_for_skip_connection(block_name + "_" + "W_conv3", [1, 1, n_featuremaps[0], n_featuremaps[2]])
 block_name = "block2"
n_featuremaps = N_FEATUREMAPS_SMALL[3: 5 + 1]
self.block2_W_conv1 = weight_variable(block_name + "_" + "W_conv1", [KERNELS_Y_SIZE_SMALL, KERNELS_X_SIZE_SMALL, n_featuremaps[0], n_featuremaps[1]])
 self.block2_b_conv1 = bias_variable(block_name + "_" + "b_conv1", [n_featuremaps[1]])
 self.block2_W_conv2 = weight_variable(block_name + "_" + "W_conv2", [KERNELS_Y_SIZE_SMALL, KERNELS_X_SIZE_SMALL, n_featuremaps[1], n_featuremaps[2]])
 self.block2_b_conv2 = bias_variable(block_name + "_" + "b_conv2", [n_featuremaps[2]])
 self.block2_W_conv3 = weight_variable_for_skip_connection(block_name + "_" + "W_conv3", [1, 1, n_featuremaps[0], n_featuremaps[2]])
 block_name = "block3"
n_featuremaps = N_FEATUREMAPS_SMALL[5: 10 + 1]
self.block3_W_conv1 = weight_variable(block_name + "_" + "W_conv1", [KERNELS_INCREASED_Y_SIZE_SMALL, KERNELS_INCREASED_X_SIZE_SMALL, n_featuremaps[0], n_featuremaps[1]])
 self.block3_b_conv1 = bias_variable(block_name + "_" + "b_conv1", [n_featuremaps[1]])
 self.block3_W_conv2 = weight_variable(block_name + "_" + "W_conv2", [KERNELS_INCREASED_Y_SIZE_SMALL, KERNELS_INCREASED_X_SIZE_SMALL, n_featuremaps[1], n_featuremaps[2]])
 self.block3_b_conv2 = bias_variable(block_name + "_" + "b_conv2", [n_featuremaps[2]])
 self.block3_W_conv4 = weight_variable(block_name + "_" + "W_conv4", [KERNELS_INCREASED_Y_SIZE_SMALL, KERNELS_INCREASED_X_SIZE_SMALL, n_featuremaps[2], n_featuremaps[3]])
 self.block3_b_conv4 = bias_variable(block_name + "_" + "b_conv4", [n_featuremaps[3]])
 self.block3_W_conv5 = weight_variable(block_name + "_" + "W_conv5", [KERNELS_INCREASED_Y_SIZE_SMALL, KERNELS_INCREASED_X_SIZE_SMALL, n_featuremaps[3], n_featuremaps[4]])
 self.block3_b_conv5 = bias_variable(block_name + "_" + "b_conv5", [n_featuremaps[4]])
 self.block3_W_conv3 = weight_variable_for_skip_connection(block_name + "_" + "W_conv3", [1, 1, n_featuremaps[0], n_featuremaps[4]])
 self.var_list = [self.W_conv1, self.b_conv1,
self.block1_W_conv1, self.block1_b_conv1, self.block1_W_conv2, self.block1_b_conv2, self.block1_W_conv3,
self.block2_W_conv1, self.block2_b_conv1, self.block2_W_conv2, self.block2_b_conv2, self.block2_W_conv3,
self.block3_W_conv1, self.block3_b_conv1, self.block3_W_conv2, self.block3_b_conv2, self.block3_W_conv3, self.block3_W_conv4, self.block3_b_conv4, self.block3_W_conv5, self.block3_b_conv5]
@tf.function(input_signature=[tf.TensorSpec(shape=[1, None, None, N_INPUT_CHANNELS_SMALL], dtype=tf.float32)])
 def predict(self, x):
h_conv1 = tf.nn.relu(conv2d(x, self.W_conv1)+ self.b_conv1)
 featuremaps_input = h_conv1
featuremaps_max_pool_1 = max_pool_2x2(featuremaps_input)
 block_h_conv1 = tf.nn.relu(conv2d(featuremaps_max_pool_1, self.block1_W_conv1)+ self.block1_b_conv1)
 featuremaps_max_pool_2 = max_pool_2x2(block_h_conv1)
 block_h_conv2 = tf.nn.relu(conv2d(featuremaps_max_pool_2, self.block1_W_conv2)+ self.block1_b_conv2)
 featuremaps_avarage_pool = avarage_pool_4x4(featuremaps_input)
 block_h_conv3 = conv2d(featuremaps_avarage_pool, self.block1_W_conv3)
 featuremaps_output = block_h_conv2 + block_h_conv3
featuremaps_input = featuremaps_output
featuremaps_max_pool_1 = max_pool_2x2(featuremaps_input)
 block_h_conv1 = tf.nn.relu(conv2d(featuremaps_max_pool_1, self.block2_W_conv1)+ self.block2_b_conv1)
 featuremaps_max_pool_2 = max_pool_2x2(block_h_conv1)
 block_h_conv2 = tf.nn.relu(conv2d(featuremaps_max_pool_2, self.block2_W_conv2)+ self.block2_b_conv2)
 featuremaps_avarage_pool = avarage_pool_4x4(featuremaps_input)
 block_h_conv3 = conv2d(featuremaps_avarage_pool, self.block2_W_conv3)
 featuremaps_output = block_h_conv2 + block_h_conv3
featuremaps_input = featuremaps_output
featuremaps_max_pool_1 = featuremaps_input
block_h_conv1 = tf.nn.relu(conv2d(featuremaps_max_pool_1, self.block3_W_conv1)+ self.block3_b_conv1)
 block_h_conv2 = tf.nn.relu(conv2d(block_h_conv1, self.block3_W_conv2)+ self.block3_b_conv2)
 block_h_conv4 = tf.nn.relu(conv2d(block_h_conv2, self.block3_W_conv4)+ self.block3_b_conv4)
 block_h_conv5 = conv2d(block_h_conv4, self.block3_W_conv5)+ self.block3_b_conv5
featuremaps_avarage_pool = featuremaps_input
block_h_conv3 = conv2d(featuremaps_max_pool_1, self.block3_W_conv3)
 output = tf.math.add(block_h_conv5, block_h_conv3, name='output')
 return output
def load_weights(self, saved_wights_path):
ckpt = tf.train.Checkpoint(var_list=self.var_list)
 ckpt.read(saved_wights_path)
 class NetModuleBigScale(tf.Module):
def __init__(self):
self.W_conv1 = weight_variable("W_conv1", [KERNELS_Y_SIZE_BIG, KERNELS_X_SIZE_BIG, N_FEATUREMAPS_BIG[0], N_FEATUREMAPS_BIG[1]])
 self.b_conv1 = bias_variable("b_conv1", [N_FEATUREMAPS_BIG[1]])
 block_name = "block1"
n_featuremaps = N_FEATUREMAPS_BIG[1: 3 + 1]
self.block1_W_conv1 = weight_variable(block_name + "_" + "W_conv1", [KERNELS_Y_SIZE_BIG, KERNELS_X_SIZE_BIG, n_featuremaps[0], n_featuremaps[1]])
 self.block1_b_conv1 = bias_variable(block_name + "_" + "b_conv1", [n_featuremaps[1]])
 self.block1_W_conv2 = weight_variable(block_name + "_" + "W_conv2", [KERNELS_Y_SIZE_BIG, KERNELS_X_SIZE_BIG, n_featuremaps[1], n_featuremaps[2]])
 self.block1_b_conv2 = bias_variable(block_name + "_" + "b_conv2", [n_featuremaps[2]])
 self.block1_W_conv3 = weight_variable_for_skip_connection(block_name + "_" + "W_conv3", [1, 1, n_featuremaps[0], n_featuremaps[2]])
 block_name = "block2"
n_featuremaps = N_FEATUREMAPS_BIG[3: 5 + 1]
self.block2_W_conv1 = weight_variable(block_name + "_" + "W_conv1", [KERNELS_Y_SIZE_BIG, KERNELS_X_SIZE_BIG, n_featuremaps[0], n_featuremaps[1]])
 self.block2_b_conv1 = bias_variable(block_name + "_" + "b_conv1", [n_featuremaps[1]])
 self.block2_W_conv2 = weight_variable(block_name + "_" + "W_conv2", [KERNELS_Y_SIZE_BIG, KERNELS_X_SIZE_BIG, n_featuremaps[1], n_featuremaps[2]])
 self.block2_b_conv2 = bias_variable(block_name + "_" + "b_conv2", [n_featuremaps[2]])
 self.block2_W_conv3 = weight_variable_for_skip_connection(block_name + "_" + "W_conv3", [1, 1, n_featuremaps[0], n_featuremaps[2]])
 block_name = "block3"
n_featuremaps = N_FEATUREMAPS_BIG[5: 10 + 1]
self.block3_W_conv1 = weight_variable(block_name + "_" + "W_conv1", [KERNELS_INCREASED_Y_SIZE_BIG, KERNELS_INCREASED_X_SIZE_BIG, n_featuremaps[0], n_featuremaps[1]])
 self.block3_b_conv1 = bias_variable(block_name + "_" + "b_conv1", [n_featuremaps[1]])
 self.block3_W_conv2 = weight_variable(block_name + "_" + "W_conv2", [N_R_OUT, KERNELS_INCREASED_X_SIZE_BIG, n_featuremaps[1], n_featuremaps[2]])
 self.block3_b_conv2 = bias_variable(block_name + "_" + "b_conv2", [n_featuremaps[2]])
 self.block3_W_conv4 = weight_variable(block_name + "_" + "W_conv4", [1, KERNELS_INCREASED_X_SIZE_BIG, n_featuremaps[2], n_featuremaps[3]])
 self.block3_b_conv4 = bias_variable(block_name + "_" + "b_conv4", [n_featuremaps[3]])
 self.block3_W_conv5 = weight_variable(block_name + "_" + "W_conv5", [1, KERNELS_INCREASED_X_SIZE_BIG, n_featuremaps[3], n_featuremaps[4]])
 self.block3_b_conv5 = bias_variable(block_name + "_" + "b_conv5", [n_featuremaps[4]])
 self.block3_W_conv3 = weight_variable_for_skip_connection(block_name + "_" + "W_conv3", [N_R_OUT, 1, n_featuremaps[0], n_featuremaps[4]])
 self.upsample_tile_parameter_tf_const = tf.constant([1, 1, 2, 1, 2, 1], tf.int32)
 self.upsample_reshape_parameter_ft_const = tf.constant([1, N_R_OUT, N_PHI_OUT_TILLED, N_FEATUREMAPS_BIG[-4]], tf.int32)
 self.var_list = [self.W_conv1, self.b_conv1,
self.block1_W_conv1, self.block1_b_conv1, self.block1_W_conv2, self.block1_b_conv2, self.block1_W_conv3,
self.block2_W_conv1, self.block2_b_conv1, self.block2_W_conv2, self.block2_b_conv2, self.block2_W_conv3,
self.block3_W_conv1, self.block3_b_conv1, self.block3_W_conv2, self.block3_b_conv2, self.block3_W_conv3, self.block3_W_conv4, self.block3_b_conv4, self.block3_W_conv5, self.block3_b_conv5]
@tf.function(input_signature=[tf.TensorSpec(shape=[1, N_R, N_PHI_TILED, N_INPUT_CHANNELS_BIG], dtype=tf.float32)])
 def predict(self, x):
h_conv1 = tf.nn.relu(conv2d(x, self.W_conv1)+ self.b_conv1)
 featuremaps_input = h_conv1
featuremaps_max_pool_1 = max_pool_2x2(featuremaps_input)
 block_h_conv1 = tf.nn.relu(conv2d(featuremaps_max_pool_1, self.block1_W_conv1)+ self.block1_b_conv1)
 featuremaps_max_pool_2 = max_pool_2x2(block_h_conv1)
 block_h_conv2 = tf.nn.relu(conv2d(featuremaps_max_pool_2, self.block1_W_conv2)+ self.block1_b_conv2)
 featuremaps_avarage_pool = avarage_pool_4x4(featuremaps_input)
 block_h_conv3 = conv2d(featuremaps_avarage_pool, self.block1_W_conv3)
 featuremaps_output = block_h_conv2 + block_h_conv3
featuremaps_input = featuremaps_output
featuremaps_max_pool_1 = max_pool_2x2(featuremaps_input)
 block_h_conv1 = tf.nn.relu(conv2d(featuremaps_max_pool_1, self.block2_W_conv1)+ self.block2_b_conv1)
 featuremaps_avarage_pool = avarage_pool_2x2(featuremaps_input)
 block_h_conv3 = conv2d(featuremaps_avarage_pool, self.block2_W_conv3)
 featuremaps_output = block_h_conv1 + block_h_conv3
input_for_long_skip_connection = featuremaps_output
featuremaps_max_pool_2 = max_pool_2x2(block_h_conv1)
 block_h_conv2 = tf.nn.relu(conv2d(featuremaps_max_pool_2, self.block2_W_conv2)+ self.block2_b_conv2)
 featuremaps_output = block_h_conv2
featuremaps_input = featuremaps_output
featuremaps_max_pool_1 = featuremaps_input
block_h_conv1 = tf.nn.relu(conv2d(featuremaps_max_pool_1, self.block3_W_conv1)+ self.block3_b_conv1)
 upsample_tmp_1 = tf.expand_dims(block_h_conv1, 2)
 upsample_tmp_2 = tf.expand_dims(upsample_tmp_1, 4)
 upsample_tmp_3 = tf.tile(upsample_tmp_2, self.upsample_tile_parameter_tf_const)
 block_h_upsample2 = tf.reshape(upsample_tmp_3, self.upsample_reshape_parameter_ft_const)
 strides = [1,1,1,1]
padding = [[0, 0], [0, 0], [KERNELS_INCREASED_X_SIZE_BIG_HALF, KERNELS_INCREASED_X_SIZE_BIG_HALF], [0, 0]]
block_h_conv2 = tf.nn.relu(tf.nn.conv2d(block_h_upsample2, self.block3_W_conv2, strides=strides, padding=padding)+ self.block3_b_conv2)
 block_h_conv4 = tf.nn.relu(tf.nn.conv2d(block_h_conv2, self.block3_W_conv4, strides=strides, padding=padding)+ self.block3_b_conv4)
 block_h_conv5 = tf.nn.conv2d(block_h_conv4, self.block3_W_conv5, strides=strides, padding=padding)+ self.block3_b_conv5
padding = [[0, 0], [0, 0], [0, 0], [0, 0]]
block_h_conv3 = tf.nn.conv2d(input_for_long_skip_connection, self.block3_W_conv3, strides=strides, padding=padding)
 output = tf.math.add(block_h_conv5, block_h_conv3, name='output')
 return output
def load_weights(self, saved_wights_path):
ckpt = tf.train.Checkpoint(var_list=self.var_list)
 ckpt.read(saved_wights_path)
 net_module_small_scale = NetModuleSmallScale()
net_module_small_scale.load_weights(WEIGHTS_SMALL)
 net_module_big_scale = NetModuleBigScale()
net_module_big_scale.load_weights(WEIGHTS_BIG)
 phi = np.linspace(0, 2*np.pi, N_PHI + 1)[0: -1]
phi = phi.astype(np.float32)
 phi_out = phi.reshape(N_PHI // NET_SCALE_BIG, NET_SCALE_BIG)
 phi_out = np.mean(phi_out, axis=1)
 r = np.linspace(0, R_MAX, N_R + 1)[0: -1]
r = r.astype(np.float32)
 r_indexes = np.arange(r.size ).astype(np.float32)
 PHI, R = np.meshgrid(phi, r)
 map_x = CENTER + R * np.cos(PHI)
 map_y = CENTER + R * np.sin(PHI)
 files = glob.glob(IMAGES_DIR + '/*.tiff')
 output_file_fid = open(OUTPUT_FILE, 'w')
 output_file_fid.write('id,predicted
')
 for file_index in range(len(files)) :
file = files[file_index]
file_base = os.path.basename(file)
 file_base_no_ext = os.path.splitext(file_base)[0]
image_id = file_base_no_ext
print(file_index, len(files)- 1)
 fid = rasterio.open(IMAGES_DIR + '/' + image_id + '.tiff', 'r', num_threads='all_cpus')
 width = fid.shape[1]
height = fid.shape[0]
if fid.count == 1:
layers = []
for subdataset_index in range(len(fid.subdatasets)) :
subdataset = fid.subdatasets[subdataset_index]
layer = rasterio.open(subdataset)
 layers.append(layer)
 starts_y, ends_y, separators_y = crop_with_overlap_1d(height, OVERLAP_SMALL, BASE_SIZE_CROP_SMALL, ROUNDING_ORIG_SMALL)
 starts_x, ends_x, separators_x = crop_with_overlap_1d(width, OVERLAP_SMALL, BASE_SIZE_CROP_SMALL, ROUNDING_ORIG_SMALL)
 x_spot_all = []
y_spot_all = []
for y_index in range(starts_y.size):
y1_small = starts_y[y_index]
y2_small = ends_y[y_index]
separator_y1 = separators_y[y_index]
separator_y2 = separators_y[y_index + 1]
for x_index in range(starts_x.size):
x1_small = starts_x[x_index]
x2_small = ends_x[x_index]
separator_x1 = separators_x[x_index]
separator_x2 = separators_x[x_index + 1]
window = Window.from_slices(( y1_small, y2_small),(x1_small, x2_small))
height_tmp = y2_small - y1_small
width_tmp = x2_small - x1_small
if fid.count == 1:
image_crop_small = np.zeros(( height_tmp, width_tmp, 3), np.uint8)
 for layer_index in range(len(layers)) :
layer = layers[layer_index]
image_crop_small[:, :, layer_index] = layer.read(1, window=window)
 else:
image_crop_small = np.moveaxis(fid.read([1, 2, 3], window=window), 0, -1)
 image_crop_small_resized = cv2.resize(image_crop_small, None, fx=RESIZE_FACTOR_SMALL, fy=RESIZE_FACTOR_SMALL, interpolation=cv2.INTER_CUBIC)
 image_crop_small_resized_4d = np.expand_dims(image_crop_small_resized, 0 ).astype(np.float32)/ 255.0
mask_crop_small = net_module_small_scale.predict(image_crop_small_resized_4d)
 mask_crop_small = mask_crop_small.numpy() [0, :, :, 0]
heatmap_thresholded = 255 *(( mask_crop_small >= THRESHOLD_SMALL ).astype(np.uint8))
n_labels, labels, stats, centroids_tmp_0 = cv2.connectedComponentsWithStats(heatmap_thresholded, connectivity=4)
 if n_labels > 1:
for max_ind in range(1, n_labels):
y_bigest_spot, x_bigest_spot = np.where(labels == max_ind)
 weights = mask_crop_small[y_bigest_spot, x_bigest_spot]
weights_sum = np.sum(weights)
 x_spot_out = np.sum(x_bigest_spot * weights)/ weights_sum
y_spot_out = np.sum(y_bigest_spot * weights)/ weights_sum
x_spot =(x_spot_out + 0.5)* NET_SCALE_SMALL / RESIZE_FACTOR_SMALL - 0.5
y_spot =(y_spot_out + 0.5)* NET_SCALE_SMALL / RESIZE_FACTOR_SMALL - 0.5
x_spot += x1_small
y_spot += y1_small
condition = True
condition = condition and separator_x1 <= x_spot
if x_index ==(starts_x.size - 1):
condition = condition and x_spot <= separator_x2
else:
condition = condition and x_spot < separator_x2
condition = condition and separator_y1 <= y_spot
if y_index ==(starts_y.size - 1):
condition = condition and y_spot <= separator_y2
else:
condition = condition and y_spot < separator_y2
if condition:
x_spot_all.append(x_spot)
 y_spot_all.append(y_spot)
 del image_crop_small_resized_4d
del image_crop_small
del image_crop_small_resized
del mask_crop_small
gc.collect()
if fid.count == 1:
for layer in layers:
layer.close()
del layers
fid.close()
gc.collect()
fid = rasterio.open(file, 'r', num_threads='all_cpus')
 if fid.count == 1:
layers = []
for subdataset_index in range(len(fid.subdatasets)) :
subdataset = fid.subdatasets[subdataset_index]
layer = rasterio.open(subdataset)
 layers.append(layer)
 mask_prediction = np.zeros(( height, width), np.bool)
 for spot_index in range(len(x_spot_all)) :
x_spot = int_round(x_spot_all[spot_index])
 y_spot = int_round(y_spot_all[spot_index])
 x1_big = x_spot - BASE_SIZE_HALF_BIG
x2_big = x1_big + BASE_SIZE_BIG
y1_big = y_spot - BASE_SIZE_HALF_BIG
y2_big = y1_big + BASE_SIZE_BIG
x1_big_src, x2_big_src, x1_big_dst, x2_big_dst = prepare_crop_indexes(x1_big, x2_big, width, BASE_SIZE_BIG)
 y1_big_src, y2_big_src, y1_big_dst, y2_big_dst = prepare_crop_indexes(y1_big, y2_big, height, BASE_SIZE_BIG)
 image_crop_big = np.zeros(( BASE_SIZE_BIG, BASE_SIZE_BIG, 3), np.uint8)
 window = Window.from_slices(( y1_big_src, y2_big_src),(x1_big_src, x2_big_src))
if fid.count == 1:
for layer_index in range(len(layers)) :
layer = layers[layer_index]
image_crop_big[y1_big_dst: y2_big_dst, x1_big_dst: x2_big_dst, layer_index] = layer.read(1, window=window)
 else:
image_crop_big[y1_big_dst: y2_big_dst, x1_big_dst: x2_big_dst, :] = np.moveaxis(fid.read([1, 2, 3], window=window), 0, -1)
 unrolled = cv2.remap(image_crop_big, map_x, map_y, cv2.INTER_CUBIC)
 unrolled = np.tile(unrolled,(1, 3, 1))
del image_crop_big
unrolled_4d = np.expand_dims(unrolled, 0 ).astype(np.float32)/ 255.0
del unrolled
big_predicted_tensor = net_module_big_scale.predict(unrolled_4d)
 del unrolled_4d
r_mean = big_predicted_tensor.numpy() [0, 0, N_PHI_OUT: 2 * N_PHI_OUT, 0]
r_mean = r_mean * r_mean_max
r_mean =(r_mean + 0.5)* NET_SCALE_BIG - 0.5
r_mean = np.clip(r_mean, 0.0, N_R - 1)
 r_mean = np.interp(r_mean, r_indexes, r)
 x_contour = CENTER + r_mean * np.cos(phi_out)
 y_contour = CENTER + r_mean * np.sin(phi_out)
 contour_cv2 = np.zeros(( x_contour.size, 1, 2), np.int32)
 contour_cv2[:, 0, 0] = np.round(x_contour ).astype(np.int32)
 contour_cv2[:, 0, 1] = np.round(y_contour ).astype(np.int32)
 mask_center_orig_u8 = np.zeros(( BASE_SIZE_BIG, BASE_SIZE_BIG), np.uint8)
 mask_center_orig_u8 = cv2.drawContours(mask_center_orig_u8, [contour_cv2], -1,(255, 255, 255), -1)
 mask_center_orig = mask_center_orig_u8 >= 128
mask_prediction[y1_big_src: y2_big_src, x1_big_src: x2_big_src] = mask_prediction[y1_big_src: y2_big_src, x1_big_src: x2_big_src] | mask_center_orig[y1_big_dst: y2_big_dst, x1_big_dst: x2_big_dst]
del mask_center_orig_u8
del mask_center_orig
del r_mean
gc.collect()
if fid.count == 1:
for layer in layers:
layer.close()
del layers
fid.close()
del fid
gc.collect()
y, x = np.nonzero(mask_prediction)
 del mask_prediction
gc.collect()
indexes = np.sort(x * height + y + 1)
 del x
gc.collect()
del y
gc.collect()
diff = np.diff(indexes)
 gaps = np.nonzero(diff > 1)[0]
del diff
gc.collect()
index_min = np.min(indexes)
 index_max = np.max(indexes)
 starts = []
lengths = []
start_tmp = index_min
for intermidieate_index in range(gaps.size):
starts.append(start_tmp)
 lengths.append(indexes[gaps[intermidieate_index]] - start_tmp + 1)
 start_tmp = indexes[gaps[intermidieate_index] + 1]
del indexes
gc.collect()
del gaps
gc.collect()
starts.append(start_tmp)
 lengths.append(index_max - start_tmp + 1)
 line = image_id + ','
for segment_index in range(len(starts)) :
start = starts[segment_index]
length = lengths[segment_index]
line = line + str(start)+ ' ' + str(length)
 if segment_index !=(len(starts)- 1):
line = line + ' '
del starts
gc.collect()
del lengths
gc.collect()
output_file_fid.write(line + '
')
 del line
gc.collect()
output_file_fid.close()