Spaces:

ho11laqe
/

nnUNet_calvingfront_detection

Sleeping

App Files Files Community

nnUNet_calvingfront_detection / evaluate_nnUNet.py

ho11laqe

init

ecf08bc about 1 year ago

raw history blame contribute delete

No virus

38.4 kB

	from argparse import ArgumentParser
	import os
	import numpy as np
	from sklearn.metrics import precision_score, recall_score, f1_score, jaccard_score
	from data_processing.data_postprocessing import postprocess_zone_segmenation, postprocess_front_segmenation, extract_front_from_zones
	import torch.nn as nn
	#from segmentation_models_pytorch.losses.dice import DiceLoss
	#from PIL import Image
	#from models.front_segmentation_model import DistanceMapBCE
	import re
	from pathlib import Path
	import cv2
	import scipy.stats as st
	from scipy.spatial import distance
	import skimage
	import matplotlib.pyplot as plt
	from skimage.morphology import skeletonize
	import json
	import plotly.graph_objects as go
	import plotly.express as px
	import plotly.io as pio
	import os
	pio.kaleido.scope.mathjax = None


	def front_error(prediction, label):
	"""
	prediction: mask of the front prediction (WxH)
	label: mask of the front label (WxH)

	returns the mean distance of the two fronts
	"""
	front_is_present_flag = True
	polyline_pred = np.nonzero(prediction)
	polyline_label = np.nonzero(label)

	# Generate Nx2 matrix of pixels that represent the front
	pred_coords = np.array(list(zip(polyline_pred[0], polyline_pred[1])))
	mask_coords = np.array(list(zip(polyline_label[0], polyline_label[1])))

	# Return NaN if front is not detected in either pred or mask
	if pred_coords.shape[0] == 0 or mask_coords.shape[0] == 0:
	front_is_present_flag = False
	return front_is_present_flag, np.nan, np.nan, np.nan

	# Generate the pairwise distances between each point and the closest point in the other array

	distances1 = distance.cdist(pred_coords, mask_coords).min(axis=1)

	distances2 = distance.cdist(mask_coords, pred_coords).min(axis=1)
	distances = np.concatenate((distances1, distances2))

	# Calculate the average distance between each point and the closest point in the other array
	mean_distance = np.mean(distances)
	median_distance = np.median(distances)
	return front_is_present_flag, mean_distance, median_distance, distances


	def multi_class_metric(metric_function, complete_predicted_mask, complete_target):
	metrics = []
	metric_na, metric_stone, metric_glacier, metric_ocean = metric_function(np.ndarray.flatten(complete_target),
	np.ndarray.flatten(complete_predicted_mask),
	average=None)
	metric_macro_average = (metric_na + metric_stone + metric_glacier + metric_ocean) / 4
	metrics.append(metric_macro_average)
	metrics.append(metric_na)
	metrics.append(metric_stone)
	metrics.append(metric_glacier)
	metrics.append(metric_ocean)
	return metrics


	def get_matching_out_of_folder(file_name, folder):
	files = os.listdir(folder)
	matching_files = [a for a in files if
	re.match(pattern=os.path.split(file_name)[1][:-4], string=os.path.split(a)[1])]
	if len(matching_files) > 1:
	print("Something went wrong!")
	print(f"targets_matching: {matching_files}")
	if len(matching_files) < 1:
	print("Something went wrong! No matches found")
	return matching_files[0]


	def turn_colors_to_class_labels_zones(mask):
	mask_class_labels = np.copy(mask)
	mask_class_labels[mask == 0] = 0
	mask_class_labels[mask == 64] = 1
	mask_class_labels[mask == 127] = 2
	mask_class_labels[mask == 254] = 3
	return mask_class_labels


	def turn_colors_to_class_labels_front(mask):
	mask_class_labels = np.copy(mask)
	mask_class_labels[mask == 0] = 0
	mask_class_labels[mask == 255] = 1
	return mask_class_labels


	def print_zone_metrics(metric_name, list_of_metrics):
	metrics = [metric for [metric, _, _, _, _] in list_of_metrics if not np.isnan(metric)]
	metrics_na = [metric_na for [_, metric_na, _, _, _] in list_of_metrics if not np.isnan(metric_na)]
	metrics_stone = [metric_stone for [_, _, metric_stone, _, _] in list_of_metrics if not np.isnan(metric_stone)]
	metrics_glacier = [metric_glacier for [_, _, _, metric_glacier, _] in list_of_metrics if not np.isnan(metric_glacier)]
	metrics_ocean = [metric_ocean for [_, _, _, _, metric_ocean] in list_of_metrics if not np.isnan(metric_ocean)]
	result = {}
	print(f"Average {metric_name}: {sum(metrics) / len(metrics)}")
	result[f'Average_{metric_name}'] = sum(metrics) / len(metrics)
	print(f"Average {metric_name} NA Area: {sum(metrics_na) / len(metrics_na)}")
	result[f'Average_{metric_name}_NA_Area'] = sum(metrics_na) / len(metrics_na)
	print(f"Average {metric_name} Stone: {sum(metrics_stone) / len(metrics_stone)}")
	result[f"Average_{metric_name}_Stone"] =sum(metrics_stone) / len(metrics_stone)
	print(f"Average {metric_name} Glacier: {sum(metrics_glacier) / len(metrics_glacier)}")
	result[f"Average_{metric_name}_Glacier"] = sum(metrics_glacier) / len(metrics_glacier)
	print(f"Average {metric_name} Ocean and Ice Melange: {sum(metrics_ocean) / len(metrics_ocean)}")
	result[f"Average_{metric_name}_Ocean_and_Ice_Melange"] = sum(metrics_ocean) / len(metrics_ocean)

	return result

	def print_front_metric(name, metric):
	result = {}
	print(f"Average {name}: {sum(metric) / len(metric)}")
	result[f"Average {name}"] = sum(metric) / len(metric)
	return result


	def mask_prediction_with_bounding_box(post_complete_predicted_mask, file_name, bounding_boxes_directory):
	matching_bounding_box_file = get_matching_out_of_folder(file_name, bounding_boxes_directory)
	with open(os.path.join(bounding_boxes_directory, matching_bounding_box_file)) as f:
	coord_file_lines = f.readlines()
	left_upper_corner_x, left_upper_corner_y = [round(float(coord)) for coord in coord_file_lines[1].split(",")]
	left_lower_corner_x, left_lower_corner_y = [round(float(coord)) for coord in coord_file_lines[2].split(",")]
	right_lower_corner_x, right_lower_corner_y = [round(float(coord)) for coord in coord_file_lines[3].split(",")]
	right_upper_corner_x, right_upper_corner_y = [round(float(coord)) for coord in coord_file_lines[4].split(",")]

	# Make sure the Bounding Box coordinates are within the image
	if left_upper_corner_x < 0: left_upper_corner_x = 0
	if left_lower_corner_x < 0: left_lower_corner_x = 0
	if right_upper_corner_x > len(post_complete_predicted_mask[0]): right_upper_corner_x = len(post_complete_predicted_mask[0]) - 1
	if right_lower_corner_x > len(post_complete_predicted_mask[0]): right_lower_corner_x = len(post_complete_predicted_mask[0]) - 1
	if left_upper_corner_y > len(post_complete_predicted_mask): left_upper_corner_y = len(post_complete_predicted_mask) - 1
	if left_lower_corner_y < 0: left_lower_corner_y = 0
	if right_upper_corner_y > len(post_complete_predicted_mask): right_upper_corner_y = len(post_complete_predicted_mask) - 1
	if right_lower_corner_y < 0: right_lower_corner_y = 0

	# remember cv2 images have the shape (height, width)
	post_complete_predicted_mask[:right_lower_corner_y, :] = 0.0
	post_complete_predicted_mask[left_upper_corner_y:, :] = 0.0
	post_complete_predicted_mask[:, :left_upper_corner_x] = 0.0
	post_complete_predicted_mask[:, right_lower_corner_x:] = 0.0

	return post_complete_predicted_mask


	def post_processing(target_masks, complete_predicted_masks, bounding_boxes_directory, complete_test_directory):
	meter_threshold = 750 # in meter
	print("Post-processing ...\n\n")
	for file_name in complete_predicted_masks:
	prediction_name = file_name
	if file_name.endswith('_zone.png'):
	file_name = file_name[:-len("_zone.png")] + ".png"
	if file_name.endswith('_front.png'):
	file_name = file_name[:-len("front.png")] +".png"

	print(f"File: {file_name}")
	resolution = int(os.path.split(file_name)[1][:-4].split('_')[-3])
	# pixel_threshold (pixel) * resolution (m/pixel) = meter_threshold (m)
	pixel_threshold = meter_threshold / resolution
	complete_predicted_mask = cv2.imread(os.path.join(complete_test_directory, prediction_name).__str__(), cv2.IMREAD_GRAYSCALE)

	if target_masks == "zones":
	post_complete_predicted_mask = postprocess_zone_segmenation(complete_predicted_mask)
	post_complete_predicted_mask = extract_front_from_zones(post_complete_predicted_mask, pixel_threshold)
	else:
	complete_predicted_mask_class_labels = turn_colors_to_class_labels_front(complete_predicted_mask)
	post_complete_predicted_mask = postprocess_front_segmenation(complete_predicted_mask_class_labels, pixel_threshold)
	post_complete_predicted_mask = post_complete_predicted_mask * 255

	post_complete_predicted_mask = mask_prediction_with_bounding_box(post_complete_predicted_mask, file_name,
	bounding_boxes_directory)
	cv2.imwrite(os.path.join(complete_postprocessed_test_directory, file_name), post_complete_predicted_mask)


	def calculate_front_delineation_metric(complete_postprocessed_test_directory, post_processed_predicted_masks, directory_of_target_fronts, bounding_boxes_directory):
	list_of_mean_front_errors = []
	list_of_median_front_errors = []
	list_of_all_front_errors = []
	number_of_images_with_no_predicted_front = 0
	results = {}
	for file_name in post_processed_predicted_masks[:]:

	post_processed_predicted_mask = cv2.imread(
	os.path.join(complete_postprocessed_test_directory, file_name).__str__(), cv2.IMREAD_GRAYSCALE)
	matching_target_file = get_matching_out_of_folder(file_name, directory_of_target_fronts)
	target_front = cv2.imread(os.path.join(directory_of_target_fronts, matching_target_file).__str__(),
	cv2.IMREAD_GRAYSCALE)
	if file_name.endswith("_front.png"):
	resolution = int(os.path.split(file_name)[1][:-4].split('_')[-4])
	else:
	resolution = int(os.path.split(file_name)[1][:-4].split('_')[-3])

	# images need to be turned into a Tensor [0, ..., n_classes-1]
	post_processed_predicted_mask_class_labels = turn_colors_to_class_labels_front(post_processed_predicted_mask)
	target_front_class_labels = turn_colors_to_class_labels_front(target_front)

	if file_name.endswith('_front.png'):
	post_processed_predicted_mask_class_labels = mask_prediction_with_bounding_box(post_processed_predicted_mask_class_labels, file_name[:-len('_front.png')]+'.png', bounding_boxes_directory)
	post_processed_predicted_mask_class_labels = skeletonize(post_processed_predicted_mask_class_labels)
	front_is_present_flag, mean_error, median_error, errors = front_error(
	post_processed_predicted_mask_class_labels, target_front_class_labels)

	if not front_is_present_flag:
	number_of_images_with_no_predicted_front += 1
	else:
	list_of_mean_front_errors.append(resolution * mean_error)
	list_of_median_front_errors.append(resolution * median_error)
	list_of_all_front_errors = np.concatenate((list_of_all_front_errors, resolution * errors))
	print(f"Number of images with no predicted front: {number_of_images_with_no_predicted_front}")
	results["Number_no_front"] = number_of_images_with_no_predicted_front
	if number_of_images_with_no_predicted_front >= len(post_processed_predicted_masks):
	print(f"Number of images with no predicted front is equal to complete set of images. No metrics can be calculated.")
	return [], {}
	list_of_mean_front_errors_without_nan = [front_error for front_error in list_of_mean_front_errors if
	not np.isnan(front_error)]
	list_of_median_front_errors_without_nan = [front_error for front_error in list_of_median_front_errors if
	not np.isnan(front_error)]
	print(f"Mean-mean distance error (in meters): {sum(list_of_mean_front_errors_without_nan) / len(list_of_mean_front_errors_without_nan)}")
	results["Mean_mean_distance"] = sum(list_of_mean_front_errors_without_nan) / len(list_of_mean_front_errors_without_nan)
	print(f"Mean-median distance error (in meters): {sum(list_of_median_front_errors_without_nan) / len(list_of_median_front_errors_without_nan)}")
	results["Mean_median_distance"] = sum(list_of_median_front_errors_without_nan) / len(list_of_median_front_errors_without_nan)

	list_of_mean_front_errors_without_nan = np.array(list_of_mean_front_errors_without_nan)
	list_of_median_front_errors_without_nan = np.array(list_of_median_front_errors_without_nan)
	print(f"Median-mean distance error (in meters): {np.median(list_of_mean_front_errors_without_nan)}")
	results["Median_mean_distance"] = np.median(list_of_mean_front_errors_without_nan)
	print(f"Median-median distance error (in meters): {np.median(list_of_median_front_errors_without_nan)}")
	results["Median_median_distance"] = np.median(list_of_median_front_errors_without_nan)

	list_of_all_front_errors_without_nan = [front_error for front_error in list_of_all_front_errors if
	not np.isnan(front_error)]
	list_of_all_front_errors_without_nan = np.array(list_of_all_front_errors_without_nan)
	confidence_interval = st.norm.interval(alpha=0.95,
	loc=np.mean(list_of_all_front_errors_without_nan),
	scale=st.sem(list_of_all_front_errors_without_nan))
	mean = np.mean(list_of_all_front_errors_without_nan)
	std = np.std(list_of_all_front_errors_without_nan)
	print(f"Confidence interval: {confidence_interval}, mean: {mean}, standard deviation: {std}")
	results["Confidence_interval"] = confidence_interval
	results['mean'] = mean
	results['standard_deviation'] = std
	return list_of_mean_front_errors_without_nan, results

	def calculate_segmentation_metrics(target_mask_modality, complete_predicted_masks, complete_test_directory,
	directory_of_complete_targets):
	print("Calculate segmentation metrics ...\n\n")
	list_of_ious = []
	list_of_precisions = []
	list_of_recalls = []
	list_of_f1_scores = []
	result = {}
	for file_name in complete_predicted_masks:
	print(f"File: {file_name}")
	complete_predicted_mask = cv2.imread(os.path.join(complete_test_directory, file_name).__str__(),
	cv2.IMREAD_GRAYSCALE)
	matching_target_file = get_matching_out_of_folder(file_name, directory_of_complete_targets)
	complete_target = cv2.imread(os.path.join(directory_of_complete_targets, matching_target_file).__str__(),
	cv2.IMREAD_GRAYSCALE)

	if target_mask_modality == "zones":
	# images need to be turned into a Tensor [0, ..., n_classes-1]
	complete_predicted_mask_class_labels = turn_colors_to_class_labels_zones(complete_predicted_mask)
	complete_target_class_labels = turn_colors_to_class_labels_zones(complete_target)
	# Segmentation evaluation metrics
	list_of_ious.append(
	multi_class_metric(jaccard_score, complete_predicted_mask_class_labels, complete_target_class_labels))
	list_of_precisions.append(
	multi_class_metric(precision_score, complete_predicted_mask_class_labels, complete_target_class_labels))
	list_of_recalls.append(
	multi_class_metric(recall_score, complete_predicted_mask_class_labels, complete_target_class_labels))
	list_of_f1_scores.append(
	multi_class_metric(f1_score, complete_predicted_mask_class_labels, complete_target_class_labels))
	else:
	# images need to be turned into a Tensor [0, ..., n_classes-1]
	complete_predicted_mask_class_labels = turn_colors_to_class_labels_front(complete_predicted_mask)
	complete_target_class_labels = turn_colors_to_class_labels_front(complete_target)
	# Segmentation evaluation metrics
	flattened_complete_target_class_labels = np.ndarray.flatten(complete_target_class_labels)
	flattened_complete_predicted_mask_class_labels = np.ndarray.flatten(complete_predicted_mask_class_labels)
	list_of_ious.append(
	jaccard_score(flattened_complete_target_class_labels, flattened_complete_predicted_mask_class_labels))
	list_of_precisions.append(
	precision_score(flattened_complete_target_class_labels, flattened_complete_predicted_mask_class_labels))
	list_of_recalls.append(
	recall_score(flattened_complete_target_class_labels, flattened_complete_predicted_mask_class_labels))
	list_of_f1_scores.append(
	f1_score(flattened_complete_target_class_labels, flattened_complete_predicted_mask_class_labels))

	if target_mask_modality == "zones":
	result_precision = print_zone_metrics("Precision", list_of_precisions)
	result["Zone_Precision"] = result_precision
	result_recal = print_zone_metrics("Recall", list_of_recalls)
	result["Zone_Recall"] = result_recal
	result_f1 = print_zone_metrics("F1 Score", list_of_f1_scores)
	result["Zone_F1"] = result_f1
	result_iou = print_zone_metrics("IoU", list_of_ious)
	result["Zone_IoU"] = result_iou
	else:
	if len(list_of_precisions) > 0:
	result_precsions = print_front_metric("Precision", list_of_precisions)
	result["Front_Precsion"] = result_precsions
	if len(list_of_recalls) > 0:
	result_recall = print_front_metric("Recall", list_of_recalls)
	result["Front_Recall"] = result_recall
	if len(list_of_f1_scores):
	result_f1 = print_front_metric("F1 Score", list_of_f1_scores)
	result["Front_F1"] = result_f1
	if len(list_of_ious)>0:
	result_iou = print_front_metric("IoU", list_of_ious)
	result["Front_IoU"] = result_iou

	return result

	def check_whether_winter_half_year(name):
	split_name = name[:-4].split('_')
	if split_name[0] == "COL" or split_name[0] == "JAC":
	nord_halbkugel = True
	else: # Jorum, Maple, Crane, SI, DBE
	nord_halbkugel = False
	month = int(split_name[1].split('-')[1])
	if nord_halbkugel:
	if month < 4 or month > 8:
	winter = True
	else:
	winter = False
	else:
	if month < 4 or month > 8:
	winter = False
	else:
	winter = True
	return winter


	def front_delineation_metric(modality, complete_postprocessed_test_directory, directory_of_target_fronts, bounding_boxes_directory):
	print("Calculating distance errors ...\n\n")
	if modality == 'front':
	post_processed_predicted_masks = list(file for file in os.listdir(complete_postprocessed_test_directory) if file.endswith('_front.png'))

	elif modality == 'zone':
	post_processed_predicted_masks = list(file for file in os.listdir(complete_postprocessed_test_directory))

	print("")
	print("####################################################################")
	print(f"# Results for all images")
	print("####################################################################")
	fig = px.box(None, points="all", template="plotly_white", log_x=True, height=300)
	G10 = px.colors.qualitative.G10
	width = 0.5
	list_of_mean_front_errors_without_nan, result_all = calculate_front_delineation_metric(complete_postprocessed_test_directory, post_processed_predicted_masks, directory_of_target_fronts, bounding_boxes_directory)
	np.savetxt(os.path.join(complete_postprocessed_test_directory, os.pardir, "distance_errors.txt"), list_of_mean_front_errors_without_nan)
	fig.add_trace(go.Box(x=list_of_mean_front_errors_without_nan, marker_color='orange', boxmean=True, boxpoints='all', name='all', width=width))

	results = {}
	results['Result_all'] = result_all

	# Season subsetting
	for season in ["winter", "summer"]:
	print("")
	print("####################################################################")
	print(f"# Results for only images in {season}")
	print("####################################################################")
	subset_of_predictions = []
	for file_name in post_processed_predicted_masks:
	winter = check_whether_winter_half_year(file_name)
	if (winter and season == "summer") or (not winter and season == "winter"):
	continue
	subset_of_predictions.append(file_name)
	if len(subset_of_predictions) == 0: continue
	all_errors, result_season = calculate_front_delineation_metric(complete_postprocessed_test_directory, subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory)
	if season == 'winter':
	color = G10[9]
	else:
	color = G10[8]
	print(season, np.mean(all_errors), np.std(all_errors))
	fig.add_trace(go.Box(x=all_errors, marker_color=color, boxmean=True, boxpoints='all', name=season, width=width, legendrank=0))

	results[season] = result_season
	fig.update_layout(showlegend=False, font=dict(family="Times New Roma", size=12))
	fig.update_xaxes(title='front delineation error (m)')
	fig.update_layout(yaxis={'categoryorder':'array', 'categoryarray':['summer','winter','all']})
	fig.update_traces(orientation='h') # horizontal box plots
	fig.write_image("create_plots_new/output/error_season.pdf", format='pdf')


	# Glacier subsetting
	fig = px.box(None, points="all", template="plotly_white", log_x=True, height=300)
	fig.add_trace(go.Box(x=list_of_mean_front_errors_without_nan, marker_color='orange', boxmean=True, boxpoints='all', name='all', width=width,legendrank=7))
	color = {'COL': G10[3], 'Mapple': G10[4]}
	for glacier in ["Mapple", "COL", "Crane", "DBE", "JAC", "Jorum", "SI"]:
	print("")
	print("####################################################################")
	print(f"# Results for only images from {glacier}")
	print("####################################################################")
	subset_of_predictions = []
	for file_name in post_processed_predicted_masks:
	if not file_name[:-4].split('_')[0] == glacier:
	continue
	subset_of_predictions.append(file_name)
	if len(subset_of_predictions) == 0: continue
	all_errors, result_glacier = calculate_front_delineation_metric(complete_postprocessed_test_directory,subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory)
	print(glacier, np.mean(all_errors), np.std(all_errors))
	fig.add_trace(
	go.Box(x=all_errors, marker_color=color[glacier], boxmean=True, boxpoints='all', name=glacier, width=width, ))
	results[glacier] = {}
	results[glacier]['all'] = result_glacier
	fig.update_layout(showlegend=False, font=dict(family="Times New Roma", size=12))
	fig.update_xaxes(title='front delineation error (m)')
	fig.update_layout(yaxis={'categoryorder':'array', 'categoryarray':['Mapple', 'COL', 'all']})
	fig.update_traces(orientation='h') # horizontal box plots
	fig.write_image("create_plots_new/output/error_glacier.pdf", format='pdf')

	color = {'ERS': G10[9], 'RSAT': G10[1], 'ENVISAT': G10[8], 'PALSAR':G10[3], 'TSX':G10[4], 'TDX':G10[5], 'S1':G10[6]}
	# Sensor subsetting
	fig = px.box(None, points="all", template="plotly_white", log_x=True, height=500)
	fig.add_trace(go.Box(x=list_of_mean_front_errors_without_nan, marker_color='orange', boxmean=True, boxpoints='all', name='all', width=width))

	for sensor in ["RSAT", "S1", "ENVISAT", "ERS", "PALSAR", "TSX", "TDX"]:
	print("")
	print("####################################################################")
	print(f"# Results for only images from {sensor}")
	print("####################################################################")
	subset_of_predictions = []
	for file_name in post_processed_predicted_masks:
	if not file_name[:-4].split('_')[2] == sensor:
	continue
	subset_of_predictions.append(file_name)
	if len(subset_of_predictions) == 0: continue
	all_errors, result_sensor = calculate_front_delineation_metric(complete_postprocessed_test_directory,subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory)
	print(sensor, np.mean(all_errors), np.std(all_errors))
	fig.add_trace(
	go.Box(x=all_errors, marker_color=color[sensor], boxmean=True, boxpoints='all', name=sensor, width=width))
	results[sensor] = result_sensor
	fig.update_layout(showlegend=False, font=dict(family="Times New Roma", size=12))
	fig.update_xaxes(title='front delineation error (m)')
	fig.update_layout(yaxis={'categoryorder':'array', 'categoryarray':[ 'S1','TDX','TSX','PALSAR', 'ENVISAT', 'ERS','all']})
	fig.update_traces(orientation='h') # horizontal box plots
	fig.write_image("create_plots_new/output/error_satellite.pdf", format='pdf')
	exit()
	# Resolution subsetting
	fig = px.box(None, points="all", template="plotly_white", log_x=True)
	fig.add_trace(go.Box(x=list_of_mean_front_errors_without_nan, marker_color='orange', boxmean=True, boxpoints='all', name='all', width=width))
	color ={20: G10[9], 17:G10[8], 7:G10[3]}
	for res in [20, 17, 7]:
	print("")
	print("####################################################################")
	print(f"# Results for only images with a resolution of {res}")
	print("####################################################################")
	subset_of_predictions = []
	for file_name in post_processed_predicted_masks:
	if not int(file_name[:-4].split('_')[3]) == res:
	continue
	subset_of_predictions.append(file_name)
	if len(subset_of_predictions) == 0: continue
	all_errors, result_res = calculate_front_delineation_metric(complete_postprocessed_test_directory,subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory)
	fig.add_trace(
	go.Box(x=all_errors, marker_color=color[res], boxmean=True, boxpoints='all', name=res, width=width))
	results[res] = result_res
	fig.update_layout(showlegend=False, font=dict(family="Times New Roma", size=12))
	fig.update_xaxes(title='front delineation error (m)')
	fig.update_layout(yaxis={'categoryorder':'array', 'categoryarray':['7', '17', '20','all']})
	fig.update_traces(orientation='h') # horizontal box plots
	fig.write_image("create_plots_new/output/error_resolution.pdf", format='pdf')

	# Season and glacier subsetting
	for glacier in ["Mapple", "COL", "Crane", "DBE", "JAC", "Jorum", "SI"]:
	for season in ["winter", "summer"]:
	print("")
	print("####################################################################")
	print(f"# Results for only images in {season} and from {glacier}")
	print("####################################################################")
	subset_of_predictions = []
	for file_name in post_processed_predicted_masks:
	winter = check_whether_winter_half_year(file_name)
	if not file_name[:-4].split('_')[0] == glacier:
	continue
	if (winter and season == "summer") or (not winter and season == "winter"):
	continue
	subset_of_predictions.append(file_name)
	if len(subset_of_predictions) == 0: continue
	_, results_gla_season = calculate_front_delineation_metric(complete_postprocessed_test_directory, subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory)
	results[glacier][season] = results_gla_season

	return results

	def visualizations(complete_postprocessed_test_directory, directory_of_target_fronts, directory_of_sar_images,
	bounding_boxes_directory, visualizations_dir):
	print("Creating visualizations ...\n\n")
	post_processed_predicted_masks = os.listdir(os.path.join(complete_postprocessed_test_directory))
	for file_name in post_processed_predicted_masks:
	if not file_name.endswith('.png'):
	continue
	resolution = int(os.path.split(file_name)[1][:-4].split('_')[-3])
	if resolution < 10:
	dilation = 5
	else:
	dilation = 3

	if file_name.endswith('_front.png'):
	post_processed_predicted_mask = cv2.imread(os.path.join(complete_postprocessed_test_directory, file_name).__str__(), cv2.IMREAD_GRAYSCALE)
	post_processed_predicted_mask = mask_prediction_with_bounding_box(post_processed_predicted_mask, file_name[:-len('_front.png')]+'.png', bounding_boxes_directory)
	post_processed_predicted_mask[post_processed_predicted_mask > 1] =1
	post_processed_predicted_mask_skeletonized = skeletonize(post_processed_predicted_mask)
	post_processed_predicted_mask = np.zeros(post_processed_predicted_mask_skeletonized.shape)
	post_processed_predicted_mask[post_processed_predicted_mask_skeletonized] = 255
	matching_target_file = get_matching_out_of_folder(file_name[:-len('_front.png')]+'.png', directory_of_target_fronts)
	target_front = cv2.imread(os.path.join(directory_of_target_fronts, matching_target_file).__str__(), cv2.IMREAD_GRAYSCALE)
	matching_sar_file = get_matching_out_of_folder(file_name[:-len('_front.png')]+'.png', directory_of_sar_images)
	sar_image = cv2.imread(os.path.join(directory_of_sar_images, matching_sar_file).__str__(), cv2.IMREAD_GRAYSCALE)
	elif file_name.endswith('_zone.png'):
	continue
	elif file_name.endswith('_recon.png'):
	continue
	else:
	post_processed_predicted_mask = cv2.imread(
	os.path.join(complete_postprocessed_test_directory, file_name).__str__(), cv2.IMREAD_GRAYSCALE)
	matching_target_file = get_matching_out_of_folder(file_name, directory_of_target_fronts)
	target_front = cv2.imread(os.path.join(directory_of_target_fronts, matching_target_file).__str__(),cv2.IMREAD_GRAYSCALE)
	matching_sar_file = get_matching_out_of_folder(file_name, directory_of_sar_images)
	sar_image = cv2.imread(os.path.join(directory_of_sar_images, matching_sar_file).__str__(),cv2.IMREAD_GRAYSCALE)


	predicted_front = np.array(post_processed_predicted_mask)
	ground_truth_front = np.array(target_front)
	kernel = np.ones((dilation, dilation), np.uint8)
	predicted_front = cv2.dilate(predicted_front, kernel, iterations=1)
	ground_truth_front = cv2.dilate(ground_truth_front, kernel, iterations=1)

	sar_image = np.array(sar_image)
	sar_image_rgb = skimage.color.gray2rgb(sar_image)
	sar_image_rgb = np.uint8(sar_image_rgb)

	sar_image_rgb[predicted_front > 0] = [0, 255, 255] # b, g, r
	sar_image_rgb[ground_truth_front > 0] = [255, 51, 51]
	correct_prediction = np.logical_and(predicted_front, ground_truth_front)
	sar_image_rgb[correct_prediction > 0] = [255, 0, 255] # [51, 255, 51] # [0, 153, 0]

	# Insert Bounding Box
	matching_bounding_box_file = get_matching_out_of_folder(file_name, bounding_boxes_directory)
	with open(os.path.join(bounding_boxes_directory, matching_bounding_box_file)) as f:
	coord_file_lines = f.readlines()
	left_upper_corner_x, left_upper_corner_y = [round(float(coord)) for coord in coord_file_lines[1].split(",")]
	left_lower_corner_x, left_lower_corner_y = [round(float(coord)) for coord in coord_file_lines[2].split(",")]
	right_lower_corner_x, right_lower_corner_y = [round(float(coord)) for coord in coord_file_lines[3].split(",")]
	right_upper_corner_x, right_upper_corner_y = [round(float(coord)) for coord in coord_file_lines[4].split(",")]

	bounding_box = np.zeros((len(sar_image_rgb), len(sar_image_rgb[0])))
	if left_upper_corner_x < 0: left_upper_corner_x = 0
	if left_lower_corner_x < 0: left_lower_corner_x = 0
	if right_upper_corner_x > len(sar_image_rgb[0]): right_upper_corner_x = len(sar_image_rgb[0]) - 1
	if right_lower_corner_x > len(sar_image_rgb[0]): right_lower_corner_x = len(sar_image_rgb[0]) - 1
	if left_upper_corner_y > len(sar_image_rgb): left_upper_corner_y = len(sar_image_rgb) - 1
	if left_lower_corner_y < 0: left_lower_corner_y = 0
	if right_upper_corner_y > len(sar_image_rgb): right_upper_corner_y = len(sar_image_rgb) - 1
	if right_lower_corner_y < 0: right_lower_corner_y = 0

	bounding_box[left_upper_corner_y, left_upper_corner_x:right_upper_corner_x] = 1
	bounding_box[left_lower_corner_y, left_lower_corner_x:right_lower_corner_x] = 1
	bounding_box[left_lower_corner_y:left_upper_corner_y, left_upper_corner_x] = 1
	bounding_box[right_lower_corner_y:right_upper_corner_y, right_lower_corner_x] = 1
	bounding_box = cv2.dilate(bounding_box, kernel, iterations=1)
	sar_image_rgb[bounding_box > 0] = [255, 255, 0]

	cv2.imwrite(os.path.join(visualizations_dir, file_name), sar_image_rgb)

	def main(complete_test_directory, directory_of_complete_targets_zones, directory_of_complete_targets_fronts, directory_of_sar_images):
	# ###############################################################################################
	# CALCULATE SEGMENTATION METRICS (IoU & Hausdorff Distance)
	# ###############################################################################################
	complete_predicted_masks_zones = list(file for file in os.listdir(complete_test_directory) if file.endswith('_zone.png'))
	complete_predicted_masks_fronts = list(file for file in os.listdir(complete_test_directory) if file.endswith('_front.png'))
	src = Path(directory_of_sar_images).parent.parent.parent
	bounding_boxes_directory = os.path.join(src, "data_raw", "bounding_boxes")
	results = {}
	# only on zone

	if len(complete_predicted_masks_zones) > 0:
	results_seg = calculate_segmentation_metrics('zones', complete_predicted_masks_zones, complete_test_directory,
	directory_of_complete_targets_zones,)
	results['Zone_Segmentation'] = results_seg

	if len(complete_predicted_masks_fronts) >0:
	results_seg = calculate_segmentation_metrics('fronts', complete_predicted_masks_fronts,
	complete_test_directory,
	directory_of_complete_targets_fronts, )
	results['Front_Segmentation'] = results_seg

	# ###############################################################################################
	# POST-PROCESSING
	# ###############################################################################################
	src = Path(directory_of_sar_images).parent.parent.parent
	print(src)


	if len(complete_predicted_masks_zones) > 0:
	post_processing('zones', complete_predicted_masks_zones, bounding_boxes_directory, complete_test_directory)

	# ###############################################################################################
	# CALCULATE FRONT DELINEATION METRIC (Mean distance error)
	# ###############################################################################################

	if len(complete_predicted_masks_zones) > 0:
	print("Front delineation from ZONE post processed")
	results_zone = front_delineation_metric('zone', complete_postprocessed_test_directory, directory_of_complete_targets_fronts, bounding_boxes_directory)
	results['Zone_Delineation'] = results_zone


	if len(complete_predicted_masks_fronts) > 0:
	print("Front delineation from FRONT directly")
	results_front = front_delineation_metric('front', complete_test_directory, directory_of_complete_targets_fronts, bounding_boxes_directory)
	results['Front_Delineation'] = results_front

	results_file = open(complete_test_directory+'/eval_results.json', "w")
	json.dump(results, results_file)

	# ###############################################################################################
	# MAKE VISUALIZATIONS
	# ###############################################################################################
	if len(complete_predicted_masks_zones) > 0:
	visualizations(complete_postprocessed_test_directory, directory_of_complete_targets_fronts, directory_of_sar_images,
	bounding_boxes_directory, visualizations_dir)

	if len(complete_predicted_masks_fronts) > 0:
	front_prediction_dir = complete_test_directory

	visualizations(front_prediction_dir, directory_of_complete_targets_fronts, directory_of_sar_images,
	bounding_boxes_directory, visualizations_dir)


	if __name__ == "__main__":
	print("Start Evaluation")
	parser = ArgumentParser(add_help=False)
	parser.add_argument('--predictions', help="Directory with predictions as png")
	parser.add_argument('--labels_fronts', help="Directory with labels as png")
	parser.add_argument('--labels_zones', help="Directory with labels as png")
	parser.add_argument('--sar_images', help="Directory with sar images")
	hparams = parser.parse_args()

	complete_test_directory = hparams.predictions
	complete_postprocessed_test_directory = os.path.join(complete_test_directory, "postprocessed")

	os.makedirs(complete_postprocessed_test_directory, exist_ok=True)

	visualizations_dir = os.path.join(complete_test_directory, "visualization")
	os.makedirs(visualizations_dir, exist_ok=True)

	main(hparams.predictions, hparams.labels_zones, hparams.labels_fronts, hparams.sar_images)