from argparse import ArgumentParser import os import numpy as np from sklearn.metrics import precision_score, recall_score, f1_score, jaccard_score from data_processing.data_postprocessing import postprocess_zone_segmenation, postprocess_front_segmenation, extract_front_from_zones import torch.nn as nn #from segmentation_models_pytorch.losses.dice import DiceLoss #from PIL import Image #from models.front_segmentation_model import DistanceMapBCE import re from pathlib import Path import cv2 import scipy.stats as st from scipy.spatial import distance import skimage import matplotlib.pyplot as plt from skimage.morphology import skeletonize import json import plotly.graph_objects as go import plotly.express as px import plotly.io as pio import os pio.kaleido.scope.mathjax = None def front_error(prediction, label): """ prediction: mask of the front prediction (WxH) label: mask of the front label (WxH) returns the mean distance of the two fronts """ front_is_present_flag = True polyline_pred = np.nonzero(prediction) polyline_label = np.nonzero(label) # Generate Nx2 matrix of pixels that represent the front pred_coords = np.array(list(zip(polyline_pred[0], polyline_pred[1]))) mask_coords = np.array(list(zip(polyline_label[0], polyline_label[1]))) # Return NaN if front is not detected in either pred or mask if pred_coords.shape[0] == 0 or mask_coords.shape[0] == 0: front_is_present_flag = False return front_is_present_flag, np.nan, np.nan, np.nan # Generate the pairwise distances between each point and the closest point in the other array distances1 = distance.cdist(pred_coords, mask_coords).min(axis=1) distances2 = distance.cdist(mask_coords, pred_coords).min(axis=1) distances = np.concatenate((distances1, distances2)) # Calculate the average distance between each point and the closest point in the other array mean_distance = np.mean(distances) median_distance = np.median(distances) return front_is_present_flag, mean_distance, median_distance, distances def multi_class_metric(metric_function, complete_predicted_mask, complete_target): metrics = [] metric_na, metric_stone, metric_glacier, metric_ocean = metric_function(np.ndarray.flatten(complete_target), np.ndarray.flatten(complete_predicted_mask), average=None) metric_macro_average = (metric_na + metric_stone + metric_glacier + metric_ocean) / 4 metrics.append(metric_macro_average) metrics.append(metric_na) metrics.append(metric_stone) metrics.append(metric_glacier) metrics.append(metric_ocean) return metrics def get_matching_out_of_folder(file_name, folder): files = os.listdir(folder) matching_files = [a for a in files if re.match(pattern=os.path.split(file_name)[1][:-4], string=os.path.split(a)[1])] if len(matching_files) > 1: print("Something went wrong!") print(f"targets_matching: {matching_files}") if len(matching_files) < 1: print("Something went wrong! No matches found") return matching_files[0] def turn_colors_to_class_labels_zones(mask): mask_class_labels = np.copy(mask) mask_class_labels[mask == 0] = 0 mask_class_labels[mask == 64] = 1 mask_class_labels[mask == 127] = 2 mask_class_labels[mask == 254] = 3 return mask_class_labels def turn_colors_to_class_labels_front(mask): mask_class_labels = np.copy(mask) mask_class_labels[mask == 0] = 0 mask_class_labels[mask == 255] = 1 return mask_class_labels def print_zone_metrics(metric_name, list_of_metrics): metrics = [metric for [metric, _, _, _, _] in list_of_metrics if not np.isnan(metric)] metrics_na = [metric_na for [_, metric_na, _, _, _] in list_of_metrics if not np.isnan(metric_na)] metrics_stone = [metric_stone for [_, _, metric_stone, _, _] in list_of_metrics if not np.isnan(metric_stone)] metrics_glacier = [metric_glacier for [_, _, _, metric_glacier, _] in list_of_metrics if not np.isnan(metric_glacier)] metrics_ocean = [metric_ocean for [_, _, _, _, metric_ocean] in list_of_metrics if not np.isnan(metric_ocean)] result = {} print(f"Average {metric_name}: {sum(metrics) / len(metrics)}") result[f'Average_{metric_name}'] = sum(metrics) / len(metrics) print(f"Average {metric_name} NA Area: {sum(metrics_na) / len(metrics_na)}") result[f'Average_{metric_name}_NA_Area'] = sum(metrics_na) / len(metrics_na) print(f"Average {metric_name} Stone: {sum(metrics_stone) / len(metrics_stone)}") result[f"Average_{metric_name}_Stone"] =sum(metrics_stone) / len(metrics_stone) print(f"Average {metric_name} Glacier: {sum(metrics_glacier) / len(metrics_glacier)}") result[f"Average_{metric_name}_Glacier"] = sum(metrics_glacier) / len(metrics_glacier) print(f"Average {metric_name} Ocean and Ice Melange: {sum(metrics_ocean) / len(metrics_ocean)}") result[f"Average_{metric_name}_Ocean_and_Ice_Melange"] = sum(metrics_ocean) / len(metrics_ocean) return result def print_front_metric(name, metric): result = {} print(f"Average {name}: {sum(metric) / len(metric)}") result[f"Average {name}"] = sum(metric) / len(metric) return result def mask_prediction_with_bounding_box(post_complete_predicted_mask, file_name, bounding_boxes_directory): matching_bounding_box_file = get_matching_out_of_folder(file_name, bounding_boxes_directory) with open(os.path.join(bounding_boxes_directory, matching_bounding_box_file)) as f: coord_file_lines = f.readlines() left_upper_corner_x, left_upper_corner_y = [round(float(coord)) for coord in coord_file_lines[1].split(",")] left_lower_corner_x, left_lower_corner_y = [round(float(coord)) for coord in coord_file_lines[2].split(",")] right_lower_corner_x, right_lower_corner_y = [round(float(coord)) for coord in coord_file_lines[3].split(",")] right_upper_corner_x, right_upper_corner_y = [round(float(coord)) for coord in coord_file_lines[4].split(",")] # Make sure the Bounding Box coordinates are within the image if left_upper_corner_x < 0: left_upper_corner_x = 0 if left_lower_corner_x < 0: left_lower_corner_x = 0 if right_upper_corner_x > len(post_complete_predicted_mask[0]): right_upper_corner_x = len(post_complete_predicted_mask[0]) - 1 if right_lower_corner_x > len(post_complete_predicted_mask[0]): right_lower_corner_x = len(post_complete_predicted_mask[0]) - 1 if left_upper_corner_y > len(post_complete_predicted_mask): left_upper_corner_y = len(post_complete_predicted_mask) - 1 if left_lower_corner_y < 0: left_lower_corner_y = 0 if right_upper_corner_y > len(post_complete_predicted_mask): right_upper_corner_y = len(post_complete_predicted_mask) - 1 if right_lower_corner_y < 0: right_lower_corner_y = 0 # remember cv2 images have the shape (height, width) post_complete_predicted_mask[:right_lower_corner_y, :] = 0.0 post_complete_predicted_mask[left_upper_corner_y:, :] = 0.0 post_complete_predicted_mask[:, :left_upper_corner_x] = 0.0 post_complete_predicted_mask[:, right_lower_corner_x:] = 0.0 return post_complete_predicted_mask def post_processing(target_masks, complete_predicted_masks, bounding_boxes_directory, complete_test_directory): meter_threshold = 750 # in meter print("Post-processing ...\n\n") for file_name in complete_predicted_masks: prediction_name = file_name if file_name.endswith('_zone.png'): file_name = file_name[:-len("_zone.png")] + ".png" if file_name.endswith('_front.png'): file_name = file_name[:-len("front.png")] +".png" print(f"File: {file_name}") resolution = int(os.path.split(file_name)[1][:-4].split('_')[-3]) # pixel_threshold (pixel) * resolution (m/pixel) = meter_threshold (m) pixel_threshold = meter_threshold / resolution complete_predicted_mask = cv2.imread(os.path.join(complete_test_directory, prediction_name).__str__(), cv2.IMREAD_GRAYSCALE) if target_masks == "zones": post_complete_predicted_mask = postprocess_zone_segmenation(complete_predicted_mask) post_complete_predicted_mask = extract_front_from_zones(post_complete_predicted_mask, pixel_threshold) else: complete_predicted_mask_class_labels = turn_colors_to_class_labels_front(complete_predicted_mask) post_complete_predicted_mask = postprocess_front_segmenation(complete_predicted_mask_class_labels, pixel_threshold) post_complete_predicted_mask = post_complete_predicted_mask * 255 post_complete_predicted_mask = mask_prediction_with_bounding_box(post_complete_predicted_mask, file_name, bounding_boxes_directory) cv2.imwrite(os.path.join(complete_postprocessed_test_directory, file_name), post_complete_predicted_mask) def calculate_front_delineation_metric(complete_postprocessed_test_directory, post_processed_predicted_masks, directory_of_target_fronts, bounding_boxes_directory): list_of_mean_front_errors = [] list_of_median_front_errors = [] list_of_all_front_errors = [] number_of_images_with_no_predicted_front = 0 results = {} for file_name in post_processed_predicted_masks[:]: post_processed_predicted_mask = cv2.imread( os.path.join(complete_postprocessed_test_directory, file_name).__str__(), cv2.IMREAD_GRAYSCALE) matching_target_file = get_matching_out_of_folder(file_name, directory_of_target_fronts) target_front = cv2.imread(os.path.join(directory_of_target_fronts, matching_target_file).__str__(), cv2.IMREAD_GRAYSCALE) if file_name.endswith("_front.png"): resolution = int(os.path.split(file_name)[1][:-4].split('_')[-4]) else: resolution = int(os.path.split(file_name)[1][:-4].split('_')[-3]) # images need to be turned into a Tensor [0, ..., n_classes-1] post_processed_predicted_mask_class_labels = turn_colors_to_class_labels_front(post_processed_predicted_mask) target_front_class_labels = turn_colors_to_class_labels_front(target_front) if file_name.endswith('_front.png'): post_processed_predicted_mask_class_labels = mask_prediction_with_bounding_box(post_processed_predicted_mask_class_labels, file_name[:-len('_front.png')]+'.png', bounding_boxes_directory) post_processed_predicted_mask_class_labels = skeletonize(post_processed_predicted_mask_class_labels) front_is_present_flag, mean_error, median_error, errors = front_error( post_processed_predicted_mask_class_labels, target_front_class_labels) if not front_is_present_flag: number_of_images_with_no_predicted_front += 1 else: list_of_mean_front_errors.append(resolution * mean_error) list_of_median_front_errors.append(resolution * median_error) list_of_all_front_errors = np.concatenate((list_of_all_front_errors, resolution * errors)) print(f"Number of images with no predicted front: {number_of_images_with_no_predicted_front}") results["Number_no_front"] = number_of_images_with_no_predicted_front if number_of_images_with_no_predicted_front >= len(post_processed_predicted_masks): print(f"Number of images with no predicted front is equal to complete set of images. No metrics can be calculated.") return [], {} list_of_mean_front_errors_without_nan = [front_error for front_error in list_of_mean_front_errors if not np.isnan(front_error)] list_of_median_front_errors_without_nan = [front_error for front_error in list_of_median_front_errors if not np.isnan(front_error)] print(f"Mean-mean distance error (in meters): {sum(list_of_mean_front_errors_without_nan) / len(list_of_mean_front_errors_without_nan)}") results["Mean_mean_distance"] = sum(list_of_mean_front_errors_without_nan) / len(list_of_mean_front_errors_without_nan) print(f"Mean-median distance error (in meters): {sum(list_of_median_front_errors_without_nan) / len(list_of_median_front_errors_without_nan)}") results["Mean_median_distance"] = sum(list_of_median_front_errors_without_nan) / len(list_of_median_front_errors_without_nan) list_of_mean_front_errors_without_nan = np.array(list_of_mean_front_errors_without_nan) list_of_median_front_errors_without_nan = np.array(list_of_median_front_errors_without_nan) print(f"Median-mean distance error (in meters): {np.median(list_of_mean_front_errors_without_nan)}") results["Median_mean_distance"] = np.median(list_of_mean_front_errors_without_nan) print(f"Median-median distance error (in meters): {np.median(list_of_median_front_errors_without_nan)}") results["Median_median_distance"] = np.median(list_of_median_front_errors_without_nan) list_of_all_front_errors_without_nan = [front_error for front_error in list_of_all_front_errors if not np.isnan(front_error)] list_of_all_front_errors_without_nan = np.array(list_of_all_front_errors_without_nan) confidence_interval = st.norm.interval(alpha=0.95, loc=np.mean(list_of_all_front_errors_without_nan), scale=st.sem(list_of_all_front_errors_without_nan)) mean = np.mean(list_of_all_front_errors_without_nan) std = np.std(list_of_all_front_errors_without_nan) print(f"Confidence interval: {confidence_interval}, mean: {mean}, standard deviation: {std}") results["Confidence_interval"] = confidence_interval results['mean'] = mean results['standard_deviation'] = std return list_of_mean_front_errors_without_nan, results def calculate_segmentation_metrics(target_mask_modality, complete_predicted_masks, complete_test_directory, directory_of_complete_targets): print("Calculate segmentation metrics ...\n\n") list_of_ious = [] list_of_precisions = [] list_of_recalls = [] list_of_f1_scores = [] result = {} for file_name in complete_predicted_masks: print(f"File: {file_name}") complete_predicted_mask = cv2.imread(os.path.join(complete_test_directory, file_name).__str__(), cv2.IMREAD_GRAYSCALE) matching_target_file = get_matching_out_of_folder(file_name, directory_of_complete_targets) complete_target = cv2.imread(os.path.join(directory_of_complete_targets, matching_target_file).__str__(), cv2.IMREAD_GRAYSCALE) if target_mask_modality == "zones": # images need to be turned into a Tensor [0, ..., n_classes-1] complete_predicted_mask_class_labels = turn_colors_to_class_labels_zones(complete_predicted_mask) complete_target_class_labels = turn_colors_to_class_labels_zones(complete_target) # Segmentation evaluation metrics list_of_ious.append( multi_class_metric(jaccard_score, complete_predicted_mask_class_labels, complete_target_class_labels)) list_of_precisions.append( multi_class_metric(precision_score, complete_predicted_mask_class_labels, complete_target_class_labels)) list_of_recalls.append( multi_class_metric(recall_score, complete_predicted_mask_class_labels, complete_target_class_labels)) list_of_f1_scores.append( multi_class_metric(f1_score, complete_predicted_mask_class_labels, complete_target_class_labels)) else: # images need to be turned into a Tensor [0, ..., n_classes-1] complete_predicted_mask_class_labels = turn_colors_to_class_labels_front(complete_predicted_mask) complete_target_class_labels = turn_colors_to_class_labels_front(complete_target) # Segmentation evaluation metrics flattened_complete_target_class_labels = np.ndarray.flatten(complete_target_class_labels) flattened_complete_predicted_mask_class_labels = np.ndarray.flatten(complete_predicted_mask_class_labels) list_of_ious.append( jaccard_score(flattened_complete_target_class_labels, flattened_complete_predicted_mask_class_labels)) list_of_precisions.append( precision_score(flattened_complete_target_class_labels, flattened_complete_predicted_mask_class_labels)) list_of_recalls.append( recall_score(flattened_complete_target_class_labels, flattened_complete_predicted_mask_class_labels)) list_of_f1_scores.append( f1_score(flattened_complete_target_class_labels, flattened_complete_predicted_mask_class_labels)) if target_mask_modality == "zones": result_precision = print_zone_metrics("Precision", list_of_precisions) result["Zone_Precision"] = result_precision result_recal = print_zone_metrics("Recall", list_of_recalls) result["Zone_Recall"] = result_recal result_f1 = print_zone_metrics("F1 Score", list_of_f1_scores) result["Zone_F1"] = result_f1 result_iou = print_zone_metrics("IoU", list_of_ious) result["Zone_IoU"] = result_iou else: if len(list_of_precisions) > 0: result_precsions = print_front_metric("Precision", list_of_precisions) result["Front_Precsion"] = result_precsions if len(list_of_recalls) > 0: result_recall = print_front_metric("Recall", list_of_recalls) result["Front_Recall"] = result_recall if len(list_of_f1_scores): result_f1 = print_front_metric("F1 Score", list_of_f1_scores) result["Front_F1"] = result_f1 if len(list_of_ious)>0: result_iou = print_front_metric("IoU", list_of_ious) result["Front_IoU"] = result_iou return result def check_whether_winter_half_year(name): split_name = name[:-4].split('_') if split_name[0] == "COL" or split_name[0] == "JAC": nord_halbkugel = True else: # Jorum, Maple, Crane, SI, DBE nord_halbkugel = False month = int(split_name[1].split('-')[1]) if nord_halbkugel: if month < 4 or month > 8: winter = True else: winter = False else: if month < 4 or month > 8: winter = False else: winter = True return winter def front_delineation_metric(modality, complete_postprocessed_test_directory, directory_of_target_fronts, bounding_boxes_directory): print("Calculating distance errors ...\n\n") if modality == 'front': post_processed_predicted_masks = list(file for file in os.listdir(complete_postprocessed_test_directory) if file.endswith('_front.png')) elif modality == 'zone': post_processed_predicted_masks = list(file for file in os.listdir(complete_postprocessed_test_directory)) print("") print("####################################################################") print(f"# Results for all images") print("####################################################################") fig = px.box(None, points="all", template="plotly_white", log_x=True, height=300) G10 = px.colors.qualitative.G10 width = 0.5 list_of_mean_front_errors_without_nan, result_all = calculate_front_delineation_metric(complete_postprocessed_test_directory, post_processed_predicted_masks, directory_of_target_fronts, bounding_boxes_directory) np.savetxt(os.path.join(complete_postprocessed_test_directory, os.pardir, "distance_errors.txt"), list_of_mean_front_errors_without_nan) fig.add_trace(go.Box(x=list_of_mean_front_errors_without_nan, marker_color='orange', boxmean=True, boxpoints='all', name='all', width=width)) results = {} results['Result_all'] = result_all # Season subsetting for season in ["winter", "summer"]: print("") print("####################################################################") print(f"# Results for only images in {season}") print("####################################################################") subset_of_predictions = [] for file_name in post_processed_predicted_masks: winter = check_whether_winter_half_year(file_name) if (winter and season == "summer") or (not winter and season == "winter"): continue subset_of_predictions.append(file_name) if len(subset_of_predictions) == 0: continue all_errors, result_season = calculate_front_delineation_metric(complete_postprocessed_test_directory, subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory) if season == 'winter': color = G10[9] else: color = G10[8] print(season, np.mean(all_errors), np.std(all_errors)) fig.add_trace(go.Box(x=all_errors, marker_color=color, boxmean=True, boxpoints='all', name=season, width=width, legendrank=0)) results[season] = result_season fig.update_layout(showlegend=False, font=dict(family="Times New Roma", size=12)) fig.update_xaxes(title='front delineation error (m)') fig.update_layout(yaxis={'categoryorder':'array', 'categoryarray':['summer','winter','all']}) fig.update_traces(orientation='h') # horizontal box plots fig.write_image("create_plots_new/output/error_season.pdf", format='pdf') # Glacier subsetting fig = px.box(None, points="all", template="plotly_white", log_x=True, height=300) fig.add_trace(go.Box(x=list_of_mean_front_errors_without_nan, marker_color='orange', boxmean=True, boxpoints='all', name='all', width=width,legendrank=7)) color = {'COL': G10[3], 'Mapple': G10[4]} for glacier in ["Mapple", "COL", "Crane", "DBE", "JAC", "Jorum", "SI"]: print("") print("####################################################################") print(f"# Results for only images from {glacier}") print("####################################################################") subset_of_predictions = [] for file_name in post_processed_predicted_masks: if not file_name[:-4].split('_')[0] == glacier: continue subset_of_predictions.append(file_name) if len(subset_of_predictions) == 0: continue all_errors, result_glacier = calculate_front_delineation_metric(complete_postprocessed_test_directory,subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory) print(glacier, np.mean(all_errors), np.std(all_errors)) fig.add_trace( go.Box(x=all_errors, marker_color=color[glacier], boxmean=True, boxpoints='all', name=glacier, width=width, )) results[glacier] = {} results[glacier]['all'] = result_glacier fig.update_layout(showlegend=False, font=dict(family="Times New Roma", size=12)) fig.update_xaxes(title='front delineation error (m)') fig.update_layout(yaxis={'categoryorder':'array', 'categoryarray':['Mapple', 'COL', 'all']}) fig.update_traces(orientation='h') # horizontal box plots fig.write_image("create_plots_new/output/error_glacier.pdf", format='pdf') color = {'ERS': G10[9], 'RSAT': G10[1], 'ENVISAT': G10[8], 'PALSAR':G10[3], 'TSX':G10[4], 'TDX':G10[5], 'S1':G10[6]} # Sensor subsetting fig = px.box(None, points="all", template="plotly_white", log_x=True, height=500) fig.add_trace(go.Box(x=list_of_mean_front_errors_without_nan, marker_color='orange', boxmean=True, boxpoints='all', name='all', width=width)) for sensor in ["RSAT", "S1", "ENVISAT", "ERS", "PALSAR", "TSX", "TDX"]: print("") print("####################################################################") print(f"# Results for only images from {sensor}") print("####################################################################") subset_of_predictions = [] for file_name in post_processed_predicted_masks: if not file_name[:-4].split('_')[2] == sensor: continue subset_of_predictions.append(file_name) if len(subset_of_predictions) == 0: continue all_errors, result_sensor = calculate_front_delineation_metric(complete_postprocessed_test_directory,subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory) print(sensor, np.mean(all_errors), np.std(all_errors)) fig.add_trace( go.Box(x=all_errors, marker_color=color[sensor], boxmean=True, boxpoints='all', name=sensor, width=width)) results[sensor] = result_sensor fig.update_layout(showlegend=False, font=dict(family="Times New Roma", size=12)) fig.update_xaxes(title='front delineation error (m)') fig.update_layout(yaxis={'categoryorder':'array', 'categoryarray':[ 'S1','TDX','TSX','PALSAR', 'ENVISAT', 'ERS','all']}) fig.update_traces(orientation='h') # horizontal box plots fig.write_image("create_plots_new/output/error_satellite.pdf", format='pdf') exit() # Resolution subsetting fig = px.box(None, points="all", template="plotly_white", log_x=True) fig.add_trace(go.Box(x=list_of_mean_front_errors_without_nan, marker_color='orange', boxmean=True, boxpoints='all', name='all', width=width)) color ={20: G10[9], 17:G10[8], 7:G10[3]} for res in [20, 17, 7]: print("") print("####################################################################") print(f"# Results for only images with a resolution of {res}") print("####################################################################") subset_of_predictions = [] for file_name in post_processed_predicted_masks: if not int(file_name[:-4].split('_')[3]) == res: continue subset_of_predictions.append(file_name) if len(subset_of_predictions) == 0: continue all_errors, result_res = calculate_front_delineation_metric(complete_postprocessed_test_directory,subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory) fig.add_trace( go.Box(x=all_errors, marker_color=color[res], boxmean=True, boxpoints='all', name=res, width=width)) results[res] = result_res fig.update_layout(showlegend=False, font=dict(family="Times New Roma", size=12)) fig.update_xaxes(title='front delineation error (m)') fig.update_layout(yaxis={'categoryorder':'array', 'categoryarray':['7', '17', '20','all']}) fig.update_traces(orientation='h') # horizontal box plots fig.write_image("create_plots_new/output/error_resolution.pdf", format='pdf') # Season and glacier subsetting for glacier in ["Mapple", "COL", "Crane", "DBE", "JAC", "Jorum", "SI"]: for season in ["winter", "summer"]: print("") print("####################################################################") print(f"# Results for only images in {season} and from {glacier}") print("####################################################################") subset_of_predictions = [] for file_name in post_processed_predicted_masks: winter = check_whether_winter_half_year(file_name) if not file_name[:-4].split('_')[0] == glacier: continue if (winter and season == "summer") or (not winter and season == "winter"): continue subset_of_predictions.append(file_name) if len(subset_of_predictions) == 0: continue _, results_gla_season = calculate_front_delineation_metric(complete_postprocessed_test_directory, subset_of_predictions, directory_of_target_fronts, bounding_boxes_directory) results[glacier][season] = results_gla_season return results def visualizations(complete_postprocessed_test_directory, directory_of_target_fronts, directory_of_sar_images, bounding_boxes_directory, visualizations_dir): print("Creating visualizations ...\n\n") post_processed_predicted_masks = os.listdir(os.path.join(complete_postprocessed_test_directory)) for file_name in post_processed_predicted_masks: if not file_name.endswith('.png'): continue resolution = int(os.path.split(file_name)[1][:-4].split('_')[-3]) if resolution < 10: dilation = 5 else: dilation = 3 if file_name.endswith('_front.png'): post_processed_predicted_mask = cv2.imread(os.path.join(complete_postprocessed_test_directory, file_name).__str__(), cv2.IMREAD_GRAYSCALE) post_processed_predicted_mask = mask_prediction_with_bounding_box(post_processed_predicted_mask, file_name[:-len('_front.png')]+'.png', bounding_boxes_directory) post_processed_predicted_mask[post_processed_predicted_mask > 1] =1 post_processed_predicted_mask_skeletonized = skeletonize(post_processed_predicted_mask) post_processed_predicted_mask = np.zeros(post_processed_predicted_mask_skeletonized.shape) post_processed_predicted_mask[post_processed_predicted_mask_skeletonized] = 255 matching_target_file = get_matching_out_of_folder(file_name[:-len('_front.png')]+'.png', directory_of_target_fronts) target_front = cv2.imread(os.path.join(directory_of_target_fronts, matching_target_file).__str__(), cv2.IMREAD_GRAYSCALE) matching_sar_file = get_matching_out_of_folder(file_name[:-len('_front.png')]+'.png', directory_of_sar_images) sar_image = cv2.imread(os.path.join(directory_of_sar_images, matching_sar_file).__str__(), cv2.IMREAD_GRAYSCALE) elif file_name.endswith('_zone.png'): continue elif file_name.endswith('_recon.png'): continue else: post_processed_predicted_mask = cv2.imread( os.path.join(complete_postprocessed_test_directory, file_name).__str__(), cv2.IMREAD_GRAYSCALE) matching_target_file = get_matching_out_of_folder(file_name, directory_of_target_fronts) target_front = cv2.imread(os.path.join(directory_of_target_fronts, matching_target_file).__str__(),cv2.IMREAD_GRAYSCALE) matching_sar_file = get_matching_out_of_folder(file_name, directory_of_sar_images) sar_image = cv2.imread(os.path.join(directory_of_sar_images, matching_sar_file).__str__(),cv2.IMREAD_GRAYSCALE) predicted_front = np.array(post_processed_predicted_mask) ground_truth_front = np.array(target_front) kernel = np.ones((dilation, dilation), np.uint8) predicted_front = cv2.dilate(predicted_front, kernel, iterations=1) ground_truth_front = cv2.dilate(ground_truth_front, kernel, iterations=1) sar_image = np.array(sar_image) sar_image_rgb = skimage.color.gray2rgb(sar_image) sar_image_rgb = np.uint8(sar_image_rgb) sar_image_rgb[predicted_front > 0] = [0, 255, 255] # b, g, r sar_image_rgb[ground_truth_front > 0] = [255, 51, 51] correct_prediction = np.logical_and(predicted_front, ground_truth_front) sar_image_rgb[correct_prediction > 0] = [255, 0, 255] # [51, 255, 51] # [0, 153, 0] # Insert Bounding Box matching_bounding_box_file = get_matching_out_of_folder(file_name, bounding_boxes_directory) with open(os.path.join(bounding_boxes_directory, matching_bounding_box_file)) as f: coord_file_lines = f.readlines() left_upper_corner_x, left_upper_corner_y = [round(float(coord)) for coord in coord_file_lines[1].split(",")] left_lower_corner_x, left_lower_corner_y = [round(float(coord)) for coord in coord_file_lines[2].split(",")] right_lower_corner_x, right_lower_corner_y = [round(float(coord)) for coord in coord_file_lines[3].split(",")] right_upper_corner_x, right_upper_corner_y = [round(float(coord)) for coord in coord_file_lines[4].split(",")] bounding_box = np.zeros((len(sar_image_rgb), len(sar_image_rgb[0]))) if left_upper_corner_x < 0: left_upper_corner_x = 0 if left_lower_corner_x < 0: left_lower_corner_x = 0 if right_upper_corner_x > len(sar_image_rgb[0]): right_upper_corner_x = len(sar_image_rgb[0]) - 1 if right_lower_corner_x > len(sar_image_rgb[0]): right_lower_corner_x = len(sar_image_rgb[0]) - 1 if left_upper_corner_y > len(sar_image_rgb): left_upper_corner_y = len(sar_image_rgb) - 1 if left_lower_corner_y < 0: left_lower_corner_y = 0 if right_upper_corner_y > len(sar_image_rgb): right_upper_corner_y = len(sar_image_rgb) - 1 if right_lower_corner_y < 0: right_lower_corner_y = 0 bounding_box[left_upper_corner_y, left_upper_corner_x:right_upper_corner_x] = 1 bounding_box[left_lower_corner_y, left_lower_corner_x:right_lower_corner_x] = 1 bounding_box[left_lower_corner_y:left_upper_corner_y, left_upper_corner_x] = 1 bounding_box[right_lower_corner_y:right_upper_corner_y, right_lower_corner_x] = 1 bounding_box = cv2.dilate(bounding_box, kernel, iterations=1) sar_image_rgb[bounding_box > 0] = [255, 255, 0] cv2.imwrite(os.path.join(visualizations_dir, file_name), sar_image_rgb) def main(complete_test_directory, directory_of_complete_targets_zones, directory_of_complete_targets_fronts, directory_of_sar_images): # ############################################################################################### # CALCULATE SEGMENTATION METRICS (IoU & Hausdorff Distance) # ############################################################################################### complete_predicted_masks_zones = list(file for file in os.listdir(complete_test_directory) if file.endswith('_zone.png')) complete_predicted_masks_fronts = list(file for file in os.listdir(complete_test_directory) if file.endswith('_front.png')) src = Path(directory_of_sar_images).parent.parent.parent bounding_boxes_directory = os.path.join(src, "data_raw", "bounding_boxes") results = {} # only on zone if len(complete_predicted_masks_zones) > 0: results_seg = calculate_segmentation_metrics('zones', complete_predicted_masks_zones, complete_test_directory, directory_of_complete_targets_zones,) results['Zone_Segmentation'] = results_seg if len(complete_predicted_masks_fronts) >0: results_seg = calculate_segmentation_metrics('fronts', complete_predicted_masks_fronts, complete_test_directory, directory_of_complete_targets_fronts, ) results['Front_Segmentation'] = results_seg # ############################################################################################### # POST-PROCESSING # ############################################################################################### src = Path(directory_of_sar_images).parent.parent.parent print(src) if len(complete_predicted_masks_zones) > 0: post_processing('zones', complete_predicted_masks_zones, bounding_boxes_directory, complete_test_directory) # ############################################################################################### # CALCULATE FRONT DELINEATION METRIC (Mean distance error) # ############################################################################################### if len(complete_predicted_masks_zones) > 0: print("Front delineation from ZONE post processed") results_zone = front_delineation_metric('zone', complete_postprocessed_test_directory, directory_of_complete_targets_fronts, bounding_boxes_directory) results['Zone_Delineation'] = results_zone if len(complete_predicted_masks_fronts) > 0: print("Front delineation from FRONT directly") results_front = front_delineation_metric('front', complete_test_directory, directory_of_complete_targets_fronts, bounding_boxes_directory) results['Front_Delineation'] = results_front results_file = open(complete_test_directory+'/eval_results.json', "w") json.dump(results, results_file) # ############################################################################################### # MAKE VISUALIZATIONS # ############################################################################################### if len(complete_predicted_masks_zones) > 0: visualizations(complete_postprocessed_test_directory, directory_of_complete_targets_fronts, directory_of_sar_images, bounding_boxes_directory, visualizations_dir) if len(complete_predicted_masks_fronts) > 0: front_prediction_dir = complete_test_directory visualizations(front_prediction_dir, directory_of_complete_targets_fronts, directory_of_sar_images, bounding_boxes_directory, visualizations_dir) if __name__ == "__main__": print("Start Evaluation") parser = ArgumentParser(add_help=False) parser.add_argument('--predictions', help="Directory with predictions as png") parser.add_argument('--labels_fronts', help="Directory with labels as png") parser.add_argument('--labels_zones', help="Directory with labels as png") parser.add_argument('--sar_images', help="Directory with sar images") hparams = parser.parse_args() complete_test_directory = hparams.predictions complete_postprocessed_test_directory = os.path.join(complete_test_directory, "postprocessed") os.makedirs(complete_postprocessed_test_directory, exist_ok=True) visualizations_dir = os.path.join(complete_test_directory, "visualization") os.makedirs(visualizations_dir, exist_ok=True) main(hparams.predictions, hparams.labels_zones, hparams.labels_fronts, hparams.sar_images)