ho11laqe's picture
init
ecf08bc
raw history blame
No virus
12.1 kB
# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections import OrderedDict
from nnunet.evaluation.add_mean_dice_to_json import foreground_mean
from batchgenerators.utilities.file_and_folder_operations import *
from nnunet.paths import network_training_output_dir
import numpy as np
def summarize(tasks, models=('2d', '3d_lowres', '3d_fullres', '3d_cascade_fullres'),
output_dir=join(network_training_output_dir, "summary_jsons"), folds=(0, 1, 2, 3, 4)):
maybe_mkdir_p(output_dir)
if len(tasks) == 1 and tasks[0] == "all":
tasks = list(range(999))
else:
tasks = [int(i) for i in tasks]
for model in models:
for t in tasks:
t = int(t)
if not isdir(join(network_training_output_dir, model)):
continue
task_name = subfolders(join(network_training_output_dir, model), prefix="Task%03.0d" % t, join=False)
if len(task_name) != 1:
print("did not find unique output folder for network %s and task %s" % (model, t))
continue
task_name = task_name[0]
out_dir_task = join(network_training_output_dir, model, task_name)
model_trainers = subdirs(out_dir_task, join=False)
for trainer in model_trainers:
if trainer.startswith("fold"):
continue
out_dir = join(out_dir_task, trainer)
validation_folders = []
for fld in folds:
d = join(out_dir, "fold%d"%fld)
if not isdir(d):
d = join(out_dir, "fold_%d"%fld)
if not isdir(d):
break
validation_folders += subfolders(d, prefix="validation", join=False)
for v in validation_folders:
ok = True
metrics = OrderedDict()
for fld in folds:
d = join(out_dir, "fold%d"%fld)
if not isdir(d):
d = join(out_dir, "fold_%d"%fld)
if not isdir(d):
ok = False
break
validation_folder = join(d, v)
if not isfile(join(validation_folder, "summary.json")):
print("summary.json missing for net %s task %s fold %d" % (model, task_name, fld))
ok = False
break
metrics_tmp = load_json(join(validation_folder, "summary.json"))["results"]["mean"]
for l in metrics_tmp.keys():
if metrics.get(l) is None:
metrics[l] = OrderedDict()
for m in metrics_tmp[l].keys():
if metrics[l].get(m) is None:
metrics[l][m] = []
metrics[l][m].append(metrics_tmp[l][m])
if ok:
for l in metrics.keys():
for m in metrics[l].keys():
assert len(metrics[l][m]) == len(folds)
metrics[l][m] = np.mean(metrics[l][m])
json_out = OrderedDict()
json_out["results"] = OrderedDict()
json_out["results"]["mean"] = metrics
json_out["task"] = task_name
json_out["description"] = model + " " + task_name + " all folds summary"
json_out["name"] = model + " " + task_name + " all folds summary"
json_out["experiment_name"] = model
save_json(json_out, join(out_dir, "summary_allFolds__%s.json" % v))
save_json(json_out, join(output_dir, "%s__%s__%s__%s.json" % (task_name, model, trainer, v)))
foreground_mean(join(out_dir, "summary_allFolds__%s.json" % v))
foreground_mean(join(output_dir, "%s__%s__%s__%s.json" % (task_name, model, trainer, v)))
def summarize2(task_ids, models=('2d', '3d_lowres', '3d_fullres', '3d_cascade_fullres'),
output_dir=join(network_training_output_dir, "summary_jsons"), folds=(0, 1, 2, 3, 4)):
maybe_mkdir_p(output_dir)
if len(task_ids) == 1 and task_ids[0] == "all":
task_ids = list(range(999))
else:
task_ids = [int(i) for i in task_ids]
for model in models:
for t in task_ids:
if not isdir(join(network_training_output_dir, model)):
continue
task_name = subfolders(join(network_training_output_dir, model), prefix="Task%03.0d" % t, join=False)
if len(task_name) != 1:
print("did not find unique output folder for network %s and task %s" % (model, t))
continue
task_name = task_name[0]
out_dir_task = join(network_training_output_dir, model, task_name)
model_trainers = subdirs(out_dir_task, join=False)
for trainer in model_trainers:
if trainer.startswith("fold"):
continue
out_dir = join(out_dir_task, trainer)
validation_folders = []
for fld in folds:
fold_output_dir = join(out_dir, "fold_%d"%fld)
if not isdir(fold_output_dir):
continue
validation_folders += subfolders(fold_output_dir, prefix="validation", join=False)
validation_folders = np.unique(validation_folders)
for v in validation_folders:
ok = True
metrics = OrderedDict()
metrics['mean'] = OrderedDict()
metrics['median'] = OrderedDict()
metrics['all'] = OrderedDict()
for fld in folds:
fold_output_dir = join(out_dir, "fold_%d"%fld)
if not isdir(fold_output_dir):
print("fold missing", model, task_name, trainer, fld)
ok = False
break
validation_folder = join(fold_output_dir, v)
if not isdir(validation_folder):
print("validation folder missing", model, task_name, trainer, fld, v)
ok = False
break
if not isfile(join(validation_folder, "summary.json")):
print("summary.json missing", model, task_name, trainer, fld, v)
ok = False
break
all_metrics = load_json(join(validation_folder, "summary.json"))["results"]
# we now need to get the mean and median metrics. We use the mean metrics just to get the
# names of computed metics, we ignore the precomputed mean and do it ourselfes again
mean_metrics = all_metrics["mean"]
all_labels = [i for i in list(mean_metrics.keys()) if i != "mean"]
if len(all_labels) == 0: print(v, fld); break
all_metrics_names = list(mean_metrics[all_labels[0]].keys())
for l in all_labels:
# initialize the data structure, no values are copied yet
for k in ['mean', 'median', 'all']:
if metrics[k].get(l) is None:
metrics[k][l] = OrderedDict()
for m in all_metrics_names:
if metrics['all'][l].get(m) is None:
metrics['all'][l][m] = []
for entry in all_metrics['all']:
for l in all_labels:
for m in all_metrics_names:
metrics['all'][l][m].append(entry[l][m])
# now compute mean and median
for l in metrics['all'].keys():
for m in metrics['all'][l].keys():
metrics['mean'][l][m] = np.nanmean(metrics['all'][l][m])
metrics['median'][l][m] = np.nanmedian(metrics['all'][l][m])
if ok:
fold_string = ""
for f in folds:
fold_string += str(f)
json_out = OrderedDict()
json_out["results"] = OrderedDict()
json_out["results"]["mean"] = metrics['mean']
json_out["results"]["median"] = metrics['median']
json_out["task"] = task_name
json_out["description"] = model + " " + task_name + "summary folds" + str(folds)
json_out["name"] = model + " " + task_name + "summary folds" + str(folds)
json_out["experiment_name"] = model
save_json(json_out, join(output_dir, "%s__%s__%s__%s__%s.json" % (task_name, model, trainer, v, fold_string)))
foreground_mean2(join(output_dir, "%s__%s__%s__%s__%s.json" % (task_name, model, trainer, v, fold_string)))
def foreground_mean2(filename):
with open(filename, 'r') as f:
res = json.load(f)
class_ids = np.array([int(i) for i in res['results']['mean'].keys() if (i != 'mean') and i != '0'])
metric_names = res['results']['mean']['1'].keys()
res['results']['mean']["mean"] = OrderedDict()
res['results']['median']["mean"] = OrderedDict()
for m in metric_names:
foreground_values = [res['results']['mean'][str(i)][m] for i in class_ids]
res['results']['mean']["mean"][m] = np.nanmean(foreground_values)
foreground_values = [res['results']['median'][str(i)][m] for i in class_ids]
res['results']['median']["mean"][m] = np.nanmean(foreground_values)
with open(filename, 'w') as f:
json.dump(res, f, indent=4, sort_keys=True)
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(usage="This is intended to identify the best model based on the five fold "
"cross-validation. Running this script requires alle models to have been run "
"already. This script will summarize the results of the five folds of all "
"models in one json each for easy interpretability")
parser.add_argument("-t", '--task_ids', nargs="+", required=True, help="task id. can be 'all'")
parser.add_argument("-f", '--folds', nargs="+", required=False, type=int, default=[0, 1, 2, 3, 4])
parser.add_argument("-m", '--models', nargs="+", required=False, default=['2d', '3d_lowres', '3d_fullres', '3d_cascade_fullres'])
args = parser.parse_args()
tasks = args.task_ids
models = args.models
folds = args.folds
summarize2(tasks, models, folds=folds, output_dir=join(network_training_output_dir, "summary_jsons_new"))