Spaces:
Runtime error
Runtime error
""" | |
Compute metrics of the performance of the masker using a set of ground-truth labels | |
run eval_masker.py --model "/miniscratch/_groups/ccai/checkpoints/model/" | |
""" | |
print("Imports...", end="") | |
import os | |
import os.path | |
from argparse import ArgumentParser | |
from pathlib import Path | |
import matplotlib.pyplot as plt | |
import numpy as np | |
import pandas as pd | |
from comet_ml import Experiment | |
import torch | |
import yaml | |
from skimage.color import rgba2rgb | |
from skimage.io import imread, imsave | |
from skimage.transform import resize | |
from skimage.util import img_as_ubyte | |
from torchvision.transforms import ToTensor | |
from climategan.data import encode_mask_label | |
from climategan.eval_metrics import ( | |
masker_classification_metrics, | |
get_confusion_matrix, | |
edges_coherence_std_min, | |
boxplot_metric, | |
clustermap_metric, | |
) | |
from climategan.transforms import PrepareTest | |
from climategan.trainer import Trainer | |
from climategan.utils import find_images | |
dict_metrics = { | |
"names": { | |
"tpr": "TPR, Recall, Sensitivity", | |
"tnr": "TNR, Specificity, Selectivity", | |
"fpr": "FPR", | |
"fpt": "False positives relative to image size", | |
"fnr": "FNR, Miss rate", | |
"fnt": "False negatives relative to image size", | |
"mpr": "May positive rate (MPR)", | |
"mnr": "May negative rate (MNR)", | |
"accuracy": "Accuracy (ignoring may)", | |
"error": "Error (ignoring may)", | |
"f05": "F0.05 score", | |
"precision": "Precision", | |
"edge_coherence": "Edge coherence", | |
"accuracy_must_may": "Accuracy (ignoring cannot)", | |
}, | |
"threshold": { | |
"tpr": 0.95, | |
"tnr": 0.95, | |
"fpr": 0.05, | |
"fpt": 0.01, | |
"fnr": 0.05, | |
"fnt": 0.01, | |
"accuracy": 0.95, | |
"error": 0.05, | |
"f05": 0.95, | |
"precision": 0.95, | |
"edge_coherence": 0.02, | |
"accuracy_must_may": 0.5, | |
}, | |
"key_metrics": ["f05", "error", "edge_coherence", "mnr"], | |
} | |
print("Ok.") | |
def parsed_args(): | |
"""Parse and returns command-line args | |
Returns: | |
argparse.Namespace: the parsed arguments | |
""" | |
parser = ArgumentParser() | |
parser.add_argument( | |
"--model", | |
type=str, | |
help="Path to a pre-trained model", | |
) | |
parser.add_argument( | |
"--images_dir", | |
default="/miniscratch/_groups/ccai/data/omnigan/masker-test-set/imgs", | |
type=str, | |
help="Directory containing the original test images", | |
) | |
parser.add_argument( | |
"--labels_dir", | |
default="/miniscratch/_groups/ccai/data/omnigan/masker-test-set/labels", | |
type=str, | |
help="Directory containing the labeled images", | |
) | |
parser.add_argument( | |
"--image_size", | |
default=640, | |
type=int, | |
help="The height and weight of the pre-processed images", | |
) | |
parser.add_argument( | |
"--max_files", | |
default=-1, | |
type=int, | |
help="Limit loaded samples", | |
) | |
parser.add_argument( | |
"--bin_value", default=0.5, type=float, help="Mask binarization threshold" | |
) | |
parser.add_argument( | |
"-y", | |
"--yaml", | |
default=None, | |
type=str, | |
help="load a yaml file to parametrize the evaluation", | |
) | |
parser.add_argument( | |
"-t", "--tags", nargs="*", help="Comet.ml tags", default=[], type=str | |
) | |
parser.add_argument( | |
"-p", | |
"--plot", | |
action="store_true", | |
default=False, | |
help="Plot masker images & their metrics overlays", | |
) | |
parser.add_argument( | |
"--no_paint", | |
action="store_true", | |
default=False, | |
help="Do not log painted images", | |
) | |
parser.add_argument( | |
"--write_metrics", | |
action="store_true", | |
default=False, | |
help="If True, write CSV file and maps images in model's path directory", | |
) | |
parser.add_argument( | |
"--load_metrics", | |
action="store_true", | |
default=False, | |
help="If True, load predictions and metrics instead of re-computing", | |
) | |
parser.add_argument( | |
"--prepare_torch", | |
action="store_true", | |
default=False, | |
help="If True, pre-process images as torch tensors", | |
) | |
parser.add_argument( | |
"--output_csv", | |
default=None, | |
type=str, | |
help="Filename of the output CSV with the metrics of all models", | |
) | |
return parser.parse_args() | |
def uint8(array): | |
return array.astype(np.uint8) | |
def crop_and_resize(image_path, label_path): | |
""" | |
Resizes an image so that it keeps the aspect ratio and the smallest dimensions | |
is 640, then crops this resized image in its center so that the output is 640x640 | |
without aspect ratio distortion | |
Args: | |
image_path (Path or str): Path to an image | |
label_path (Path or str): Path to the image's associated label | |
Returns: | |
tuple((np.ndarray, np.ndarray)): (new image, new label) | |
""" | |
img = imread(image_path) | |
lab = imread(label_path) | |
# if img.shape[-1] == 4: | |
# img = uint8(rgba2rgb(img) * 255) | |
# TODO: remove (debug) | |
if img.shape[:2] != lab.shape[:2]: | |
print( | |
"\nWARNING: shape mismatch: im -> ({}) {}, lab -> ({}) {}".format( | |
img.shape[:2], image_path.name, lab.shape[:2], label_path.name | |
) | |
) | |
# breakpoint() | |
# resize keeping aspect ratio: smallest dim is 640 | |
i_h, i_w = img.shape[:2] | |
if i_h < i_w: | |
i_size = (640, int(640 * i_w / i_h)) | |
else: | |
i_size = (int(640 * i_h / i_w), 640) | |
l_h, l_w = img.shape[:2] | |
if l_h < l_w: | |
l_size = (640, int(640 * l_w / l_h)) | |
else: | |
l_size = (int(640 * l_h / l_w), 640) | |
r_img = resize(img, i_size, preserve_range=True, anti_aliasing=True) | |
r_img = uint8(r_img) | |
r_lab = resize(lab, l_size, preserve_range=True, anti_aliasing=False, order=0) | |
r_lab = uint8(r_lab) | |
# crop in the center | |
H, W = r_img.shape[:2] | |
top = (H - 640) // 2 | |
left = (W - 640) // 2 | |
rc_img = r_img[top : top + 640, left : left + 640, :] | |
rc_lab = ( | |
r_lab[top : top + 640, left : left + 640, :] | |
if r_lab.ndim == 3 | |
else r_lab[top : top + 640, left : left + 640] | |
) | |
return rc_img, rc_lab | |
def plot_images( | |
output_filename, | |
img, | |
label, | |
pred, | |
metrics_dict, | |
maps_dict, | |
edge_coherence=-1, | |
pred_edge=None, | |
label_edge=None, | |
dpi=300, | |
alpha=0.5, | |
vmin=0.0, | |
vmax=1.0, | |
fontsize="xx-small", | |
cmap={ | |
"fp": "Reds", | |
"fn": "Reds", | |
"may_neg": "Oranges", | |
"may_pos": "Purples", | |
"pred": "Greens", | |
}, | |
): | |
f, axes = plt.subplots(1, 5, dpi=dpi) | |
# FPR (predicted mask on cannot flood) | |
axes[0].imshow(img) | |
fp_map_plt = axes[0].imshow( # noqa: F841 | |
maps_dict["fp"], vmin=vmin, vmax=vmax, cmap=cmap["fp"], alpha=alpha | |
) | |
axes[0].axis("off") | |
axes[0].set_title("FPR: {:.4f}".format(metrics_dict["fpr"]), fontsize=fontsize) | |
# FNR (missed mask on must flood) | |
axes[1].imshow(img) | |
fn_map_plt = axes[1].imshow( # noqa: F841 | |
maps_dict["fn"], vmin=vmin, vmax=vmax, cmap=cmap["fn"], alpha=alpha | |
) | |
axes[1].axis("off") | |
axes[1].set_title("FNR: {:.4f}".format(metrics_dict["fnr"]), fontsize=fontsize) | |
# May flood | |
axes[2].imshow(img) | |
if edge_coherence != -1: | |
title = "MNR: {:.2f} | MPR: {:.2f}\nEdge coh.: {:.4f}".format( | |
metrics_dict["mnr"], metrics_dict["mpr"], edge_coherence | |
) | |
# alpha_here = alpha / 4. | |
# pred_edge_plt = axes[2].imshow( | |
# 1.0 - pred_edge, cmap="gray", alpha=alpha_here | |
# ) | |
# label_edge_plt = axes[2].imshow( | |
# 1.0 - label_edge, cmap="gray", alpha=alpha_here | |
# ) | |
else: | |
title = "MNR: {:.2f} | MPR: {:.2f}".format(mnr, mpr) # noqa: F821 | |
# alpha_here = alpha / 2. | |
may_neg_map_plt = axes[2].imshow( # noqa: F841 | |
maps_dict["may_neg"], vmin=vmin, vmax=vmax, cmap=cmap["may_neg"], alpha=alpha | |
) | |
may_pos_map_plt = axes[2].imshow( # noqa: F841 | |
maps_dict["may_pos"], vmin=vmin, vmax=vmax, cmap=cmap["may_pos"], alpha=alpha | |
) | |
axes[2].set_title(title, fontsize=fontsize) | |
axes[2].axis("off") | |
# Prediction | |
axes[3].imshow(img) | |
pred_mask = axes[3].imshow( # noqa: F841 | |
pred, vmin=vmin, vmax=vmax, cmap=cmap["pred"], alpha=alpha | |
) | |
axes[3].set_title("Predicted mask", fontsize=fontsize) | |
axes[3].axis("off") | |
# Labels | |
axes[4].imshow(img) | |
label_mask = axes[4].imshow(label, alpha=alpha) # noqa: F841 | |
axes[4].set_title("Labels", fontsize=fontsize) | |
axes[4].axis("off") | |
f.savefig( | |
output_filename, | |
dpi=f.dpi, | |
bbox_inches="tight", | |
facecolor="white", | |
transparent=False, | |
) | |
plt.close(f) | |
def load_ground(ground_output_path, ref_image_path): | |
gop = Path(ground_output_path) | |
rip = Path(ref_image_path) | |
ground_paths = list((gop / "eval-metrics" / "pred").glob(f"{rip.stem}.jpg")) + list( | |
(gop / "eval-metrics" / "pred").glob(f"{rip.stem}.png") | |
) | |
if len(ground_paths) == 0: | |
raise ValueError( | |
f"Could not find a ground match in {str(gop)} for image {str(rip)}" | |
) | |
elif len(ground_paths) > 1: | |
raise ValueError( | |
f"Found more than 1 ground match in {str(gop)} for image {str(rip)}:" | |
+ f" {list(map(str, ground_paths))}" | |
) | |
ground_path = ground_paths[0] | |
_, ground = crop_and_resize(rip, ground_path) | |
if ground.ndim == 3: | |
ground = ground[:, :, 0] | |
ground = (ground > 0).astype(np.float32) | |
return torch.from_numpy(ground).unsqueeze(0).unsqueeze(0).cuda() | |
def get_inferences( | |
image_arrays, model_path, image_paths, paint=False, bin_value=0.5, verbose=0 | |
): | |
""" | |
Obtains the mask predictions of a model for a set of images | |
Parameters | |
---------- | |
image_arrays : array-like | |
A list of (1, CH, H, W) images | |
image_paths: list(Path) | |
A list of paths for images, in the same order as image_arrays | |
model_path : str | |
The path to a pre-trained model | |
Returns | |
------- | |
masks : list | |
A list of (H, W) predicted masks | |
""" | |
device = torch.device("cuda:0") | |
torch.set_grad_enabled(False) | |
to_tensor = ToTensor() | |
is_ground = "ground" in Path(model_path).name | |
is_instagan = "instagan" in Path(model_path).name | |
if is_ground or is_instagan: | |
# we just care about he painter here | |
ground_path = model_path | |
model_path = ( | |
"/miniscratch/_groups/ccai/experiments/runs/ablation-v1/out--38858350" | |
) | |
xs = [to_tensor(array).unsqueeze(0) for array in image_arrays] | |
xs = [x.to(torch.float32).to(device) for x in xs] | |
xs = [(x - 0.5) * 2 for x in xs] | |
trainer = Trainer.resume_from_path( | |
model_path, inference=True, new_exp=None, device=device | |
) | |
masks = [] | |
painted = [] | |
for idx, x in enumerate(xs): | |
if verbose > 0: | |
print(idx, "/", len(xs), end="\r") | |
if not is_ground and not is_instagan: | |
m = trainer.G.mask(x=x) | |
else: | |
m = load_ground(ground_path, image_paths[idx]) | |
masks.append(m.squeeze().cpu()) | |
if paint: | |
p = trainer.G.paint(m > bin_value, x) | |
painted.append(p.squeeze().cpu()) | |
return masks, painted | |
if __name__ == "__main__": | |
# ----------------------------- | |
# ----- Parse arguments ----- | |
# ----------------------------- | |
args = parsed_args() | |
print("Args:\n" + "\n".join([f" {k:20}: {v}" for k, v in vars(args).items()])) | |
# Determine output dir | |
try: | |
tmp_dir = Path(os.environ["SLURM_TMPDIR"]) | |
except Exception as e: | |
print(e) | |
tmp_dir = Path(input("Enter tmp output directory: ")).resolve() | |
plot_dir = tmp_dir / "plots" | |
plot_dir.mkdir(parents=True, exist_ok=True) | |
# Build paths to data | |
imgs_paths = sorted( | |
find_images(args.images_dir, recursive=False), key=lambda x: x.name | |
) | |
labels_paths = sorted( | |
find_images(args.labels_dir, recursive=False), | |
key=lambda x: x.name.replace("_labeled.", "."), | |
) | |
if args.max_files > 0: | |
imgs_paths = imgs_paths[: args.max_files] | |
labels_paths = labels_paths[: args.max_files] | |
print(f"Loading {len(imgs_paths)} images and labels...") | |
# Pre-process images: resize + crop | |
# TODO: ? make cropping more flexible, not only central | |
if not args.prepare_torch: | |
ims_labs = [crop_and_resize(i, l) for i, l in zip(imgs_paths, labels_paths)] | |
imgs = [d[0] for d in ims_labs] | |
labels = [d[1] for d in ims_labs] | |
else: | |
prepare = PrepareTest() | |
imgs = prepare(imgs_paths, normalize=False, rescale=False) | |
labels = prepare(labels_paths, normalize=False, rescale=False) | |
imgs = [i.squeeze(0).permute(1, 2, 0).numpy().astype(np.uint8) for i in imgs] | |
labels = [ | |
lab.squeeze(0).permute(1, 2, 0).numpy().astype(np.uint8) for lab in labels | |
] | |
imgs = [rgba2rgb(img) if img.shape[-1] == 4 else img for img in imgs] | |
print(" Done.") | |
# Encode labels | |
print("Encode labels...", end="", flush=True) | |
# HW label | |
labels = [np.squeeze(encode_mask_label(label, "flood")) for label in labels] | |
print("Done.") | |
if args.yaml: | |
y_path = Path(args.yaml) | |
assert y_path.exists() | |
assert y_path.suffix in {".yaml", ".yml"} | |
with y_path.open("r") as f: | |
data = yaml.safe_load(f) | |
assert "models" in data | |
evaluations = [m for m in data["models"]] | |
else: | |
evaluations = [args.model] | |
for e, eval_path in enumerate(evaluations): | |
print("\n>>>>> Evaluation", e, ":", eval_path) | |
print("=" * 50) | |
print("=" * 50) | |
model_metrics_path = Path(eval_path) / "eval-metrics" | |
model_metrics_path.mkdir(exist_ok=True) | |
if args.load_metrics: | |
f_csv = model_metrics_path / "eval_masker.csv" | |
pred_out = model_metrics_path / "pred" | |
if f_csv.exists() and pred_out.exists(): | |
print("Skipping model because pre-computed metrics exist") | |
continue | |
# Initialize New Comet Experiment | |
exp = Experiment( | |
project_name="climategan-masker-metrics", display_summary_level=0 | |
) | |
# Obtain mask predictions | |
# TODO: remove (debug) | |
print("Obtain mask predictions", end="", flush=True) | |
preds, painted = get_inferences( | |
imgs, | |
eval_path, | |
imgs_paths, | |
paint=not args.no_paint, | |
bin_value=args.bin_value, | |
verbose=1, | |
) | |
preds = [pred.numpy() for pred in preds] | |
print(" Done.") | |
if args.bin_value > 0: | |
preds = [pred > args.bin_value for pred in preds] | |
# Compute metrics | |
df = pd.DataFrame( | |
columns=[ | |
"tpr", | |
"tpt", | |
"tnr", | |
"tnt", | |
"fpr", | |
"fpt", | |
"fnr", | |
"fnt", | |
"mnr", | |
"mpr", | |
"accuracy", | |
"error", | |
"precision", | |
"f05", | |
"accuracy_must_may", | |
"edge_coherence", | |
"filename", | |
] | |
) | |
print("Compute metrics and plot images") | |
for idx, (img, label, pred) in enumerate(zip(*(imgs, labels, preds))): | |
print(idx, "/", len(imgs), end="\r") | |
# Basic classification metrics | |
metrics_dict, maps_dict = masker_classification_metrics( | |
pred, label, labels_dict={"cannot": 0, "must": 1, "may": 2} | |
) | |
# Edges coherence | |
edge_coherence, pred_edge, label_edge = edges_coherence_std_min(pred, label) | |
series_dict = { | |
"tpr": metrics_dict["tpr"], | |
"tpt": metrics_dict["tpt"], | |
"tnr": metrics_dict["tnr"], | |
"tnt": metrics_dict["tnt"], | |
"fpr": metrics_dict["fpr"], | |
"fpt": metrics_dict["fpt"], | |
"fnr": metrics_dict["fnr"], | |
"fnt": metrics_dict["fnt"], | |
"mnr": metrics_dict["mnr"], | |
"mpr": metrics_dict["mpr"], | |
"accuracy": metrics_dict["accuracy"], | |
"error": metrics_dict["error"], | |
"precision": metrics_dict["precision"], | |
"f05": metrics_dict["f05"], | |
"accuracy_must_may": metrics_dict["accuracy_must_may"], | |
"edge_coherence": edge_coherence, | |
"filename": str(imgs_paths[idx].name), | |
} | |
df.loc[idx] = pd.Series(series_dict) | |
for k, v in series_dict.items(): | |
if k == "filename": | |
continue | |
exp.log_metric(f"img_{k}", v, step=idx) | |
# Confusion matrix | |
confmat, _ = get_confusion_matrix( | |
metrics_dict["tpr"], | |
metrics_dict["tnr"], | |
metrics_dict["fpr"], | |
metrics_dict["fnr"], | |
metrics_dict["mnr"], | |
metrics_dict["mpr"], | |
) | |
confmat = np.around(confmat, decimals=3) | |
exp.log_confusion_matrix( | |
file_name=imgs_paths[idx].name + ".json", | |
title=imgs_paths[idx].name, | |
matrix=confmat, | |
labels=["Cannot", "Must", "May"], | |
row_label="Predicted", | |
column_label="Ground truth", | |
) | |
if args.plot: | |
# Plot prediction images | |
fig_filename = plot_dir / imgs_paths[idx].name | |
plot_images( | |
fig_filename, | |
img, | |
label, | |
pred, | |
metrics_dict, | |
maps_dict, | |
edge_coherence, | |
pred_edge, | |
label_edge, | |
) | |
exp.log_image(fig_filename) | |
if not args.no_paint: | |
masked = img * (1 - pred[..., None]) | |
flooded = img_as_ubyte( | |
(painted[idx].permute(1, 2, 0).cpu().numpy() + 1) / 2 | |
) | |
combined = np.concatenate([img, masked, flooded], 1) | |
exp.log_image(combined, imgs_paths[idx].name) | |
if args.write_metrics: | |
pred_out = model_metrics_path / "pred" | |
pred_out.mkdir(exist_ok=True) | |
imsave( | |
pred_out / f"{imgs_paths[idx].stem}_pred.png", | |
pred.astype(np.uint8), | |
) | |
for k, v in maps_dict.items(): | |
metric_out = model_metrics_path / k | |
metric_out.mkdir(exist_ok=True) | |
imsave( | |
metric_out / f"{imgs_paths[idx].stem}_{k}.png", | |
v.astype(np.uint8), | |
) | |
# -------------------------------- | |
# ----- END OF IMAGES LOOP ----- | |
# -------------------------------- | |
if args.write_metrics: | |
print(f"Writing metrics in {str(model_metrics_path)}") | |
f_csv = model_metrics_path / "eval_masker.csv" | |
df.to_csv(f_csv, index_label="idx") | |
print(" Done.") | |
# Summary statistics | |
means = df.mean(axis=0) | |
confmat_mean, confmat_std = get_confusion_matrix( | |
df.tpr, df.tnr, df.fpr, df.fnr, df.mpr, df.mnr | |
) | |
confmat_mean = np.around(confmat_mean, decimals=3) | |
confmat_std = np.around(confmat_std, decimals=3) | |
# Log to comet | |
exp.log_confusion_matrix( | |
file_name="confusion_matrix_mean.json", | |
title="confusion_matrix_mean.json", | |
matrix=confmat_mean, | |
labels=["Cannot", "Must", "May"], | |
row_label="Predicted", | |
column_label="Ground truth", | |
) | |
exp.log_confusion_matrix( | |
file_name="confusion_matrix_std.json", | |
title="confusion_matrix_std.json", | |
matrix=confmat_std, | |
labels=["Cannot", "Must", "May"], | |
row_label="Predicted", | |
column_label="Ground truth", | |
) | |
exp.log_metrics(dict(means)) | |
exp.log_table("metrics.csv", df) | |
exp.log_html(df.to_html(col_space="80px")) | |
exp.log_parameters(vars(args)) | |
exp.log_parameter("eval_path", str(eval_path)) | |
exp.add_tag("eval_masker") | |
if args.tags: | |
exp.add_tags(args.tags) | |
exp.log_parameter("model_id", Path(eval_path).name) | |
# Close comet | |
exp.end() | |
# -------------------------------- | |
# ----- END OF MODElS LOOP ----- | |
# -------------------------------- | |
# Compare models | |
if (args.load_metrics or args.write_metrics) and len(evaluations) > 1: | |
print( | |
"Plots for comparing the input models will be created and logged to comet" | |
) | |
# Initialize New Comet Experiment | |
exp = Experiment( | |
project_name="climategan-masker-metrics", display_summary_level=0 | |
) | |
if args.tags: | |
exp.add_tags(args.tags) | |
# Build DataFrame with all models | |
print("Building pandas DataFrame...") | |
models_df = {} | |
for (m, model_path) in enumerate(evaluations): | |
model_path = Path(model_path) | |
with open(model_path / "opts.yaml", "r") as f: | |
opt = yaml.safe_load(f) | |
model_feats = ", ".join( | |
[ | |
t | |
for t in sorted(opt["comet"]["tags"]) | |
if "branch" not in t and "ablation" not in t and "trash" not in t | |
] | |
) | |
model_id = f"{model_path.parent.name[-2:]}/{model_path.name}" | |
df_m = pd.read_csv( | |
model_path / "eval-metrics" / "eval_masker.csv", index_col=False | |
) | |
df_m["model"] = [model_id] * len(df_m) | |
df_m["model_idx"] = [m] * len(df_m) | |
df_m["model_feats"] = [model_feats] * len(df_m) | |
models_df.update({model_id: df_m}) | |
df = pd.concat(list(models_df.values()), ignore_index=True) | |
df["model_img_idx"] = df.model.astype(str) + "-" + df.idx.astype(str) | |
df.rename(columns={"idx": "img_idx"}, inplace=True) | |
dict_models_labels = { | |
k: f"{v['model_idx'][0]}: {v['model_feats'][0]}" | |
for k, v in models_df.items() | |
} | |
print("Done") | |
if args.output_csv: | |
print(f"Writing DataFrame to {args.output_csv}") | |
df.to_csv(args.output_csv, index_label="model_img_idx") | |
# Determine images with low metrics in any model | |
print("Constructing filter based on metrics thresholds...") | |
idx_not_good_in_any = [] | |
for idx in df.img_idx.unique(): | |
df_th = df.loc[ | |
( | |
# TODO: rethink thresholds | |
(df.tpr <= dict_metrics["threshold"]["tpr"]) | |
| (df.fpr >= dict_metrics["threshold"]["fpr"]) | |
| (df.edge_coherence >= dict_metrics["threshold"]["edge_coherence"]) | |
) | |
& ((df.img_idx == idx) & (df.model.isin(df.model.unique()))) | |
] | |
if len(df_th) > 0: | |
idx_not_good_in_any.append(idx) | |
filters = {"all": df.img_idx.unique(), "not_good_in_any": idx_not_good_in_any} | |
print("Done") | |
# Boxplots of metrics | |
print("Plotting boxplots of metrics...") | |
for k, f in filters.items(): | |
print(f"\tDistribution of [{k}] images...") | |
for metric in dict_metrics["names"].keys(): | |
fig_filename = plot_dir / f"boxplot_{metric}_{k}.png" | |
if metric in ["mnr", "mpr", "accuracy_must_may"]: | |
boxplot_metric( | |
fig_filename, | |
df.loc[df.img_idx.isin(f)], | |
metric=metric, | |
dict_metrics=dict_metrics["names"], | |
do_stripplot=True, | |
dict_models=dict_models_labels, | |
order=list(df.model.unique()), | |
) | |
else: | |
boxplot_metric( | |
fig_filename, | |
df.loc[df.img_idx.isin(f)], | |
metric=metric, | |
dict_metrics=dict_metrics["names"], | |
dict_models=dict_models_labels, | |
fliersize=1.0, | |
order=list(df.model.unique()), | |
) | |
exp.log_image(fig_filename) | |
print("Done") | |
# Cluster Maps | |
print("Plotting clustermaps...") | |
for k, f in filters.items(): | |
print(f"\tDistribution of [{k}] images...") | |
for metric in dict_metrics["names"].keys(): | |
fig_filename = plot_dir / f"clustermap_{metric}_{k}.png" | |
df_mf = df.loc[df.img_idx.isin(f)].pivot("img_idx", "model", metric) | |
clustermap_metric( | |
output_filename=fig_filename, | |
df=df_mf, | |
metric=metric, | |
dict_metrics=dict_metrics["names"], | |
method="average", | |
cluster_metric="euclidean", | |
dict_models=dict_models_labels, | |
row_cluster=False, | |
) | |
exp.log_image(fig_filename) | |
print("Done") | |
# Close comet | |
exp.end() | |