climategan/logger.py

from pathlib import Path

import numpy as np
import torch
import torchvision.utils as vutils
from addict import Dict
from PIL import Image
from torch.nn.functional import interpolate, sigmoid

from climategan.data import decode_segmap_merged_labels
from climategan.tutils import (
    all_texts_to_tensors,
    decode_bucketed_depth,
    normalize_tensor,
    write_architecture,
)
from climategan.utils import flatten_opts


class Logger:
    def __init__(self, trainer):
        self.losses = Dict()
        self.time = Dict()
        self.trainer = trainer
        self.global_step = 0
        self.epoch = 0

    def log_comet_images(self, mode, domain, minimal=False, all_only=False):
        trainer = self.trainer
        save_images = {}
        all_images = []
        n_all_ims = None
        all_legends = ["Input"]
        task_legends = {}

        if domain not in trainer.display_images[mode]:
            return

        # --------------------
        # -----  Masker  -----
        # --------------------
        n_ims = len(trainer.display_images[mode][domain])
        print(" " * 60, end="\r")
        if domain != "rf":
            for j, display_dict in enumerate(trainer.display_images[mode][domain]):

                print(f"Inferring sample {mode} {domain} {j+1}/{n_ims}", end="\r")

                x = display_dict["data"]["x"].unsqueeze(0).to(trainer.device)
                z = trainer.G.encode(x)

                s_pred = decoded_s_pred = d_pred = z_depth = None
                for k, task in enumerate(["d", "s", "m"]):

                    if (
                        task not in display_dict["data"]
                        or task not in trainer.opts.tasks
                    ):
                        continue

                    task_legend = ["Input"]
                    target = display_dict["data"][task]
                    target = target.unsqueeze(0).to(trainer.device)
                    task_saves = []

                    if task not in save_images:
                        save_images[task] = []

                    prediction = None
                    if task == "m":
                        cond = None
                        if s_pred is not None and d_pred is not None:
                            cond = trainer.G.make_m_cond(d_pred, s_pred, x)

                        prediction = trainer.G.decoders[task](z, cond, z_depth)
                    elif task == "d":
                        prediction, z_depth = trainer.G.decoders[task](z)
                    elif task == "s":
                        prediction = trainer.G.decoders[task](z, z_depth)

                    if task == "s":
                        # Log fire
                        wildfire_tens = trainer.compute_fire(x, prediction)
                        task_saves.append(wildfire_tens)
                        task_legend.append("Wildfire")
                        # Log seg output
                        s_pred = prediction.clone()
                        target = (
                            decode_segmap_merged_labels(target, domain, True)
                            .float()
                            .to(trainer.device)
                        )
                        prediction = (
                            decode_segmap_merged_labels(prediction, domain, False)
                            .float()
                            .to(trainer.device)
                        )
                        decoded_s_pred = prediction
                        task_saves.append(target)
                        task_legend.append("Target Segmentation")

                    elif task == "m":
                        prediction = sigmoid(prediction).repeat(1, 3, 1, 1)
                        task_saves.append(x * (1.0 - prediction))
                        if not minimal:
                            task_saves.append(
                                x * (1.0 - (prediction > 0.1).to(torch.int))
                            )
                            task_saves.append(
                                x * (1.0 - (prediction > 0.5).to(torch.int))
                            )

                        task_saves.append(x * (1.0 - target.repeat(1, 3, 1, 1)))
                        task_legend.append("Masked input")

                        if not minimal:
                            task_legend.append("Masked input (>0.1)")
                            task_legend.append("Masked input (>0.5)")

                        task_legend.append("Masked input (target)")
                        # dummy pixels to fool scaling and preserve mask range
                        prediction[:, :, 0, 0] = 1.0
                        prediction[:, :, -1, -1] = 0.0

                    elif task == "d":
                        # prediction is a log depth tensor
                        d_pred = prediction
                        target = normalize_tensor(target) * 255
                        if prediction.shape[1] > 1:
                            prediction = decode_bucketed_depth(
                                prediction, self.trainer.opts
                            )
                        smogged = self.trainer.compute_smog(
                            x, d=prediction, s=decoded_s_pred, use_sky_seg=False
                        )
                        prediction = normalize_tensor(prediction)
                        prediction = prediction.repeat(1, 3, 1, 1)
                        task_saves.append(smogged)
                        task_legend.append("Smogged")
                        task_saves.append(target.repeat(1, 3, 1, 1))
                        task_legend.append("Depth target")

                    task_saves.append(prediction)
                    task_legend.append(f"Predicted {task}")

                    save_images[task].append(x.cpu().detach())
                    if k == 0:
                        all_images.append(save_images[task][-1])

                    task_legends[task] = task_legend
                    if j == 0:
                        all_legends += task_legend[1:]

                    for im in task_saves:
                        save_images[task].append(im.cpu().detach())
                        all_images.append(save_images[task][-1])

                if j == 0:
                    n_all_ims = len(all_images)

            if not all_only:
                for task in save_images.keys():
                    # Write images:
                    self.upload_images(
                        image_outputs=save_images[task],
                        mode=mode,
                        domain=domain,
                        task=task,
                        im_per_row=trainer.opts.comet.im_per_row.get(task, 4),
                        rows_per_log=trainer.opts.comet.get("rows_per_log", 5),
                        legends=task_legends[task],
                    )

            if len(save_images) > 1:
                self.upload_images(
                    image_outputs=all_images,
                    mode=mode,
                    domain=domain,
                    task="all",
                    im_per_row=n_all_ims,
                    rows_per_log=trainer.opts.comet.get("rows_per_log", 5),
                    legends=all_legends,
                )
        # ---------------------
        # -----  Painter  -----
        # ---------------------
        else:
            # in the rf domain display_size may be different from fid.n_images
            limit = trainer.opts.comet.display_size
            image_outputs = []
            legends = []
            for im_set in trainer.display_images[mode][domain][:limit]:
                x = im_set["data"]["x"].unsqueeze(0).to(trainer.device)
                m = im_set["data"]["m"].unsqueeze(0).to(trainer.device)

                prediction = trainer.G.paint(m, x)

                image_outputs.append(x * (1.0 - m))
                image_outputs.append(prediction)
                image_outputs.append(x)
                image_outputs.append(prediction * m)
                if not legends:
                    legends.append("Masked Input")
                    legends.append("Painted Input")
                    legends.append("Input")
                    legends.append("Isolated Water")
            # Write images
            self.upload_images(
                image_outputs=image_outputs,
                mode=mode,
                domain=domain,
                task="painter",
                im_per_row=trainer.opts.comet.im_per_row.get("p", 4),
                rows_per_log=trainer.opts.comet.get("rows_per_log", 5),
                legends=legends,
            )

        return 0

    def log_losses(self, model_to_update="G", mode="train"):
        """Logs metrics on comet.ml

        Args:
            model_to_update (str, optional): One of "G", "D". Defaults to "G".
        """
        trainer = self.trainer
        loss_names = {"G": "gen", "D": "disc"}

        if trainer.opts.train.log_level < 1:
            return

        if trainer.exp is None:
            return

        assert model_to_update in {
            "G",
            "D",
        }, "unknown model to log losses {}".format(model_to_update)

        loss_to_update = self.losses[loss_names[model_to_update]]

        losses = loss_to_update.copy()

        if trainer.opts.train.log_level == 1:
            # Only log aggregated losses: delete other keys in losses
            for k in loss_to_update:
                if k not in {"masker", "total_loss", "painter"}:
                    del losses[k]
        # convert losses into a single-level dictionnary

        losses = flatten_opts(losses)
        trainer.exp.log_metrics(
            losses, prefix=f"{model_to_update}_{mode}", step=self.global_step
        )

    def log_learning_rates(self):
        if self.trainer.exp is None:
            return
        lrs = {}
        trainer = self.trainer
        if trainer.g_scheduler is not None:
            for name, lr in zip(
                trainer.lr_names["G"], trainer.g_scheduler.get_last_lr()
            ):
                lrs[f"lr_G_{name}"] = lr
        if trainer.d_scheduler is not None:
            for name, lr in zip(
                trainer.lr_names["D"], trainer.d_scheduler.get_last_lr()
            ):
                lrs[f"lr_D_{name}"] = lr

        trainer.exp.log_metrics(lrs, step=self.global_step)

    def log_step_time(self, time):
        """Logs step-time on comet.ml

        Args:
            step_time (float): step-time in seconds
        """
        if self.trainer.exp:
            self.trainer.exp.log_metric(
                "step-time", time - self.time.step_start, step=self.global_step
            )

    def log_epoch_time(self, time):
        """Logs step-time on comet.ml

        Args:
            step_time (float): step-time in seconds
        """
        if self.trainer.exp:
            self.trainer.exp.log_metric(
                "epoch-time", time - self.time.epoch_start, step=self.global_step
            )

    def log_comet_combined_images(self, mode, domain):

        trainer = self.trainer
        image_outputs = []
        legends = []
        im_per_row = 0
        for i, im_set in enumerate(trainer.display_images[mode][domain]):
            x = im_set["data"]["x"].unsqueeze(0).to(trainer.device)
            # m = im_set["data"]["m"].unsqueeze(0).to(trainer.device)

            m = trainer.G.mask(x=x)
            m_bin = (m > 0.5).to(m.dtype)
            prediction = trainer.G.paint(m, x)
            prediction_bin = trainer.G.paint(m_bin, x)

            image_outputs.append(x)
            legends.append("Input")
            image_outputs.append(x * (1.0 - m))
            legends.append("Soft Masked Input")
            image_outputs.append(prediction)
            legends.append("Painted")
            image_outputs.append(prediction * m)
            legends.append("Soft Masked Painted")
            image_outputs.append(x * (1.0 - m_bin))
            legends.append("Binary (0.5) Masked Input")
            image_outputs.append(prediction_bin)
            legends.append("Binary (0.5) Painted")
            image_outputs.append(prediction_bin * m_bin)
            legends.append("Binary (0.5) Masked Painted")

            if i == 0:
                im_per_row = len(image_outputs)
        # Upload images
        self.upload_images(
            image_outputs=image_outputs,
            mode=mode,
            domain=domain,
            task="combined",
            im_per_row=im_per_row or 7,
            rows_per_log=trainer.opts.comet.get("rows_per_log", 5),
            legends=legends,
        )

        return 0

    def upload_images(
        self,
        image_outputs,
        mode,
        domain,
        task,
        im_per_row=3,
        rows_per_log=5,
        legends=[],
    ):
        """
        Save output image

        Args:
            image_outputs (list(torch.Tensor)): all the images to log
            mode (str): train or val
            domain (str): current domain
            task (str): current task
            im_per_row (int, optional): umber of images to be displayed per row.
                Typically, for a given task: 3 because [input prediction, target].
                Defaults to 3.
            rows_per_log (int, optional): Number of rows (=samples) per uploaded image.
                Defaults to 5.
            comet_exp (comet_ml.Experiment, optional): experiment to use.
                Defaults to None.
        """
        trainer = self.trainer
        if trainer.exp is None:
            return
        curr_iter = self.global_step
        nb_per_log = im_per_row * rows_per_log
        n_logs = len(image_outputs) // nb_per_log + 1

        header = None
        if len(legends) == im_per_row and all(isinstance(t, str) for t in legends):
            header_width = max(im.shape[-1] for im in image_outputs)
            headers = all_texts_to_tensors(legends, width=header_width)
            header = torch.cat(headers, dim=-1)

        for logidx in range(n_logs):
            print(" " * 100, end="\r", flush=True)
            print(
                "Uploading images for {} {} {} {}/{}".format(
                    mode, domain, task, logidx + 1, n_logs
                ),
                end="...",
                flush=True,
            )
            ims = image_outputs[logidx * nb_per_log : (logidx + 1) * nb_per_log]
            if not ims:
                continue

            ims = self.upsample(ims)
            ims = torch.stack([im.squeeze() for im in ims]).squeeze()
            image_grid = vutils.make_grid(
                ims, nrow=im_per_row, normalize=True, scale_each=True, padding=0
            )

            if header is not None:
                image_grid = torch.cat(
                    [header.to(image_grid.device), image_grid], dim=1
                )

            image_grid = image_grid.permute(1, 2, 0).cpu().numpy()
            trainer.exp.log_image(
                Image.fromarray((image_grid * 255).astype(np.uint8)),
                name=f"{mode}_{domain}_{task}_{str(curr_iter)}_#{logidx}",
                step=curr_iter,
            )

    def upsample(self, ims):
        h = max(im.shape[-2] for im in ims)
        w = max(im.shape[-1] for im in ims)
        new_ims = []
        for im in ims:
            im = interpolate(im, (h, w), mode="bilinear")
            new_ims.append(im)
        return new_ims

    def padd(self, ims):
        h = max(im.shape[-2] for im in ims)
        w = max(im.shape[-1] for im in ims)
        new_ims = []
        for im in ims:
            ih = im.shape[-2]
            iw = im.shape[-1]
            if ih != h or iw != w:
                padded = torch.zeros(im.shape[-3], h, w)
                padded[
                    :, (h - ih) // 2 : (h + ih) // 2, (w - iw) // 2 : (w + iw) // 2
                ] = im
                new_ims.append(padded)
            else:
                new_ims.append(im)

        return new_ims

    def log_architecture(self):
        write_architecture(self.trainer)

        if self.trainer.exp is None:
            return

        for f in Path(self.trainer.opts.output_path).glob("archi*.txt"):
            self.trainer.exp.log_asset(str(f), overwrite=True)