model.py

from __future__ import annotations

import pathlib
import pickle
import sys

import lpips
import numpy as np
import torch
import torch.nn as nn
from huggingface_hub import hf_hub_download

current_dir = pathlib.Path(__file__).parent
submodule_dir = current_dir / "stylegan3"
sys.path.insert(0, submodule_dir.as_posix())


class LPIPS(lpips.LPIPS):
    @staticmethod
    def preprocess(image: np.ndarray) -> torch.Tensor:
        data = torch.from_numpy(image).float() / 255
        data = data * 2 - 1
        return data.permute(2, 0, 1).unsqueeze(0)

    @torch.inference_mode()
    def compute_features(self, data: torch.Tensor) -> list[torch.Tensor]:
        data = self.scaling_layer(data)
        data = self.net(data)
        return [lpips.normalize_tensor(x) for x in data]

    @torch.inference_mode()
    def compute_distance(self, features0: list[torch.Tensor], features1: list[torch.Tensor]) -> float:
        res = 0
        for lin, x0, x1 in zip(self.lins, features0, features1):
            d = (x0 - x1) ** 2
            y = lin(d)
            y = lpips.lpips.spatial_average(y)
            res += y.item()
        return res


class Model:

    MODEL_NAMES = [
        "dogs_1024",
        "elephants_512",
        "horses_256",
        "bicycles_256",
        "lions_512",
        "giraffes_512",
        "parrots_512",
    ]
    TRUNCATION_TYPES = [
        "Multimodal (LPIPS)",
        "Multimodal (L2)",
        "Global",
    ]

    def __init__(self):
        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self._download_all_models()
        self._download_all_cluster_centers()
        self._download_all_cluster_center_images()

        self.model_name = self.MODEL_NAMES[0]
        self.model = self._load_model(self.model_name)
        self.cluster_centers = self._load_cluster_centers(self.model_name)
        self.cluster_center_images = self._load_cluster_center_images(self.model_name)

        self.lpips = LPIPS()
        self.cluster_center_lpips_feature_dict = self._compute_cluster_center_lpips_features()

    def _load_model(self, model_name: str) -> nn.Module:
        path = hf_hub_download("public-data/Self-Distilled-StyleGAN", f"models/{model_name}_pytorch.pkl")
        with open(path, "rb") as f:
            model = pickle.load(f)["G_ema"]
        model.eval()
        model.to(self.device)
        return model

    def _load_cluster_centers(self, model_name: str) -> torch.Tensor:
        path = hf_hub_download("public-data/Self-Distilled-StyleGAN", f"cluster_centers/{model_name}.npy")
        centers = np.load(path)
        centers = torch.from_numpy(centers).float().to(self.device)
        return centers

    def _load_cluster_center_images(self, model_name: str) -> np.ndarray:
        path = hf_hub_download("public-data/Self-Distilled-StyleGAN", f"cluster_center_images/{model_name}.npy")
        return np.load(path)

    def set_model(self, model_name: str) -> None:
        if model_name == self.model_name:
            return
        self.model_name = model_name
        self.model = self._load_model(model_name)
        self.cluster_centers = self._load_cluster_centers(model_name)
        self.cluster_center_images = self._load_cluster_center_images(model_name)

    def _download_all_models(self):
        for name in self.MODEL_NAMES:
            self._load_model(name)

    def _download_all_cluster_centers(self):
        for name in self.MODEL_NAMES:
            self._load_cluster_centers(name)

    def _download_all_cluster_center_images(self):
        for name in self.MODEL_NAMES:
            self._load_cluster_center_images(name)

    def generate_z(self, seed: int) -> torch.Tensor:
        seed = int(np.clip(seed, 0, np.iinfo(np.uint32).max))
        return torch.from_numpy(np.random.RandomState(seed).randn(1, self.model.z_dim)).float().to(self.device)

    def compute_w(self, z: torch.Tensor) -> torch.Tensor:
        label = torch.zeros((1, self.model.c_dim), device=self.device)
        w = self.model.mapping(z, label)
        return w

    @staticmethod
    def truncate_w(w_center: torch.Tensor, w: torch.Tensor, psi: float) -> torch.Tensor:
        if psi == 1:
            return w
        return w_center.lerp(w, psi)

    @torch.inference_mode()
    def synthesize(self, w: torch.Tensor) -> torch.Tensor:
        return self.model.synthesis(w)

    def postprocess(self, tensor: torch.Tensor) -> np.ndarray:
        tensor = (tensor.permute(0, 2, 3, 1) * 127.5 + 128).clamp(0, 255).to(torch.uint8)
        return tensor.cpu().numpy()

    def compute_lpips_features(self, image: np.ndarray) -> list[torch.Tensor]:
        data = self.lpips.preprocess(image)
        return self.lpips.compute_features(data)

    def _compute_cluster_center_lpips_features(self) -> dict[str, list[list[torch.Tensor]]]:
        res = dict()
        for name in self.MODEL_NAMES:
            images = self._load_cluster_center_images(name)
            res[name] = [self.compute_lpips_features(image) for image in images]
        return res

    def compute_distance_to_cluster_centers(self, ws: torch.Tensor, distance_type: str) -> list[torch.Tensor]:
        if distance_type == "l2":
            return self._compute_l2_distance_to_cluster_centers(ws)
        elif distance_type == "lpips":
            return self._compute_lpips_distance_to_cluster_centers(ws)
        else:
            raise ValueError

    def _compute_l2_distance_to_cluster_centers(self, ws: torch.Tensor) -> np.ndarray:
        dist2 = ((self.cluster_centers - ws[0, 0]) ** 2).sum(dim=1)
        return dist2.cpu().numpy()

    def _compute_lpips_distance_to_cluster_centers(self, ws: torch.Tensor) -> np.ndarray:
        x = self.synthesize(ws)
        x = self.postprocess(x)[0]
        feat0 = self.compute_lpips_features(x)
        cluster_center_features = self.cluster_center_lpips_feature_dict[self.model_name]
        distances = [self.lpips.compute_distance(feat0, feat1) for feat1 in cluster_center_features]
        return np.asarray(distances)

    def find_nearest_cluster_center(self, ws: torch.Tensor, distance_type: str) -> int:
        distances = self.compute_distance_to_cluster_centers(ws, distance_type)
        return int(np.argmin(distances))

    def generate_image(self, seed: int, truncation_psi: float, truncation_type: str) -> np.ndarray:
        z = self.generate_z(seed)
        ws = self.compute_w(z)
        if truncation_type == self.TRUNCATION_TYPES[2]:
            w0 = self.model.mapping.w_avg
        else:
            if truncation_type == self.TRUNCATION_TYPES[0]:
                distance_type = "lpips"
            elif truncation_type == self.TRUNCATION_TYPES[1]:
                distance_type = "l2"
            else:
                raise ValueError
            cluster_index = self.find_nearest_cluster_center(ws, distance_type)
            w0 = self.cluster_centers[cluster_index]
        new_ws = self.truncate_w(w0, ws, truncation_psi)
        out = self.synthesize(new_ws)
        out = self.postprocess(out)
        return out[0]

    def set_model_and_generate_image(
        self, model_name: str, seed: int, truncation_psi: float, truncation_type: str
    ) -> np.ndarray:
        self.set_model(model_name)
        return self.generate_image(seed, truncation_psi, truncation_type)