Spaces:
Runtime error
Runtime error
| from __future__ import annotations | |
| import os | |
| import pathlib | |
| import pickle | |
| import sys | |
| import lpips | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| from huggingface_hub import hf_hub_download | |
| current_dir = pathlib.Path(__file__).parent | |
| submodule_dir = current_dir / 'stylegan3' | |
| sys.path.insert(0, submodule_dir.as_posix()) | |
| HF_TOKEN = os.environ['HF_TOKEN'] | |
| class LPIPS(lpips.LPIPS): | |
| def preprocess(image: np.ndarray) -> torch.Tensor: | |
| data = torch.from_numpy(image).float() / 255 | |
| data = data * 2 - 1 | |
| return data.permute(2, 0, 1).unsqueeze(0) | |
| def compute_features(self, data: torch.Tensor) -> list[torch.Tensor]: | |
| data = self.scaling_layer(data) | |
| data = self.net(data) | |
| return [lpips.normalize_tensor(x) for x in data] | |
| def compute_distance(self, features0: list[torch.Tensor], | |
| features1: list[torch.Tensor]) -> float: | |
| res = 0 | |
| for lin, x0, x1 in zip(self.lins, features0, features1): | |
| d = (x0 - x1)**2 | |
| y = lin(d) | |
| y = lpips.lpips.spatial_average(y) | |
| res += y.item() | |
| return res | |
| class Model: | |
| MODEL_NAMES = [ | |
| 'dogs_1024', | |
| 'elephants_512', | |
| 'horses_256', | |
| 'bicycles_256', | |
| 'lions_512', | |
| 'giraffes_512', | |
| 'parrots_512', | |
| ] | |
| TRUNCATION_TYPES = [ | |
| 'Multimodal (LPIPS)', | |
| 'Multimodal (L2)', | |
| 'Global', | |
| ] | |
| def __init__(self, device: str | torch.device): | |
| self.device = torch.device(device) | |
| self._download_all_models() | |
| self._download_all_cluster_centers() | |
| self._download_all_cluster_center_images() | |
| self.model_name = self.MODEL_NAMES[0] | |
| self.model = self._load_model(self.model_name) | |
| self.cluster_centers = self._load_cluster_centers(self.model_name) | |
| self.cluster_center_images = self._load_cluster_center_images( | |
| self.model_name) | |
| self.lpips = LPIPS() | |
| self.cluster_center_lpips_feature_dict = self._compute_cluster_center_lpips_features( | |
| ) | |
| def _load_model(self, model_name: str) -> nn.Module: | |
| path = hf_hub_download('hysts/Self-Distilled-StyleGAN', | |
| f'models/{model_name}_pytorch.pkl', | |
| use_auth_token=HF_TOKEN) | |
| with open(path, 'rb') as f: | |
| model = pickle.load(f)['G_ema'] | |
| model.eval() | |
| model.to(self.device) | |
| return model | |
| def _load_cluster_centers(self, model_name: str) -> torch.Tensor: | |
| path = hf_hub_download('hysts/Self-Distilled-StyleGAN', | |
| f'cluster_centers/{model_name}.npy', | |
| use_auth_token=HF_TOKEN) | |
| centers = np.load(path) | |
| centers = torch.from_numpy(centers).float().to(self.device) | |
| return centers | |
| def _load_cluster_center_images(self, model_name: str) -> np.ndarray: | |
| path = hf_hub_download('hysts/Self-Distilled-StyleGAN', | |
| f'cluster_center_images/{model_name}.npy', | |
| use_auth_token=HF_TOKEN) | |
| return np.load(path) | |
| def set_model(self, model_name: str) -> None: | |
| if model_name == self.model_name: | |
| return | |
| self.model_name = model_name | |
| self.model = self._load_model(model_name) | |
| self.cluster_centers = self._load_cluster_centers(model_name) | |
| self.cluster_center_images = self._load_cluster_center_images( | |
| model_name) | |
| def _download_all_models(self): | |
| for name in self.MODEL_NAMES: | |
| self._load_model(name) | |
| def _download_all_cluster_centers(self): | |
| for name in self.MODEL_NAMES: | |
| self._load_cluster_centers(name) | |
| def _download_all_cluster_center_images(self): | |
| for name in self.MODEL_NAMES: | |
| self._load_cluster_center_images(name) | |
| def generate_z(self, seed: int) -> torch.Tensor: | |
| seed = int(np.clip(seed, 0, np.iinfo(np.uint32).max)) | |
| return torch.from_numpy( | |
| np.random.RandomState(seed).randn(1, self.model.z_dim)).float().to( | |
| self.device) | |
| def compute_w(self, z: torch.Tensor) -> torch.Tensor: | |
| label = torch.zeros((1, self.model.c_dim), device=self.device) | |
| w = self.model.mapping(z, label) | |
| return w | |
| def truncate_w(w_center: torch.Tensor, w: torch.Tensor, | |
| psi: float) -> torch.Tensor: | |
| if psi == 1: | |
| return w | |
| return w_center.lerp(w, psi) | |
| def synthesize(self, w: torch.Tensor) -> torch.Tensor: | |
| return self.model.synthesis(w) | |
| def postprocess(self, tensor: torch.Tensor) -> np.ndarray: | |
| tensor = (tensor.permute(0, 2, 3, 1) * 127.5 + 128).clamp(0, 255).to( | |
| torch.uint8) | |
| return tensor.cpu().numpy() | |
| def compute_lpips_features(self, image: np.ndarray) -> list[torch.Tensor]: | |
| data = self.lpips.preprocess(image) | |
| return self.lpips.compute_features(data) | |
| def _compute_cluster_center_lpips_features( | |
| self) -> dict[str, list[list[torch.Tensor]]]: | |
| res = dict() | |
| for name in self.MODEL_NAMES: | |
| images = self._load_cluster_center_images(name) | |
| res[name] = [ | |
| self.compute_lpips_features(image) for image in images | |
| ] | |
| return res | |
| def compute_distance_to_cluster_centers( | |
| self, ws: torch.Tensor, distance_type: str) -> list[torch.Tensor]: | |
| if distance_type == 'l2': | |
| return self._compute_l2_distance_to_cluster_centers(ws) | |
| elif distance_type == 'lpips': | |
| return self._compute_lpips_distance_to_cluster_centers(ws) | |
| else: | |
| raise ValueError | |
| def _compute_l2_distance_to_cluster_centers( | |
| self, ws: torch.Tensor) -> np.ndarray: | |
| dist2 = ((self.cluster_centers - ws[0, 0])**2).sum(dim=1) | |
| return dist2.cpu().numpy() | |
| def _compute_lpips_distance_to_cluster_centers( | |
| self, ws: torch.Tensor) -> np.ndarray: | |
| x = self.synthesize(ws) | |
| x = self.postprocess(x)[0] | |
| feat0 = self.compute_lpips_features(x) | |
| cluster_center_features = self.cluster_center_lpips_feature_dict[ | |
| self.model_name] | |
| distances = [ | |
| self.lpips.compute_distance(feat0, feat1) | |
| for feat1 in cluster_center_features | |
| ] | |
| return np.asarray(distances) | |
| def find_nearest_cluster_center(self, ws: torch.Tensor, | |
| distance_type: str) -> int: | |
| distances = self.compute_distance_to_cluster_centers(ws, distance_type) | |
| return int(np.argmin(distances)) | |
| def generate_image(self, seed: int, truncation_psi: float, | |
| truncation_type: str) -> np.ndarray: | |
| z = self.generate_z(seed) | |
| ws = self.compute_w(z) | |
| if truncation_type == self.TRUNCATION_TYPES[2]: | |
| w0 = self.model.mapping.w_avg | |
| else: | |
| if truncation_type == self.TRUNCATION_TYPES[0]: | |
| distance_type = 'lpips' | |
| elif truncation_type == self.TRUNCATION_TYPES[1]: | |
| distance_type = 'l2' | |
| else: | |
| raise ValueError | |
| cluster_index = self.find_nearest_cluster_center(ws, distance_type) | |
| w0 = self.cluster_centers[cluster_index] | |
| new_ws = self.truncate_w(w0, ws, truncation_psi) | |
| out = self.synthesize(new_ws) | |
| out = self.postprocess(out) | |
| return out[0] | |
| def set_model_and_generate_image(self, model_name: str, seed: int, | |
| truncation_psi: float, | |
| truncation_type: str) -> np.ndarray: | |
| self.set_model(model_name) | |
| return self.generate_image(seed, truncation_psi, truncation_type) | |