inference/.ipynb_checkpoints/icgan_colab-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bb75a2b9",
   "metadata": {},
   "source": [
    "Copyright (c) Facebook, Inc. and its affiliates.\n",
    "All rights reserved.\n",
    "\n",
    "This source code is licensed under the license found in the\n",
    "LICENSE file in the root directory of this source tree."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81a8ddb6",
   "metadata": {},
   "source": [
    "# IC-GAN\n",
    "\n",
    "\n",
    "Official Colab notebook from the paper <b>\"Instance-Conditioned GAN\"</b> by Arantxa Casanova, Marlene Careil, Jakob Verbeek, Michał Drożdżal, Adriana Romero-Soriano.\n",
    "\n",
    "This Colab provides the code to generate images with IC-GAN, with the option of further guiding the generation with captions (CLIP). \n",
    "\n",
    "Based on the Colab [WanderClip](https://j.mp/wanderclip) by Eyal Gruss [@eyaler](https://twitter.com/eyaler) [eyalgruss.com](https://eyalgruss.com)\n",
    "\n",
    "Using the work from [our repository](https://github.com/facebookresearch/ic_gan)\n",
    "\n",
    "https://github.com/openai/CLIP, Copyright (c) 2021 OpenAI\n",
    "\n",
    "https://github.com/huggingface/pytorch-pretrained-BigGAN, Copyright (c) 2019 Thomas Wolf\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9442671e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Restart after running this cell!\n",
    "\n",
    "!nvidia-smi -L\n",
    "\n",
    "import subprocess\n",
    "\n",
    "CUDA_version = [s for s in subprocess.check_output([\"nvcc\", \"--version\"]).decode(\"UTF-8\").split(\", \") if s.startswith(\"release\")][0].split(\" \")[-1]\n",
    "print(\"CUDA version:\", CUDA_version)\n",
    "\n",
    "if CUDA_version == \"10.1\":\n",
    "    torch_version_suffix = \"+cu101\"\n",
    "elif CUDA_version == \"10.2\":\n",
    "    torch_version_suffix = \"+cu102\"\n",
    "else:\n",
    "    torch_version_suffix = \"+cu111\"\n",
    "\n",
    "!pip install torch==1.8.0{torch_version_suffix} torchvision==0.8.2{torch_version_suffix} -f https://download.pytorch.org/whl/torch_stable.html ftfy regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b01f51f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Setup\n",
    "!git clone https://github.com/facebookresearch/ic_gan.git\n",
    "\n",
    "%cd /content\n",
    "# Uncompress required files\n",
    "!wget https://dl.fbaipublicfiles.com/ic_gan/cc_icgan_biggan_imagenet_res256.tar.gz\n",
    "!tar -xvf cc_icgan_biggan_imagenet_res256.tar.gz\n",
    "!wget https://dl.fbaipublicfiles.com/ic_gan/icgan_biggan_imagenet_res256.tar.gz\n",
    "!tar -xvf icgan_biggan_imagenet_res256.tar.gz\n",
    "!wget https://dl.fbaipublicfiles.com/ic_gan/stored_instances.tar.gz\n",
    "!tar -xvf stored_instances.tar.gz\n",
    "\n",
    "!pip install pytorch-pretrained-biggan\n",
    "\n",
    "!git clone --depth 1 https://github.com/openai/CLIP\n",
    "!pip install ftfy\n",
    "%cd /content/CLIP\n",
    "import clip\n",
    "last_clip_model = 'ViT-B/32'\n",
    "perceptor, preprocess = clip.load(last_clip_model)\n",
    "\n",
    "import nltk\n",
    "nltk.download('wordnet')\n",
    "\n",
    "!pip install cma\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa6c7629",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Prepare functions\n",
    "from pytorch_pretrained_biggan import BigGAN, convert_to_images, one_hot_from_names, utils\n",
    "\n",
    "%cd /content/ic_gan/\n",
    "import sys\n",
    "import os\n",
    "sys.path[0] = '/content/ic_gan/inference'\n",
    "sys.path.insert(1, os.path.join(sys.path[0], \"..\"))\n",
    "import torch \n",
    "\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchvision\n",
    "import sys\n",
    "torch.manual_seed(np.random.randint(sys.maxsize))\n",
    "import imageio\n",
    "from IPython.display import HTML, Image, clear_output\n",
    "from PIL import Image as Image_PIL\n",
    "from scipy.stats import truncnorm, dirichlet\n",
    "from torch import nn\n",
    "from nltk.corpus import wordnet as wn\n",
    "from base64 import b64encode\n",
    "from time import time\n",
    "import cma\n",
    "from cma.sigma_adaptation import CMAAdaptSigmaCSA, CMAAdaptSigmaTPA\n",
    "import warnings\n",
    "warnings.simplefilter(\"ignore\", cma.evolution_strategy.InjectionWarning)\n",
    "import torchvision.transforms as transforms\n",
    "import inference.utils as inference_utils\n",
    "import data_utils.utils as data_utils\n",
    "from BigGAN_PyTorch.BigGAN import Generator as generator\n",
    "import sklearn.metrics\n",
    "\n",
    "def replace_to_inplace_relu(model): #saves memory; from https://github.com/minyoungg/pix2latent/blob/master/pix2latent/model/biggan.py\n",
    "    for child_name, child in model.named_children():\n",
    "        if isinstance(child, nn.ReLU):\n",
    "            setattr(model, child_name, nn.ReLU(inplace=False))\n",
    "        else:\n",
    "            replace_to_inplace_relu(child)\n",
    "    return\n",
    "    \n",
    "def save(out,name=None, torch_format=True):\n",
    "  if torch_format:\n",
    "    with torch.no_grad():\n",
    "      out = out.cpu().numpy()\n",
    "  img = convert_to_images(out)[0]\n",
    "  if name:\n",
    "    imageio.imwrite(name, np.asarray(img))\n",
    "  return img\n",
    "\n",
    "hist = []\n",
    "def checkin(i, best_ind, total_losses, losses, regs, out, noise=None, emb=None, probs=None):\n",
    "  global sample_num, hist\n",
    "  name = None\n",
    "  if save_every and i%save_every==0:\n",
    "    name = '/content/output/frame_%05d.jpg'%sample_num\n",
    "  pil_image = save(out, name)\n",
    "  vals0 = [sample_num, i, total_losses[best_ind], losses[best_ind], regs[best_ind], np.mean(total_losses), np.mean(losses), np.mean(regs), np.std(total_losses), np.std(losses), np.std(regs)]\n",
    "  stats = 'sample=%d iter=%d best: total=%.2f cos=%.2f reg=%.3f avg: total=%.2f cos=%.2f reg=%.3f std: total=%.2f cos=%.2f reg=%.3f'%tuple(vals0)\n",
    "  vals1 = []\n",
    "  if noise is not None:\n",
    "    vals1 = [np.mean(noise), np.std(noise)]\n",
    "    stats += ' noise: avg=%.2f std=%.3f'%tuple(vals1)\n",
    "  vals2 = []\n",
    "  if emb is not None:\n",
    "    vals2 = [emb.mean(),emb.std()]\n",
    "    stats += ' emb: avg=%.2f std=%.3f'%tuple(vals2)\n",
    "  elif probs:\n",
    "    best = probs[best_ind]\n",
    "    inds = np.argsort(best)[::-1]\n",
    "    probs = np.array(probs)\n",
    "    vals2 = [ind2name[inds[0]], best[inds[0]], ind2name[inds[1]], best[inds[1]], ind2name[inds[2]], best[inds[2]], np.sum(probs >= 0.5)/pop_size,np.sum(probs >= 0.3)/pop_size,np.sum(probs >= 0.1)/pop_size]\n",
    "    stats += ' 1st=%s(%.2f) 2nd=%s(%.2f) 3rd=%s(%.2f) components: >=0.5:%.0f, >=0.3:%.0f, >=0.1:%.0f'%tuple(vals2)\n",
    "  hist.append(vals0+vals1+vals2)\n",
    "  if show_every and i%show_every==0:\n",
    "    clear_output()\n",
    "    display(pil_image)  \n",
    "  print(stats)\n",
    "  sample_num += 1\n",
    "\n",
    "def load_icgan(experiment_name, root_ = '/content'):\n",
    "  root = os.path.join(root_, experiment_name)\n",
    "  config = torch.load(\"%s/%s.pth\" %\n",
    "                      (root, \"state_dict_best0\"))['config']\n",
    "\n",
    "  config[\"weights_root\"] = root_\n",
    "  config[\"model_backbone\"] = 'biggan'\n",
    "  config[\"experiment_name\"] = experiment_name\n",
    "  # TODO: delete this line\n",
    "  G, config = inference_utils.load_model_inference(config)\n",
    "  G.cuda()\n",
    "  G.eval()\n",
    "  return G\n",
    "\n",
    "def get_output(noise_vector, input_label, input_features):  \n",
    "  if stochastic_truncation: #https://arxiv.org/abs/1702.04782\n",
    "    with torch.no_grad():\n",
    "      trunc_indices = noise_vector.abs() > 2*truncation\n",
    "      size = torch.count_nonzero(trunc_indices).cpu().numpy()\n",
    "      trunc = truncnorm.rvs(-2*truncation, 2*truncation, size=(1,size)).astype(np.float32)\n",
    "      noise_vector.data[trunc_indices] = torch.tensor(trunc, requires_grad=requires_grad, device='cuda')\n",
    "  else:\n",
    "    noise_vector = noise_vector.clamp(-2*truncation, 2*truncation)\n",
    "  if input_label is not None:\n",
    "    input_label = torch.LongTensor(input_label)\n",
    "  else:\n",
    "    input_label = None\n",
    "\n",
    "  out = model(noise_vector, input_label.cuda() if input_label is not None else None, input_features.cuda() if input_features is not None else None)\n",
    "  \n",
    "  if channels==1:\n",
    "    out = out.mean(dim=1, keepdim=True)\n",
    "    out = out.repeat(1,3,1,1)\n",
    "  return out\n",
    "\n",
    "def normality_loss(vec): #https://arxiv.org/abs/1903.00925\n",
    "    mu2 = vec.mean().square()\n",
    "    sigma2 = vec.var()\n",
    "    return mu2+sigma2-torch.log(sigma2)-1\n",
    "    \n",
    "\n",
    "def load_generative_model(gen_model, last_gen_model, experiment_name, model):\n",
    "  # Load generative model\n",
    "  if gen_model != last_gen_model:\n",
    "    model = load_icgan(experiment_name, root_ = '/content')\n",
    "    last_gen_model = gen_model\n",
    "  return model, last_gen_model\n",
    "\n",
    "def load_feature_extractor(gen_model, last_feature_extractor, feature_extractor):\n",
    "  # Load feature extractor to obtain instance features\n",
    "  feat_ext_name = 'classification' if gen_model == 'cc_icgan' else 'selfsupervised'\n",
    "  if last_feature_extractor != feat_ext_name:\n",
    "    if feat_ext_name == 'classification':\n",
    "      feat_ext_path = ''\n",
    "    else:\n",
    "      !curl -L -o /content/swav_pretrained.pth.tar -C - 'https://dl.fbaipublicfiles.com/deepcluster/swav_800ep_pretrain.pth.tar' \n",
    "      feat_ext_path = '/content/swav_pretrained.pth.tar'\n",
    "    last_feature_extractor = feat_ext_name\n",
    "    feature_extractor = data_utils.load_pretrained_feature_extractor(feat_ext_path, feature_extractor = feat_ext_name)\n",
    "    feature_extractor.eval()\n",
    "  return feature_extractor, last_feature_extractor\n",
    "\n",
    "norm_mean = torch.Tensor([0.485, 0.456, 0.406]).view(3, 1, 1)\n",
    "norm_std = torch.Tensor([0.229, 0.224, 0.225]).view(3, 1, 1)\n",
    "\n",
    "def preprocess_input_image(input_image_path, size): \n",
    "  pil_image = Image_PIL.open(input_image_path).convert('RGB')\n",
    "  transform_list =  transforms.Compose([data_utils.CenterCropLongEdge(), transforms.Resize((size,size)), transforms.ToTensor(), transforms.Normalize(norm_mean, norm_std)])\n",
    "  tensor_image = transform_list(pil_image)\n",
    "  tensor_image = torch.nn.functional.interpolate(tensor_image.unsqueeze(0), 224, mode=\"bicubic\", align_corners=True)\n",
    "  return tensor_image\n",
    "\n",
    "def preprocess_generated_image(image): \n",
    "  transform_list =  transforms.Normalize(norm_mean, norm_std)\n",
    "  image = transform_list(image*0.5 + 0.5)\n",
    "  image = torch.nn.functional.interpolate(image, 224, mode=\"bicubic\", align_corners=True)\n",
    "  return image\n",
    "\n",
    "last_gen_model = None\n",
    "last_feature_extractor = None\n",
    "model = None\n",
    "feature_extractor = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17278e04",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Generate images with IC-GAN!\n",
    "#@markdown 1. Select type of IC-GAN model with **gen_model**: \"icgan\" is conditioned on an instance; \"cc_icgan\" is conditioned on both instance and a class index.\n",
    "#@markdown 1. Select which instance to condition on, following one of the following options:\n",
    "#@markdown     1. **input_image_instance** is the path to an input image, from either the mounted Google Drive or a manually uploaded image to \"Files\" (left part of the screen).\n",
    "#@markdown     1. **input_feature_index** write an integer from 0 to 1000. This will change the instance conditioning and therefore the style and semantics of the generated images. This will select one of the 1000 instance features pre-selected from ImageNet using k-means.\n",
    "#@markdown 1. For **class_index** (only valid for gen_model=\"cc_icgan\") write an integer from 0 to 1000. This will change the ImageNet class to condition on. Consult [this link](https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a) for a correspondence between class name and indexes.\n",
    "#@markdown 1. **num_samples_ranked** (default=16) indicates the number of generated images to output in a mosaic. These generated images are the ones that scored a higher cosine similarity with the conditioning instance, out of **num_samples_total** (default=160) generated samples. Increasing \"num_samples_total\" will result in higher run times, but more generated images to choose the top \"num_samples_ranked\" from, and therefore higher chance of better image quality. Reducing \"num_samples_total\" too much could result in generated images with poorer visual quality. A ratio of 10:1 (num_samples_total:num_samples_ranked) is recommended.\n",
    "#@markdown 1. Vary **truncation** (default=0.7) from 0 to 1 to apply the [truncation trick](https://arxiv.org/abs/1809.11096). Truncation=1 will provide more diverse but possibly poorer quality images. Trucation values between 0.7 and 0.9 seem to empirically work well.\n",
    "#@markdown 1. **seed**=0 means no seed.\n",
    "\n",
    "gen_model = 'icgan' #@param ['icgan', 'cc_icgan']\n",
    "if gen_model == 'icgan':  \n",
    "  experiment_name = 'icgan_biggan_imagenet_res256'\n",
    "else:\n",
    "  experiment_name = 'cc_icgan_biggan_imagenet_res256'\n",
    "#last_gen_model = experiment_name\n",
    "size = '256'\n",
    "input_image_instance = \"\"#@param {type:\"string\"}\n",
    "input_feature_index =   3#@param {type:'integer'}\n",
    "class_index =   538#@param {type:'integer'}\n",
    "num_samples_ranked =   16#@param {type:'integer'}\n",
    "num_samples_total =    160#@param {type:'integer'}\n",
    "truncation =  0.7#@param {type:'number'}\n",
    "stochastic_truncation = False #@param {type:'boolean'}\n",
    "download_file = True #@param {type:'boolean'}\n",
    "seed =  50#@param {type:'number'}\n",
    "if seed == 0:\n",
    "  seed = None\n",
    "noise_size = 128\n",
    "class_size = 1000\n",
    "channels = 3\n",
    "batch_size = 4\n",
    "if gen_model == 'icgan':\n",
    "  class_index = None\n",
    "if 'biggan' in gen_model:\n",
    "  input_feature_index = None\n",
    "  input_image_instance = None\n",
    "\n",
    "assert(num_samples_ranked <=num_samples_total)\n",
    "import numpy as np\n",
    "state = None if not seed else np.random.RandomState(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "feature_extractor_name = 'classification' if gen_model == 'cc_icgan' else 'selfsupervised'\n",
    "\n",
    "# Load feature extractor (outlier filtering and optionally input image feature extraction)\n",
    "feature_extractor, last_feature_extractor = load_feature_extractor(gen_model, last_feature_extractor, feature_extractor)\n",
    "# Load features \n",
    "if input_image_instance not in ['None', \"\"]:\n",
    "  print('Obtaining instance features from input image!')\n",
    "  input_feature_index = None\n",
    "  input_image_tensor = preprocess_input_image(input_image_instance, int(size))\n",
    "  print('Displaying instance conditioning:')\n",
    "  display(convert_to_images(((input_image_tensor*norm_std + norm_mean)-0.5) / 0.5)[0])\n",
    "  with torch.no_grad():\n",
    "    input_features, _ = feature_extractor(input_image_tensor.cuda())\n",
    "  input_features/=torch.linalg.norm(input_features,dim=-1, keepdims=True)\n",
    "elif input_feature_index is not None:\n",
    "  print('Selecting an instance from pre-extracted vectors!')\n",
    "  input_features = np.load('/content/stored_instances/imagenet_res'+str(size)+'_rn50_'+feature_extractor_name+'_kmeans_k1000_instance_features.npy', allow_pickle=True).item()[\"instance_features\"][input_feature_index:input_feature_index+1]\n",
    "else:\n",
    "  input_features = None\n",
    "\n",
    "# Load generative model\n",
    "model, last_gen_model = load_generative_model(gen_model, last_gen_model, experiment_name, model)\n",
    "# Prepare other variables\n",
    "name_file = '%s_class_index%s_instance_index%s'%(gen_model, str(class_index) if class_index is not None else 'None', str(input_feature_index) if input_feature_index is not None else 'None')\n",
    "\n",
    "!rm -rf /content/output\n",
    "!mkdir -p /content/output\n",
    "\n",
    "replace_to_inplace_relu(model)\n",
    "ind2name = {index: wn.of2ss('%08dn'%offset).lemma_names()[0] for offset, index in utils.IMAGENET.items()}\n",
    "\n",
    "from google.colab import files, output\n",
    "\n",
    "eps = 1e-8\n",
    "\n",
    "# Create noise, instance and class vector\n",
    "noise_vector = truncnorm.rvs(-2*truncation, 2*truncation, size=(num_samples_total, noise_size), random_state=state).astype(np.float32) #see https://github.com/tensorflow/hub/issues/214\n",
    "noise_vector = torch.tensor(noise_vector, requires_grad=False, device='cuda')\n",
    "if input_features is not None:\n",
    "  instance_vector = torch.tensor(input_features, requires_grad=False, device='cuda').repeat(num_samples_total, 1)\n",
    "else: \n",
    "  instance_vector = None\n",
    "if class_index is not None:\n",
    "  print('Conditioning on class: ', ind2name[class_index])\n",
    "  input_label = torch.LongTensor([class_index]*num_samples_total)\n",
    "else:\n",
    "  input_label = None\n",
    "if input_feature_index is not None:\n",
    "  print('Conditioning on instance with index: ', input_feature_index)\n",
    "\n",
    "size = int(size)\n",
    "all_outs, all_dists = [], []\n",
    "for i_bs in range(num_samples_total//batch_size+1):\n",
    "  start = i_bs*batch_size\n",
    "  end = min(start+batch_size, num_samples_total)\n",
    "  if start == end:\n",
    "    break\n",
    "  out = get_output(noise_vector[start:end], input_label[start:end] if input_label is not None else None, instance_vector[start:end] if instance_vector is not None else None)\n",
    "\n",
    "  if instance_vector is not None:\n",
    "    # Get features from generated images + feature extractor\n",
    "    out_ = preprocess_generated_image(out)\n",
    "    with torch.no_grad():\n",
    "      out_features, _ = feature_extractor(out_.cuda())\n",
    "    out_features/=torch.linalg.norm(out_features,dim=-1, keepdims=True)\n",
    "    dists = sklearn.metrics.pairwise_distances(\n",
    "            out_features.cpu(), instance_vector[start:end].cpu(), metric=\"euclidean\", n_jobs=-1)\n",
    "    all_dists.append(np.diagonal(dists))\n",
    "    all_outs.append(out.detach().cpu())\n",
    "  del (out)\n",
    "all_outs = torch.cat(all_outs)\n",
    "all_dists = np.concatenate(all_dists)\n",
    "\n",
    "# Order samples by distance to conditioning feature vector and select only num_samples_ranked images\n",
    "selected_idxs =np.argsort(all_dists)[:num_samples_ranked]\n",
    "#print('All distances re-ordered ', np.sort(all_dists))\n",
    "# Create figure                \n",
    "row_i, col_i, i_im = 0, 0, 0\n",
    "all_images_mosaic = np.zeros((3,size*(int(np.sqrt(num_samples_ranked))), size*(int(np.sqrt(num_samples_ranked)))))\n",
    "for j in selected_idxs:\n",
    "  all_images_mosaic[:,row_i*size:row_i*size+size, col_i*size:col_i*size+size] = all_outs[j]\n",
    "  if row_i == int(np.sqrt(num_samples_ranked))-1:\n",
    "    row_i = 0\n",
    "    if col_i == int(np.sqrt(num_samples_ranked))-1:\n",
    "      col_i = 0\n",
    "    else:\n",
    "      col_i +=1\n",
    "  else:\n",
    "    row_i+=1\n",
    "  i_im +=1\n",
    "\n",
    "name = '/content/%s_seed%i.png'%(name_file,seed if seed is not None else -1)\n",
    "pil_image = save(all_images_mosaic[np.newaxis,...],name, torch_format=False)  \n",
    "print('Displaying generated images')\n",
    "display(pil_image)\n",
    "\n",
    "if download_file:\n",
    "  files.download(name)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da5ee254",
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Generate images with IC-GAN + CLIP!\n",
    "#@markdown 1. For **prompt** OpenAI suggest to use the template \"A photo of a X.\" or \"A photo of a X, a type of Y.\" [[paper]](https://cdn.openai.com/papers/Learning_Transferable_Visual_Models_From_Natural_Language_Supervision.pdf)\n",
    "#@markdown 1. Select type of IC-GAN model with **gen_model**: \"icgan\" is conditioned on an instance; \"cc_icgan\" is conditioned on both instance and a class index.\n",
    "#@markdown 1. Select which instance to condition on, following one of the following options:\n",
    "#@markdown     1. **input_image_instance** is the path to an input image, from either the mounted Google Drive or a manually uploaded image to \"Files\" (left part of the screen).\n",
    "#@markdown     1. **input_feature_index** write an integer from 0 to 1000. This will change the instance conditioning and therefore the style and semantics of the generated images. This will select one of the 1000 instance features pre-selected from ImageNet using k-means.\n",
    "#@markdown 1. For **class_index** (only valid for gen_model=\"cc_icgan\") write an integer from 0 to 1000. This will change the ImageNet class to condition on. Consult [this link](https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a) for a correspondence between class name and indexes.\n",
    "#@markdown 1. Vary **truncation** from 0 to 1 to apply the [truncation trick](https://arxiv.org/abs/1809.11096). Truncation=1 will provide more diverse but possibly poorer quality images. Trucation values between 0.7 and 0.9 seem to empirically work well.\n",
    "#@markdown 4. **seed**=0 means no seed.\n",
    "prompt = 'A dragon' #@param {type:'string'}\n",
    "gen_model = 'icgan' #@param ['icgan', 'cc_icgan']\n",
    "if gen_model == 'icgan':  \n",
    "  experiment_name = 'icgan_biggan_imagenet_res256_nofeataug'\n",
    "else:\n",
    "  experiment_name = 'cc_icgan_biggan_imagenet_res256_nofeataug'\n",
    "#last_gen_model = experiment_name\n",
    "size = '256'\n",
    "input_image_instance = \"\"#@param {type:\"string\"}\n",
    "\n",
    "input_feature_index =   500#@param {type:'integer'}\n",
    "class_index =  627 #@param {type:'integer'} (only with cc_icgan)\n",
    "download_image = False #@param {type:'boolean'}\n",
    "download_video = False #@param {type:'boolean'}\n",
    "truncation = 0.7 #@param {type:'number'}\n",
    "stochastic_truncation = False #@param {type:'boolean'}\n",
    "optimizer = 'CMA-ES' #@param ['SGD','Adam','CMA-ES','CMA-ES + SGD interleaved','CMA-ES + Adam interleaved','CMA-ES + terminal SGD','CMA-ES + terminal Adam']\n",
    "pop_size = 50 #@param {type:'integer'}\n",
    "clip_model = 'ViT-B/32' #@param ['ViT-B/32','RN50','RN101','RN50x4']\n",
    "augmentations =  64#@param {type:'integer'}\n",
    "learning_rate =  0.1#@param {type:'number'}\n",
    "noise_normality_loss =  0#@param {type:'number'}\n",
    "minimum_entropy_loss = 0.0001 #@param {type:'number'}\n",
    "total_variation_loss = 0.1 #@param {type:'number'}\n",
    "iterations =  100#@param {type:'integer'}\n",
    "terminal_iterations =  100#@param {type:'integer'}\n",
    "show_every = 1 #@param {type:'integer'}\n",
    "save_every = 1 #@param {type:'integer'}\n",
    "fps =  2#@param {type:'number'}\n",
    "freeze_secs = 0 #@param {type:'number'}\n",
    "seed =  10#@param {type:'number'}\n",
    "if seed == 0:\n",
    "  seed = None\n",
    "\n",
    "softmax_temp = 1\n",
    "emb_factor = 0.067 #calculated empirically \n",
    "loss_factor = 100\n",
    "sigma0 = 0.5 #http://cma.gforge.inria.fr/cmaes_sourcecode_page.html#practical\n",
    "cma_adapt = True\n",
    "cma_diag = False\n",
    "cma_active = True\n",
    "cma_elitist = False\n",
    "noise_size = 128\n",
    "class_size = 1000\n",
    "channels = 3\n",
    "if gen_model == 'icgan':\n",
    "  class_index = None\n",
    "\n",
    "import numpy as np\n",
    "state = None if not seed else np.random.RandomState(seed)\n",
    "np.random.seed(seed)\n",
    "# Load features \n",
    "if input_image_instance not in ['None',\"\"]:\n",
    "  print('Obtaining instance features from input image!')\n",
    "  input_feature_index = None\n",
    "  feature_extractor, last_feature_extractor = load_feature_extractor(gen_model, last_feature_extractor, feature_extractor)\n",
    "  input_image_tensor = preprocess_input_image(input_image_instance, int(size))\n",
    "  input_features, _ = feature_extractor(input_image_tensor.cuda())\n",
    "  input_features/=torch.linalg.norm(input_features,dim=-1, keepdims=True)\n",
    "elif input_feature_index is not None:\n",
    "  print('Selecting an instance from pre-extracted vectors!')\n",
    "  feature_extractor_name = 'classification' if gen_model == 'cc_icgan' else 'selfsupervised'\n",
    "  input_features = np.load('/content/stored_instances/imagenet_res'+str(size)+'_rn50_'+feature_extractor_name+'_kmeans_k1000_instance_features.npy', allow_pickle=True).item()[\"instance_features\"][input_feature_index:input_feature_index+1]\n",
    "else:\n",
    "  input_features = None\n",
    "\n",
    "\n",
    "# Load generative model\n",
    "model, last_gen_model = load_generative_model(gen_model, last_gen_model, experiment_name, model)\n",
    "\n",
    "# Load CLIP model\n",
    "if clip_model != last_clip_model:\n",
    "  perceptor, preprocess = clip.load(clip_model)\n",
    "  last_clip_model = clip_model\n",
    "clip_res = perceptor.visual.input_resolution\n",
    "sideX = sideY = int(size)\n",
    "if sideX<=clip_res and sideY<=clip_res:\n",
    "  augmentations = 1\n",
    "if 'CMA' not in optimizer:\n",
    "  pop_size = 1\n",
    "\n",
    "# Prepare other variables\n",
    "name_file = '%s_%s_class_index%s_instance_index%s'%(gen_model, prompt, str(class_index) if class_index is not None else 'None', str(input_feature_index) if input_feature_index is not None else 'None')\n",
    "requires_grad = ('SGD' in optimizer or 'Adam' in optimizer) and ('terminal' not in optimizer or terminal_iterations>0)\n",
    "total_iterations = iterations + terminal_iterations*('terminal' in optimizer)\n",
    "\n",
    "!rm -rf /content/output\n",
    "!mkdir -p /content/output\n",
    "\n",
    "replace_to_inplace_relu(model)\n",
    "replace_to_inplace_relu(perceptor)\n",
    "ind2name = {index: wn.of2ss('%08dn'%offset).lemma_names()[0] for offset, index in utils.IMAGENET.items()}\n",
    "eps = 1e-8\n",
    "\n",
    "# Create noise and instance vector\n",
    "noise_vector = truncnorm.rvs(-2*truncation, 2*truncation, size=(pop_size, noise_size), random_state=state).astype(np.float32) #see https://github.com/tensorflow/hub/issues/214\n",
    "noise_vector = torch.tensor(noise_vector, requires_grad=requires_grad, device='cuda')\n",
    "if input_features is not None:\n",
    "  instance_vector = torch.tensor(input_features, requires_grad=False, device='cuda')\n",
    "else: \n",
    "  instance_vector = None\n",
    "if class_index is not None:\n",
    "  print('Conditioning on class: ', ind2name[class_index])\n",
    "if input_feature_index is not None:\n",
    "  print('Conditioning on instance with index: ', input_feature_index)\n",
    "\n",
    "# Prepare optimizer\n",
    "if requires_grad:\n",
    "  params = [noise_vector]\n",
    "  if 'SGD' in optimizer:\n",
    "    optim = torch.optim.SGD(params, lr=learning_rate, momentum=0.9)  \n",
    "  else:\n",
    "    optim = torch.optim.Adam(params, lr=learning_rate)\n",
    "\n",
    "def ascend_txt(i, grad_step=False, show_save=False):\n",
    "  global global_best_loss, global_best_iteration, global_best_noise_vector, global_best_class_vector\n",
    "  regs = []\n",
    "  losses = []\n",
    "  total_losses = []\n",
    "  best_loss = np.inf\n",
    "  global_reg = torch.tensor(0, device='cuda', dtype=torch.float32, requires_grad=grad_step)\n",
    "  if noise_normality_loss:\n",
    "    global_reg = global_reg+noise_normality_loss*normality_loss(noise_vector)\n",
    "  global_reg = loss_factor*global_reg  \n",
    "  if grad_step:\n",
    "    global_reg.backward()\n",
    "  for j in range(pop_size):\n",
    "    p_s = []\n",
    "    out = get_output(noise_vector[j:j+1], [class_index] if class_index is not None else None, instance_vector)\n",
    "    for aug in range(augmentations):\n",
    "      if sideX<=clip_res and sideY<=clip_res or augmentations==1:\n",
    "        apper = out  \n",
    "      else:\n",
    "        size = torch.randint(int(.7*sideX), int(.98*sideX), ())\n",
    "        offsetx = torch.randint(0, sideX - size, ())\n",
    "        offsety = torch.randint(0, sideX - size, ())\n",
    "        apper = out[:, :, offsetx:offsetx + size, offsety:offsety + size]\n",
    "      apper = (apper+1)/2\n",
    "      apper = nn.functional.interpolate(apper, clip_res, mode='bilinear')\n",
    "      #apper = apper.clamp(0,1)\n",
    "      p_s.append(apper)\n",
    "    into = nom(torch.cat(p_s, 0))\n",
    "    predict_clip = perceptor.encode_image(into)\n",
    "    loss = loss_factor*(1-torch.cosine_similarity(predict_clip, target_clip).mean())\n",
    "    total_loss = loss\n",
    "    regs.append(global_reg.item())\n",
    "\n",
    "    with torch.no_grad():\n",
    "      losses.append(loss.item())\n",
    "      total_losses.append(total_loss.item()+global_reg.item())\n",
    "    if total_losses[-1]<best_loss:\n",
    "      best_loss = total_losses[-1]\n",
    "      best_ind = j\n",
    "      best_out = out\n",
    "      if best_loss < global_best_loss:\n",
    "        global_best_loss = best_loss\n",
    "        global_best_iteration = i\n",
    "        with torch.no_grad():\n",
    "          global_best_noise_vector = noise_vector[best_ind]\n",
    "    if grad_step:    \n",
    "      total_loss.backward()\n",
    "\n",
    "  if grad_step:\n",
    "    optim.step()\n",
    "    optim.zero_grad()\n",
    "\n",
    "  if show_save and (save_every and i % save_every == 0 or show_every and i % show_every == 0):\n",
    "    noise = None\n",
    "    emb = None\n",
    "    with torch.no_grad():\n",
    "      noise = noise_vector.cpu().numpy()\n",
    "    checkin(i, best_ind, total_losses, losses, regs, best_out, noise, emb)  \n",
    "  return total_losses, best_ind\n",
    "\n",
    "# Obtain target CLIP representation\n",
    "tx = clip.tokenize(prompt)\n",
    "with torch.no_grad():\n",
    "  target_clip = perceptor.encode_text(tx.cuda())\n",
    "\n",
    "\n",
    "global_best_loss = np.inf\n",
    "global_best_iteration = 0\n",
    "global_best_noise_vector = None\n",
    "\n",
    "nom = torchvision.transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))\n",
    "if 'CMA' in optimizer:\n",
    "  initial_vector = np.zeros(noise_size)\n",
    "  bounds = None\n",
    "  cma_opts = {'popsize': pop_size, 'seed': np.nan, 'AdaptSigma': cma_adapt, 'CMA_diagonal': cma_diag, 'CMA_active': cma_active, 'CMA_elitist':cma_elitist, 'bounds':bounds}\n",
    "  cmaes = cma.CMAEvolutionStrategy(initial_vector, sigma0, inopts=cma_opts)\n",
    "\n",
    "sample_num = 0\n",
    "machine = !nvidia-smi -L\n",
    "start = time()\n",
    "\n",
    "# Start noise vector optimization\n",
    "for i in range(total_iterations):    \n",
    "  if 'CMA' in optimizer and i<iterations:\n",
    "    with torch.no_grad():\n",
    "      cma_results = torch.tensor(cmaes.ask(), dtype=torch.float32).cuda()\n",
    "      noise_vector.data = cma_results      \n",
    "  if requires_grad and ('terminal' not in optimizer or i>=iterations):\n",
    "    losses, best_ind = ascend_txt(i, grad_step=True, show_save='CMA' not in optimizer or i>=iterations)\n",
    "    assert noise_vector.requires_grad and noise_vector.is_leaf and (not optimize_class or class_vector.requires_grad and class_vector.is_leaf), (noise_vector.requires_grad, noise_vector.is_leaf, class_vector.requires_grad, class_vector.is_leaf)\n",
    "  if 'CMA' in optimizer and i<iterations:\n",
    "    with torch.no_grad():\n",
    "      losses, best_ind = ascend_txt(i, show_save=True)\n",
    "      if i<iterations-1:\n",
    "        vectors = noise_vector\n",
    "        cmaes.tell(vectors.cpu().numpy(), losses)\n",
    "      elif 'terminal' in optimizer and terminal_iterations:\n",
    "        pop_size = 1\n",
    "        noise_vector[0] = global_best_noise_vector\n",
    "  if save_every and i % save_every == 0 or show_every and i % show_every == 0:\n",
    "    print('took: %d secs (%.2f sec/iter) on %s. CUDA memory: %.1f GB'%(time()-start,(time()-start)/(i+1), machine[0], torch.cuda.max_memory_allocated()/1024**3))\n",
    "\n",
    "# Obtain generated image with lowest loss.\n",
    "out = get_output(global_best_noise_vector.unsqueeze(0), [class_index] if class_index is not None else None, instance_vector)\n",
    "name = '/content/%s_best_seed%i.png'%(name_file,seed if seed is not None else -1)\n",
    "pil_image = save(out,name)  \n",
    "display(pil_image)  \n",
    "print('best_loss=%.2f best_iter=%d'%(global_best_loss,global_best_iteration))\n",
    "\n",
    "if download_image:\n",
    "  from google.colab import files, output\n",
    "  files.download(name)\n",
    "\n",
    "if download_video:\n",
    "  out = '\"/content/%s_seed%i.mp4\"'%(name_file, seed if seed is not None else -1)\n",
    "  file_name = '/content/%s_seed%i.mp4'%(name_file, seed if seed is not None else -1)\n",
    "\n",
    "  with open('/content/list.txt','w') as f:\n",
    "    for i in range(sample_num):\n",
    "      f.write('file /content/output/frame_%05d.jpg\\n'%i)\n",
    "    for j in range(int(freeze_secs*fps)):\n",
    "      f.write('file /content/output/frame_%05d.jpg\\n'%i)\n",
    "  !ffmpeg -r $fps -f concat -safe 0 -i /content/list.txt -c:v libx264 -pix_fmt yuv420p -profile:v baseline -movflags +faststart -r $fps $out -y\n",
    "  with open(file_name, 'rb') as f:\n",
    "    data_url = \"data:video/mp4;base64,\" + b64encode(f.read()).decode()\n",
    "  display(HTML(\"\"\"\n",
    "    <video controls autoplay loop>\n",
    "          <source src=\"%s\" type=\"video/mp4\">\n",
    "    </video>\"\"\" % data_url))\n",
    "\n",
    "  from google.colab import files, output\n",
    "  output.eval_js('new Audio(\"https://freesound.org/data/previews/80/80921_1022651-lq.ogg\").play()')\n",
    "  files.download(file_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}