Upload 9 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/garden_in.jpg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,187 @@
+from cgitb import enable
+from ctypes.wintypes import HFONT
+import os
+import sys
+import torch
+import gradio as gr
+import numpy as np
+import torchvision.transforms as transforms
+
+
+from torch.autograd import Variable
+from network.Transformer import Transformer
+from huggingface_hub import hf_hub_download
+
+from PIL import Image
+
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Constants
+
+MAX_DIMENSION = 1280
+MODEL_PATH = "models"
+COLOUR_MODEL = "RGB"
+
+STYLE_SHINKAI = "Makoto Shinkai"
+STYLE_HOSODA = "Mamoru Hosoda"
+STYLE_MIYAZAKI = "Hayao Miyazaki"
+STYLE_KON = "Satoshi Kon"
+DEFAULT_STYLE = STYLE_SHINKAI
+STYLE_CHOICE_LIST = [STYLE_SHINKAI, STYLE_HOSODA, STYLE_MIYAZAKI, STYLE_KON]
+
+MODEL_REPO_SHINKAI = "akiyamasho/AnimeBackgroundGAN-Shinkai"
+MODEL_FILE_SHINKAI = "shinkai_makoto.pth"
+
+MODEL_REPO_HOSODA = "akiyamasho/AnimeBackgroundGAN-Hosoda"
+MODEL_FILE_HOSODA = "hosoda_mamoru.pth"
+
+MODEL_REPO_MIYAZAKI = "akiyamasho/AnimeBackgroundGAN-Miyazaki"
+MODEL_FILE_MIYAZAKI = "miyazaki_hayao.pth"
+
+MODEL_REPO_KON = "akiyamasho/AnimeBackgroundGAN-Kon"
+MODEL_FILE_KON = "kon_satoshi.pth"
+
+# Model Initalisation
+shinkai_model_hfhub = hf_hub_download(repo_id=MODEL_REPO_SHINKAI, filename=MODEL_FILE_SHINKAI)
+hosoda_model_hfhub = hf_hub_download(repo_id=MODEL_REPO_HOSODA, filename=MODEL_FILE_HOSODA)
+miyazaki_model_hfhub = hf_hub_download(repo_id=MODEL_REPO_MIYAZAKI, filename=MODEL_FILE_MIYAZAKI)
+kon_model_hfhub = hf_hub_download(repo_id=MODEL_REPO_KON, filename=MODEL_FILE_KON)
+
+shinkai_model = Transformer()
+hosoda_model = Transformer()
+miyazaki_model = Transformer()
+kon_model = Transformer()
+
+enable_gpu = torch.cuda.is_available()
+
+if enable_gpu:
+    # If you have multiple cards,
+    # you can assign to a specific card, eg: "cuda:0"("cuda") or "cuda:1"
+    # Use the first card by default: "cuda"
+    device = torch.device("cuda")
+else:
+    device = "cpu"
+
+shinkai_model.load_state_dict(
+    torch.load(shinkai_model_hfhub, device)
+)
+hosoda_model.load_state_dict(
+    torch.load(hosoda_model_hfhub, device)
+)
+miyazaki_model.load_state_dict(
+    torch.load(miyazaki_model_hfhub, device)
+)
+kon_model.load_state_dict(
+    torch.load(kon_model_hfhub, device)
+)
+
+if enable_gpu:
+    shinkai_model = shinkai_model.to(device)
+    hosoda_model = hosoda_model.to(device)
+    miyazaki_model = miyazaki_model.to(device)
+    kon_model = kon_model.to(device)
+
+shinkai_model.eval()
+hosoda_model.eval()
+miyazaki_model.eval()
+kon_model.eval()
+
+
+# Functions
+
+def get_model(style):
+    if style == STYLE_SHINKAI:
+        return shinkai_model
+    elif style == STYLE_HOSODA:
+        return hosoda_model
+    elif style == STYLE_MIYAZAKI:
+        return miyazaki_model
+    elif style == STYLE_KON:
+        return kon_model
+    else:
+        logger.warning(
+            f"Style {style} not found. Defaulting to Makoto Shinkai"
+        )
+        return shinkai_model
+
+
+def adjust_image_for_model(img):
+    logger.info(f"Image Height: {img.height}, Image Width: {img.width}")
+    if img.height > MAX_DIMENSION or img.width > MAX_DIMENSION:
+        logger.info(f"Dimensions too large. Resizing to {MAX_DIMENSION}px.")
+        img.thumbnail((MAX_DIMENSION, MAX_DIMENSION), Image.ANTIALIAS)
+
+    return img
+
+
+def inference(img, style):
+    img = adjust_image_for_model(img)
+
+    # load image
+    input_image = img.convert(COLOUR_MODEL)
+    input_image = np.asarray(input_image)
+    # RGB -> BGR
+    input_image = input_image[:, :, [2, 1, 0]]
+    input_image = transforms.ToTensor()(input_image).unsqueeze(0)
+    # preprocess, (-1, 1)
+    input_image = -1 + 2 * input_image
+
+    if enable_gpu:
+        logger.info(f"CUDA found. Using GPU.")
+        # Allows to specify a card for calculation
+        input_image = Variable(input_image).to(device)
+    else:
+        logger.info(f"CUDA not found. Using CPU.")
+        input_image = Variable(input_image).float()
+
+    # forward
+    model = get_model(style)
+    output_image = model(input_image)
+    output_image = output_image[0]
+    # BGR -> RGB
+    output_image = output_image[[2, 1, 0], :, :]
+    output_image = output_image.data.cpu().float() * 0.5 + 0.5
+
+    return transforms.ToPILImage()(output_image)
+
+
+# Gradio setup
+
+title = "Anime Background GAN"
+description = "Gradio Demo for CartoonGAN by Chen Et. Al. Models are Shinkai Makoto, Hosoda Mamoru, Kon Satoshi, and Miyazaki Hayao."
+article = "<p style='text-align: center'><a href='http://openaccess.thecvf.com/content_cvpr_2018/CameraReady/2205.pdf' target='_blank'>CartoonGAN Whitepaper from Chen et.al</a></p><p style='text-align: center'><a href='https://github.com/venture-anime/cartoongan-pytorch' target='_blank'>Github Repo</a></p><p style='text-align: center'><a href='https://github.com/Yijunmaverick/CartoonGAN-Test-Pytorch-Torch' target='_blank'>Original Implementation from Yijunmaverick</a></p><center><img src='https://visitor-badge.glitch.me/badge?page_id=akiyamasho' alt='visitor badge'></center></p>"
+
+examples = [
+    ["examples/garden_in.jpg", STYLE_SHINKAI],
+    ["examples/library_in.jpg", STYLE_KON],
+]
+
+
+gr.Interface(
+    fn=inference,
+    inputs=[
+        gr.inputs.Image(
+            type="pil",
+            label="Input Photo (less than 1280px on both width and height)",
+        ),
+        gr.inputs.Dropdown(
+            STYLE_CHOICE_LIST,
+            type="value",
+            default=DEFAULT_STYLE,
+            label="Style",
+        ),
+    ],
+    outputs=gr.outputs.Image(
+        type="pil",
+        label="Output Image",
+    ),
+    title=title,
+    description=description,
+    article=article,
+    examples=examples,
+    allow_flagging="never",
+    allow_screenshot=False,
+).launch(enable_queue=True)

models/hosoda_mamoru.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:813504096c42ab7fa965c67cdbc24608400dd2c5a9ddaf8171d165d7344492d1
+size 133

models/kon_satoshi.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d5f9a0b193c1d7c019951a9886289a0536661d1ec3a2dcd98fcd213402bad28
+size 133

models/miyazaki_hayao.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c2aee56380168b266a7c747e0c26b6f939b7fdac41a8fd620d94450dad12061
+size 133

models/shinkai_makoto.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e662cf1194c6633f409dfbffcc8454118593f96719e92dc268b74d0a74892cd
+size 133

network/Transformer.py

@@ -0,0 +1,180 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class Transformer(nn.Module):
+    def __init__(self):
+        super(Transformer, self).__init__()
+        #
+        self.refpad01_1 = nn.ReflectionPad2d(3)
+        self.conv01_1 = nn.Conv2d(3, 64, 7)
+        self.in01_1 = InstanceNormalization(64)
+        # relu
+        self.conv02_1 = nn.Conv2d(64, 128, 3, 2, 1)
+        self.conv02_2 = nn.Conv2d(128, 128, 3, 1, 1)
+        self.in02_1 = InstanceNormalization(128)
+        # relu
+        self.conv03_1 = nn.Conv2d(128, 256, 3, 2, 1)
+        self.conv03_2 = nn.Conv2d(256, 256, 3, 1, 1)
+        self.in03_1 = InstanceNormalization(256)
+        # relu
+
+        ## res block 1
+        self.refpad04_1 = nn.ReflectionPad2d(1)
+        self.conv04_1 = nn.Conv2d(256, 256, 3)
+        self.in04_1 = InstanceNormalization(256)
+        # relu
+        self.refpad04_2 = nn.ReflectionPad2d(1)
+        self.conv04_2 = nn.Conv2d(256, 256, 3)
+        self.in04_2 = InstanceNormalization(256)
+        # + input
+
+        ## res block 2
+        self.refpad05_1 = nn.ReflectionPad2d(1)
+        self.conv05_1 = nn.Conv2d(256, 256, 3)
+        self.in05_1 = InstanceNormalization(256)
+        # relu
+        self.refpad05_2 = nn.ReflectionPad2d(1)
+        self.conv05_2 = nn.Conv2d(256, 256, 3)
+        self.in05_2 = InstanceNormalization(256)
+        # + input
+
+        ## res block 3
+        self.refpad06_1 = nn.ReflectionPad2d(1)
+        self.conv06_1 = nn.Conv2d(256, 256, 3)
+        self.in06_1 = InstanceNormalization(256)
+        # relu
+        self.refpad06_2 = nn.ReflectionPad2d(1)
+        self.conv06_2 = nn.Conv2d(256, 256, 3)
+        self.in06_2 = InstanceNormalization(256)
+        # + input
+
+        ## res block 4
+        self.refpad07_1 = nn.ReflectionPad2d(1)
+        self.conv07_1 = nn.Conv2d(256, 256, 3)
+        self.in07_1 = InstanceNormalization(256)
+        # relu
+        self.refpad07_2 = nn.ReflectionPad2d(1)
+        self.conv07_2 = nn.Conv2d(256, 256, 3)
+        self.in07_2 = InstanceNormalization(256)
+        # + input
+
+        ## res block 5
+        self.refpad08_1 = nn.ReflectionPad2d(1)
+        self.conv08_1 = nn.Conv2d(256, 256, 3)
+        self.in08_1 = InstanceNormalization(256)
+        # relu
+        self.refpad08_2 = nn.ReflectionPad2d(1)
+        self.conv08_2 = nn.Conv2d(256, 256, 3)
+        self.in08_2 = InstanceNormalization(256)
+        # + input
+
+        ## res block 6
+        self.refpad09_1 = nn.ReflectionPad2d(1)
+        self.conv09_1 = nn.Conv2d(256, 256, 3)
+        self.in09_1 = InstanceNormalization(256)
+        # relu
+        self.refpad09_2 = nn.ReflectionPad2d(1)
+        self.conv09_2 = nn.Conv2d(256, 256, 3)
+        self.in09_2 = InstanceNormalization(256)
+        # + input
+
+        ## res block 7
+        self.refpad10_1 = nn.ReflectionPad2d(1)
+        self.conv10_1 = nn.Conv2d(256, 256, 3)
+        self.in10_1 = InstanceNormalization(256)
+        # relu
+        self.refpad10_2 = nn.ReflectionPad2d(1)
+        self.conv10_2 = nn.Conv2d(256, 256, 3)
+        self.in10_2 = InstanceNormalization(256)
+        # + input
+
+        ## res block 8
+        self.refpad11_1 = nn.ReflectionPad2d(1)
+        self.conv11_1 = nn.Conv2d(256, 256, 3)
+        self.in11_1 = InstanceNormalization(256)
+        # relu
+        self.refpad11_2 = nn.ReflectionPad2d(1)
+        self.conv11_2 = nn.Conv2d(256, 256, 3)
+        self.in11_2 = InstanceNormalization(256)
+        # + input
+
+        ##------------------------------------##
+        self.deconv01_1 = nn.ConvTranspose2d(256, 128, 3, 2, 1, 1)
+        self.deconv01_2 = nn.Conv2d(128, 128, 3, 1, 1)
+        self.in12_1 = InstanceNormalization(128)
+        # relu
+        self.deconv02_1 = nn.ConvTranspose2d(128, 64, 3, 2, 1, 1)
+        self.deconv02_2 = nn.Conv2d(64, 64, 3, 1, 1)
+        self.in13_1 = InstanceNormalization(64)
+        # relu
+        self.refpad12_1 = nn.ReflectionPad2d(3)
+        self.deconv03_1 = nn.Conv2d(64, 3, 7)
+        # tanh
+
+    def forward(self, x):
+        y = F.relu(self.in01_1(self.conv01_1(self.refpad01_1(x))))
+        y = F.relu(self.in02_1(self.conv02_2(self.conv02_1(y))))
+        t04 = F.relu(self.in03_1(self.conv03_2(self.conv03_1(y))))
+
+        ##
+        y = F.relu(self.in04_1(self.conv04_1(self.refpad04_1(t04))))
+        t05 = self.in04_2(self.conv04_2(self.refpad04_2(y))) + t04
+
+        y = F.relu(self.in05_1(self.conv05_1(self.refpad05_1(t05))))
+        t06 = self.in05_2(self.conv05_2(self.refpad05_2(y))) + t05
+
+        y = F.relu(self.in06_1(self.conv06_1(self.refpad06_1(t06))))
+        t07 = self.in06_2(self.conv06_2(self.refpad06_2(y))) + t06
+
+        y = F.relu(self.in07_1(self.conv07_1(self.refpad07_1(t07))))
+        t08 = self.in07_2(self.conv07_2(self.refpad07_2(y))) + t07
+
+        y = F.relu(self.in08_1(self.conv08_1(self.refpad08_1(t08))))
+        t09 = self.in08_2(self.conv08_2(self.refpad08_2(y))) + t08
+
+        y = F.relu(self.in09_1(self.conv09_1(self.refpad09_1(t09))))
+        t10 = self.in09_2(self.conv09_2(self.refpad09_2(y))) + t09
+
+        y = F.relu(self.in10_1(self.conv10_1(self.refpad10_1(t10))))
+        t11 = self.in10_2(self.conv10_2(self.refpad10_2(y))) + t10
+
+        y = F.relu(self.in11_1(self.conv11_1(self.refpad11_1(t11))))
+        y = self.in11_2(self.conv11_2(self.refpad11_2(y))) + t11
+        ##
+
+        y = F.relu(self.in12_1(self.deconv01_2(self.deconv01_1(y))))
+        y = F.relu(self.in13_1(self.deconv02_2(self.deconv02_1(y))))
+        y = torch.tanh(self.deconv03_1(self.refpad12_1(y)))
+
+        return y
+
+
+class InstanceNormalization(nn.Module):
+    def __init__(self, dim, eps=1e-9):
+        super(InstanceNormalization, self).__init__()
+        self.scale = nn.Parameter(torch.FloatTensor(dim))
+        self.shift = nn.Parameter(torch.FloatTensor(dim))
+        self.eps = eps
+        self._reset_parameters()
+
+    def _reset_parameters(self):
+        self.scale.data.uniform_()
+        self.shift.data.zero_()
+
+    def __call__(self, x):
+        n = x.size(2) * x.size(3)
+        t = x.view(x.size(0), x.size(1), n)
+        mean = torch.mean(t, 2).unsqueeze(2).unsqueeze(3).expand_as(x)
+        # Calculate the biased var. torch.var returns unbiased var
+        var = torch.var(t, 2).unsqueeze(2).unsqueeze(3).expand_as(x) * (
+            (n - 1) / float(n)
+        )
+        scale_broadcast = self.scale.unsqueeze(1).unsqueeze(1).unsqueeze(0)
+        scale_broadcast = scale_broadcast.expand_as(x)
+        shift_broadcast = self.shift.unsqueeze(1).unsqueeze(1).unsqueeze(0)
+        shift_broadcast = shift_broadcast.expand_as(x)
+        out = (x - mean) / torch.sqrt(var + self.eps)
+        out = out * scale_broadcast + shift_broadcast
+        return out