Add app

README.md

@@ -1,12 +1,22 @@
----
-title: Line Art Colorization
-emoji: 🐠
-colorFrom: purple
-colorTo: red
-sdk: gradio
-sdk_version: 3.17.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Line Art Colorization
+This project is a minimalistic implementation of AlacGan and it is based on the paper called User-Guided Deep Anime Line Art Colorization with Conditional Adversarial Networks (https://arxiv.org/pdf/1808.03240.pdf) as well as its github repository.
+
+## Colab example to play with the model (just run!)
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1jInaIELLo-Y1M8MnIgA-7aXSWRYlHnC9?usp=sharing)
+
+## Differences from the original implementation
+1. Less variants of line thickness (as it did not make the model performance significantly worse)
+2. Different images in the dataset
+3. No local features network added
+4. All image pairs are acquired via the xdog algorithm whereas in the paper, real line art images were also used to train the model
+
+Because of these differences, the results are slightly worse but the model was trained significantly faster and the process of collecting data did not take too long.
+
+## Model weights
+https://download938.mediafire.com/nd1xp1xdgitg/aig8n36f4vrne6t/gen_373000.pth
+
+## Colorization examples
+All these images were colorized by the alacgan neural network
+
+![Results of colorization](https://i.imgur.com/qngw4BI.png)
+

data_utils.py

@@ -0,0 +1,280 @@
+# +
+import os
+import math
+import random
+import numbers
+import requests
+import shutil
+import numpy as np
+import scipy.stats as stats
+from PIL import Image
+from tqdm.auto import tqdm
+
+from xdog import to_sketch
+# -
+
+import torch
+import torch.nn as nn
+import torch.utils.data as data
+from torch.utils.data.sampler import Sampler
+
+from torchvision import transforms
+from torchvision.transforms import Resize, CenterCrop
+
+mu, sigma = 1, 0.005
+X = stats.truncnorm((0 - mu) / sigma, (1 - mu) / sigma, loc=mu, scale=sigma)
+
+denormalize = transforms.Compose([ transforms.Normalize(mean = [ 0., 0., 0. ],
+                                                        std = [ 1/0.5, 1/0.5, 1/0.5 ]),
+                                  transforms.Normalize(mean = [ -0.5, -0.5, -0.5 ],
+                                                       std = [ 1., 1., 1. ]),])
+
+etrans = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.5), (0.5))
+])
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+def predict_img(gen, sk, hnt = None):
+    #sk = Image.open(sketch_path).convert('L')
+    sk = etrans(sk)
+
+    pad_w = 16 - sk.shape[1] % 16 if sk.shape[1] % 16 != 0 else 0
+    pad_h = 16 - sk.shape[2] % 16 if sk.shape[2] % 16 != 0 else 0
+    pad = nn.ZeroPad2d((pad_h, 0, pad_w, 0))
+    sk = pad(sk)
+
+    sk = sk.unsqueeze(0)
+    sk = sk.to(device)
+
+    if hnt == None:
+        hnt = torch.zeros((1, 4, sk.shape[2]//4, sk.shape[3]//4))
+
+    hnt = hnt.to(device)
+
+    img_gen = gen(sk, hnt, sketch_feat=None).squeeze(0)
+    img_gen = denormalize(img_gen) * 255
+    img_gen = img_gen.permute(1,2,0).detach().cpu().numpy().astype(np.uint8)
+    #return img_gen[pad_w:, pad_h:]
+    return Image.fromarray(img_gen[pad_w:, pad_h:])
+
+def files(img_path, img_size=512):
+    img_path = os.path.abspath(img_path)
+    line_widths = sorted([el for el in os.listdir(os.path.join(img_path, 'pics_sketch')) if el != '.ipynb_checkpoints'])
+    images_names = sorted([el for el in os.listdir(os.path.join(img_path, 'pics_sketch', line_widths[0])) if '.jpg' in el])
+    
+    images_names = [el for el in images_names if np.all(np.array(Image.open(os.path.join(img_path, 'pics', el)).size) >= np.array([img_size, img_size]))]
+    
+    images_color = [os.path.join(img_path, 'pics', el) for el in images_names]
+    images_sketch = {line_width:[os.path.join(img_path, 'pics_sketch', line_width, el) for el in images_names] for line_width in line_widths}
+    return images_color, images_sketch
+
+def mask_gen(img_size=512, bs=4):
+    maskS = img_size // 4
+
+    mask1 = torch.cat([torch.rand(1, 1, maskS, maskS).ge(X.rvs(1)[0]).float() for _ in range(bs // 2)], 0)
+    mask2 = torch.cat([torch.zeros(1, 1, maskS, maskS).float() for _ in range(bs // 2)], 0)
+    mask = torch.cat([mask1, mask2], 0)
+    return mask
+
+def jitter(x):
+    ran = random.uniform(0.7, 1)
+    return x * ran + 1 - ran
+
+def make_trans(img_size):
+    vtrans = transforms.Compose([
+            RandomSizedCrop(img_size // 4, Image.BICUBIC),
+            transforms.ToTensor(),
+            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+    ])
+
+    ctrans = transforms.Compose([
+            transforms.Resize(img_size, Image.BICUBIC),
+            transforms.ToTensor(),
+            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+    ])
+
+    strans = transforms.Compose([
+            transforms.Resize(img_size, Image.BICUBIC),
+            transforms.ToTensor(),
+            transforms.Lambda(jitter),
+            transforms.Normalize((0.5), (0.5))
+    ])
+    
+    return vtrans, ctrans, strans
+
+class RandomCrop(object):
+    """Crops the given PIL.Image at a random location to have a region of
+    the given size. size can be a tuple (target_height, target_width)
+    or an integer, in which case the target will be of a square shape (size, size)
+    """
+
+    def __init__(self, size):
+        if isinstance(size, numbers.Number):
+            self.size = (int(size), int(size))
+        else:
+            self.size = size
+
+    def __call__(self, img1, img2):
+        w, h = img1.size
+        th, tw = self.size
+        if w == tw and h == th:  # ValueError: empty range for randrange() (0,0, 0)
+            return img1, img2
+
+        if w == tw:
+            x1 = 0
+            y1 = random.randint(0, h - th)
+            return img1.crop((x1, y1, x1 + tw, y1 + th)), img2.crop((x1, y1, x1 + tw, y1 + th))
+
+        elif h == th:
+            x1 = random.randint(0, w - tw)
+            y1 = 0
+            return img1.crop((x1, y1, x1 + tw, y1 + th)), img2.crop((x1, y1, x1 + tw, y1 + th))
+
+        else:
+            x1 = random.randint(0, w - tw)
+            y1 = random.randint(0, h - th)
+            return img1.crop((x1, y1, x1 + tw, y1 + th)), img2.crop((x1, y1, x1 + tw, y1 + th))
+
+class RandomSizedCrop(object):
+    """Random crop the given PIL.Image to a random size of (0.08 to 1.0) of the original size
+    and and a random aspect ratio of 3/4 to 4/3 of the original aspect ratio
+    This is popularly used to train the Inception networks
+    size: size of the smaller edge
+    interpolation: Default: PIL.Image.BILINEAR
+    """
+
+    def __init__(self, size, interpolation=Image.BICUBIC):
+        self.size = size
+        self.interpolation = interpolation
+
+    def __call__(self, img):
+        for attempt in range(10):
+            area = img.size[0] * img.size[1]
+            target_area = random.uniform(0.9, 1.) * area
+            aspect_ratio = random.uniform(7. / 8, 8. / 7)
+
+            w = int(round(math.sqrt(target_area * aspect_ratio)))
+            h = int(round(math.sqrt(target_area / aspect_ratio)))
+
+            if random.random() < 0.5:
+                w, h = h, w
+
+            if w <= img.size[0] and h <= img.size[1]:
+                x1 = random.randint(0, img.size[0] - w)
+                y1 = random.randint(0, img.size[1] - h)
+
+                img = img.crop((x1, y1, x1 + w, y1 + h))
+                assert (img.size == (w, h))
+
+                return img.resize((self.size, self.size), self.interpolation)
+
+        # Fallback
+        Resize = Resize(self.size, interpolation=self.interpolation)
+        crop = CenterCrop(self.size)
+        return crop(Resize(img))
+
+
+class ImageFolder(data.Dataset):
+    def __init__(self, img_path, img_size):
+        
+        self.images_color, self.images_sketch = files(img_path, img_size)
+        if (any([self.images_sketch[key] == 0 for key in self.images_sketch])) or (len(self.images_color) == 0):
+            raise (RuntimeError("Found 0 images in one of the folders."))
+        if any([len(self.images_sketch[key]) != len(self.images_color) for key in self.images_sketch]):
+            raise (RuntimeError("The number of sketches is not equal to the number of colorized images."))
+        self.img_path = img_path
+        self.img_size = img_size
+        self.vtrans, self.ctrans, self.strans = make_trans(img_size)
+
+    def __getitem__(self, index):
+        color = Image.open(self.images_color[index]).convert('RGB')
+        
+        random_line_width = random.choice(list(self.images_sketch.keys()))
+        sketch = Image.open(self.images_sketch[random_line_width][index]).convert('L')
+        #the image can be smaller than img_size, fix!
+        color, sketch = RandomCrop(self.img_size)(color, sketch)
+        if random.random() < 0.5:
+            color, sketch = color.transpose(Image.FLIP_LEFT_RIGHT), sketch.transpose(Image.FLIP_LEFT_RIGHT)
+            
+        color, color_down, sketch = self.ctrans(color), self.vtrans(color), self.strans(sketch)
+                
+        return color, color_down, sketch
+
+    def __len__(self):
+        return len(self.images_color)
+
+
+class GivenIterationSampler(Sampler):
+    def __init__(self, dataset, total_iter, batch_size, diter, last_iter=-1):
+        self.dataset = dataset
+        self.total_iter = total_iter
+        self.batch_size = batch_size
+        self.diter = diter
+        self.last_iter = last_iter
+
+        self.total_size = self.total_iter * self.batch_size * (self.diter + 1)
+
+        self.indices = self.gen_new_list()
+        self.call = 0
+        
+
+    def __iter__(self):
+        #if self.call == 0:
+            #self.call = 1
+        return iter(self.indices[(self.last_iter + 1) * self.batch_size * (self.diter + 1):])
+        #else:
+        #    raise RuntimeError("this sampler is not designed to be called more than once!!")
+
+    def gen_new_list(self):
+        # each process shuffle all list with same seed
+        np.random.seed(0)
+
+        indices = np.arange(len(self.dataset))
+        indices = indices[:self.total_size]
+        num_repeat = (self.total_size - 1) // indices.shape[0] + 1
+        indices = np.tile(indices, num_repeat)
+        indices = indices[:self.total_size]
+
+        np.random.shuffle(indices)
+        assert len(indices) == self.total_size
+        return indices
+
+    def __len__(self):
+        # note here we do not take last iter into consideration, since __len__
+        # should only be used for displaying, the correct remaining size is
+        # handled by dataloader
+        # return self.total_size - (self.last_iter+1)*self.batch_size
+        return self.total_size
+
+def get_dataloader(img_path, img_size=512, seed=0, total_iter=250000, bs=4, diters=1, last_iter=-1):
+    
+    random.seed(seed)
+    
+    train_dataset = ImageFolder(img_path=img_path, img_size=img_size)
+
+    train_sampler = GivenIterationSampler(train_dataset, total_iter, bs, diters, last_iter=last_iter)
+
+    return data.DataLoader(train_dataset, batch_size=bs, shuffle=False, pin_memory=True, num_workers=4, sampler=train_sampler)
+
+
+def get_data(links, img_path='alacgan_data', line_widths=[0.3, 0.5]):
+    c = 0
+    
+    for line_width in line_widths:
+        lw = str(line_width)
+        if lw not in os.listdir(os.path.join(img_path, 'pics_sketch')):
+            os.mkdir(os.path.join(img_path, 'pics_sketch', lw))
+        else:
+            shutil.rmtree(os.path.join(img_path, 'pics_sketch', lw))
+            os.mkdir(os.path.join(img_path, 'pics_sketch', lw))
+            
+    for link in tqdm(links):
+        img_orig = Image.open(requests.get(link, stream=True).raw).convert('RGB')
+        img_orig.save(os.path.join(img_path, 'pics', str(c) + '.jpg'), 'JPEG')
+        for line_width in line_widths:
+            sketch_test = to_sketch(img_orig, sigma=line_width, k=5, gamma=0.96, epsilon=-1, phi=10e15, area_min=2)
+            sketch_test.save(os.path.join(img_path, 'pics_sketch', str(line_width), str(c) + '.jpg'), 'JPEG')
+            
+        c += 1

gradio_app.py

@@ -0,0 +1,50 @@
+import gradio as gr
+from xdog import to_sketch
+from model import Generator, ResNeXtBottleneck
+import torch
+from data_utils import *
+import glob
+gen = torch.load('model/model.pth')
+
+def convert_to_lineart(img, sigma, k, gamma, epsilon, phi, area_min):
+    phi = 10 * phi
+    out = to_sketch(img, sigma=sigma, k=k, gamma=gamma, epsilon=epsilon, phi=phi, area_min=area_min)
+    return out
+
+def inference(sk):
+    return predict_img(gen, sk, hnt = None)
+
+title = "To Line Art"
+description = "Line art colorization showcase. "
+article = "Github Repo"
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        with gr.Column():
+            image = gr.Image(type="pil", value='examples/Genshin-Impact-anime.jpg')
+            to_lineart_button = gr.Button("To Lineart")
+
+            gr.Examples(
+                examples=glob.glob('examples/*.jpg'),
+                inputs=image,
+                outputs=image,
+                fn=None,
+                cache_examples=False,
+            )
+
+        with gr.Column():
+            sigma = gr.Slider(0.1, 0.5, value=0.3, step=0.1, label='σ')
+            k = gr.Slider(1.0, 8.0, value=4.5, step=0.5, label='k')
+            gamma = gr.Slider(0.05, 1.0, value=0.95, step=0.05, label='γ')
+            epsilon = gr.Slider(-2, 2, value=-1, step=0.5, label='ε')
+            phi = gr.Slider(10, 20, label = 'φ', value=15)
+            min_area = gr.Slider(1, 5, value=2, step=1, label='Minimal Area')
+        
+        with gr.Column():
+            lineart = gr.Image(type="pil", image_mode='L')
+            inpaint_button = gr.Button("Inpaint")
+
+    to_lineart_button.click(convert_to_lineart, inputs=[image, sigma, k, gamma, epsilon, phi, min_area], outputs=lineart)
+    inpaint_button.click(inference, inputs=lineart, outputs=lineart)
+
+demo.launch()

model.py

@@ -0,0 +1,192 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+class ResNeXtBottleneck(nn.Module):
+    def __init__(self, in_channels=256, out_channels=256, stride=1, cardinality=32, dilate=1):
+        super(ResNeXtBottleneck, self).__init__()
+        
+        D = out_channels // 2
+        self.out_channels = out_channels
+        self.conv_reduce = nn.Conv2d(in_channels, D, kernel_size=1, stride=1, padding=0, bias=False)
+        self.conv_conv = nn.Conv2d(D, D, kernel_size=2 + stride, stride=stride, padding=dilate, dilation=dilate, groups=cardinality, bias=False)
+        self.conv_expand = nn.Conv2d(D, out_channels, kernel_size=1, stride=1, padding=0, bias=False)
+        self.shortcut = nn.Sequential()
+        
+        if stride != 1:
+            self.shortcut.add_module('shortcut', nn.AvgPool2d(2, stride=2))
+            
+    def forward(self, x):
+        bottleneck = self.conv_reduce.forward(x)
+        bottleneck = F.leaky_relu(bottleneck, 0.2, True)
+        bottleneck = self.conv_conv.forward(bottleneck)
+        bottleneck = F.leaky_relu(bottleneck, 0.2, True)
+        bottleneck = self.conv_expand.forward(bottleneck)
+        x = self.shortcut.forward(x)
+        
+        return x + bottleneck
+    
+    
+class Generator(nn.Module):
+    def __init__(self, ngf=64, feat=True):
+        super(Generator, self).__init__()
+        self.feat = feat
+        if feat:
+            add_channels = 512
+        else:
+            add_channels = 0
+        
+        self.toH = self._block(4, ngf, kernel_size=7, stride=1, padding=3)
+        self.to0 = self._block(1, ngf // 2, kernel_size=3, stride=1, padding=1)
+        self.to1 = self._block(ngf // 2, ngf, kernel_size=4, stride=2, padding=1)
+        self.to2 = self._block(ngf, ngf * 2, kernel_size=4, stride=2, padding=1)
+        self.to3 = self._block(ngf * 3, ngf * 4, kernel_size=4, stride=2, padding=1)
+        self.to4 = self._block(ngf * 4, ngf * 8, kernel_size=4, stride=2, padding=1)
+        
+        tunnel4 = nn.Sequential(*[ResNeXtBottleneck(ngf * 8, ngf * 8, cardinality=32, dilate=1) for _ in range(20)])
+        
+        self.tunnel4 = nn.Sequential(self._block(ngf * 8 + add_channels, ngf * 8, kernel_size=3, stride=1, padding=1),
+                                     tunnel4,
+                                     nn.Conv2d(ngf * 8, ngf * 16, kernel_size = 3, stride=1, padding=1),
+                                     nn.PixelShuffle(2),
+                                     nn.LeakyReLU(0.2, True))
+        
+        depth = 2
+        
+        tunnel = [ResNeXtBottleneck(ngf * 4, ngf * 4, cardinality=32, dilate=1) for _ in range(depth)]
+        tunnel += [ResNeXtBottleneck(ngf * 4, ngf * 4, cardinality=32, dilate=2) for _ in range(depth)]
+        tunnel += [ResNeXtBottleneck(ngf * 4, ngf * 4, cardinality=32, dilate=4) for _ in range(depth)]
+        tunnel += [ResNeXtBottleneck(ngf * 4, ngf * 4, cardinality=32, dilate=2),
+                   ResNeXtBottleneck(ngf * 4, ngf * 4, cardinality=32, dilate=1)]
+        tunnel3 = nn.Sequential(*tunnel)
+        
+        self.tunnel3 = nn.Sequential(self._block(ngf * 8, ngf * 4, kernel_size=3, stride=1, padding=1),
+                                     tunnel3,
+                                     nn.Conv2d(ngf * 4, ngf * 8, kernel_size=3, stride=1, padding=1),
+                                     nn.PixelShuffle(2), 
+                                     nn.LeakyReLU(0.2, True))
+        
+        tunnel = [ResNeXtBottleneck(ngf * 2, ngf * 2, cardinality=32, dilate=1) for _ in range(depth)]
+        tunnel += [ResNeXtBottleneck(ngf * 2, ngf * 2, cardinality=32, dilate=2) for _ in range(depth)]
+        tunnel += [ResNeXtBottleneck(ngf * 2, ngf * 2, cardinality=32, dilate=4) for _ in range(depth)]
+        tunnel += [ResNeXtBottleneck(ngf * 2, ngf * 2, cardinality=32, dilate=2),
+                   ResNeXtBottleneck(ngf * 2, ngf * 2, cardinality=32, dilate=1)]
+        tunnel2 = nn.Sequential(*tunnel)
+        
+        self.tunnel2 = nn.Sequential(self._block(ngf * 4, ngf * 2, kernel_size=3, stride=1, padding=1),
+                                     tunnel2,
+                                     nn.Conv2d(ngf * 2, ngf * 4, kernel_size=3, stride=1, padding=1),
+                                     nn.PixelShuffle(2), 
+                                     nn.LeakyReLU(0.2, True))
+        
+        tunnel = [ResNeXtBottleneck(ngf, ngf, cardinality=16, dilate=1)]
+        tunnel += [ResNeXtBottleneck(ngf, ngf, cardinality=16, dilate=2)]
+        tunnel += [ResNeXtBottleneck(ngf, ngf, cardinality=16, dilate=4)]
+        tunnel += [ResNeXtBottleneck(ngf, ngf, cardinality=16, dilate=2),
+                   ResNeXtBottleneck(ngf, ngf, cardinality=16, dilate=1)]
+        tunnel1 = nn.Sequential(*tunnel)
+        
+        self.tunnel1 = nn.Sequential(self._block(ngf * 2, ngf, kernel_size=3, stride=1, padding=1),
+                                     tunnel1,
+                                     nn.Conv2d(ngf, ngf * 2, kernel_size=3, stride=1, padding=1),
+                                     nn.PixelShuffle(2), 
+                                     nn.LeakyReLU(0.2, True))
+        
+        self.exit = nn.Conv2d(ngf, 3, kernel_size=3, stride=1, padding=1)
+        
+    def _block(self, in_channels, out_channels, kernel_size, stride, padding):
+        return nn.Sequential(
+            nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias=False),
+            nn.LeakyReLU(0.2, True)
+        )
+    
+    
+    def forward(self, sketch, hint, sketch_feat):
+        hint = self.toH(hint)
+        
+        x0 = self.to0(sketch)
+        x1 = self.to1(x0)
+        x2 = self.to2(x1)
+        x3 = self.to3(torch.cat([x2, hint], 1))
+        x4 = self.to4(x3)
+        
+        if self.feat:
+            x = self.tunnel4(torch.cat([x4, sketch_feat], 1))
+            x = self.tunnel3(torch.cat([x, x3], 1))
+            x = self.tunnel2(torch.cat([x, x2], 1))
+            x = self.tunnel1(torch.cat([x, x1], 1))
+            x = torch.tanh(self.exit(torch.cat([x, x0], 1)))
+        else:
+            x = self.tunnel4(x4)
+            x = self.tunnel3(torch.cat([x, x3], 1))
+            x = self.tunnel2(torch.cat([x, x2], 1))
+            x = self.tunnel1(torch.cat([x, x1], 1))
+            x = torch.tanh(self.exit(torch.cat([x, x0], 1)))
+        return x
+    
+
+class Discriminator(nn.Module):
+    def __init__(self, ndf=64, feat=True):
+        super(Discriminator, self).__init__()
+        self.feat = feat
+        
+        if feat:
+            add_channels = ndf * 8
+            ks = 4
+        else:
+            add_channels = 0
+            ks = 3
+            
+        self.feed = nn.Sequential(
+            self._block(3, ndf, kernel_size=7, stride=1, padding=1),
+            self._block(ndf, ndf, kernel_size=4, stride=2, padding=1),
+            
+            ResNeXtBottleneck(ndf, ndf, cardinality=8, dilate=1),
+            ResNeXtBottleneck(ndf, ndf, cardinality=8, dilate=1, stride=2),
+            self._block(ndf, ndf * 2, kernel_size=1, stride=1, padding=0),
+            
+            ResNeXtBottleneck(ndf * 2, ndf * 2, cardinality=8, dilate=1),
+            ResNeXtBottleneck(ndf * 2, ndf * 2, cardinality=8, dilate=1, stride=2),
+            self._block(ndf * 2, ndf * 4, kernel_size=1, stride=1, padding=0),
+            
+            ResNeXtBottleneck(ndf * 4, ndf * 4, cardinality=8, dilate=1),
+            ResNeXtBottleneck(ndf * 4, ndf * 4, cardinality=8, dilate=1, stride=2)
+        )
+        
+        self.feed2 = nn.Sequential(
+            self._block(ndf * 4 + add_channels, ndf * 8, kernel_size=3, stride=1, padding=1),
+            
+            ResNeXtBottleneck(ndf * 8, ndf * 8, cardinality=8, dilate=1),
+            ResNeXtBottleneck(ndf * 8, ndf * 8, cardinality=8, dilate=1, stride=2),
+            ResNeXtBottleneck(ndf * 8, ndf * 8, cardinality=8, dilate=1),
+            ResNeXtBottleneck(ndf * 8, ndf * 8, cardinality=8, dilate=1, stride=2),
+            ResNeXtBottleneck(ndf * 8, ndf * 8, cardinality=8, dilate=1),
+            ResNeXtBottleneck(ndf * 8, ndf * 8, cardinality=8, dilate=1, stride=2),
+            ResNeXtBottleneck(ndf * 8, ndf * 8, cardinality=8, dilate=1),
+            
+            self._block(ndf * 8, ndf * 8, kernel_size=ks, stride=1, padding=0),
+        )
+        
+        self.out = nn.Linear(512, 1)
+        
+    def _block(self, in_channels, out_channels, kernel_size, stride, padding):
+        return nn.Sequential(
+            nn.Conv2d(in_channels,
+                      out_channels,
+                      kernel_size,
+                      stride,
+                      padding,
+                      bias=False),
+            nn.LeakyReLU(0.2, True)
+        )
+    
+    def forward(self, color, sketch_feat=None):
+        x = self.feed(color)
+        
+        if self.feat:
+            x = self.feed2(torch.cat([x, sketch_feat], 1))
+        else:
+            x = self.feed2(x)
+        
+        out = self.out(x.view(color.size(0), -1))
+        return out

model/model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08d8483543bbc2fcae2d86a89422dc7be566c239ba99b6b889a7dc29a289a01c
+size 78392739

requirements.txt

@@ -0,0 +1,3 @@
+torch
+torchvision
+opencv

xdog.py

@@ -0,0 +1,66 @@
+import cv2
+import numpy as np
+from PIL import Image
+import requests
+
+# Difference of Gaussians applied to img input
+def dog(img,size=(0,0),k=1.6,sigma=0.5,gamma=1):
+	img1 = cv2.GaussianBlur(img,size,sigma)
+	img2 = cv2.GaussianBlur(img,size,sigma*k)
+	return (img1-gamma*img2)
+
+# Threshold the dog image, with dog(sigma,k) > 0 ? 1(255):0(0)
+def edge_dog(img,sigma=0.5,k=200,gamma=0.98):
+	aux = dog(img,sigma=sigma,k=k,gamma=0.98)
+	for i in range(0,aux.shape[0]):
+		for j in range(0,aux.shape[1]):
+			if(aux[i,j] > 0):
+				aux[i,j] = 255
+			else:
+				aux[i,j] = 0
+	return aux
+
+# garygrossi xdog version
+def xdog_garygrossi(img,sigma=0.5,k=200, gamma=0.98,epsilon=0.1,phi=10):
+	aux = dog(img,sigma=sigma,k=k,gamma=gamma)/255
+	for i in range(0,aux.shape[0]):
+		for j in range(0,aux.shape[1]):
+			if(aux[i,j] >= epsilon):
+				aux[i,j] = 1
+			else:
+				ht = np.tanh(phi*(aux[i][j] - epsilon))
+				aux[i][j] = 1 + ht
+	return aux*255
+
+def hatchBlend(image):
+	xdogImage = xdog(image,sigma=1,k=200, gamma=0.5,epsilon=-0.5,phi=10)
+	hatchTexture = cv2.imread('./imgs/hatch.jpg', cv2.CV_LOAD_IMAGE_GRAYSCALE)
+	hatchTexture = cv2.resize(hatchTexture,(image.shape[1],image.shape[0]))
+	alpha = 0.120
+	return (1-alpha)*xdogImage + alpha*hatchTexture
+
+# version of xdog inspired by article
+def xdog(img,sigma=0.5,k=1.6, gamma=1,epsilon=1,phi=1):
+	aux = dog(img,sigma=sigma,k=k,gamma=gamma)/255
+	for i in range(0,aux.shape[0]):
+		for j in range(0,aux.shape[1]):
+			if(aux[i,j] < epsilon):
+				aux[i,j] = 1*255
+			else:
+				aux[i,j] = 255*(1 + np.tanh(phi*(aux[i,j])))
+	return aux
+
+def to_sketch(img_orig, sigma=0.3, k=4.5, gamma=0.95, epsilon=-1, phi=10e15, area_min=2):
+    img_cnts = []
+    img = cv2.cvtColor(np.array(img_orig), cv2.COLOR_RGB2GRAY)
+    img_xdog = xdog(img, sigma=sigma, k=k, gamma=gamma, epsilon=epsilon, phi=phi).astype(np.uint8)
+    new_img = np.zeros_like(img_xdog)
+    thresh = cv2.threshold(img_xdog, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)[1]
+    cnts = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
+    cnts = cnts[0] if len(cnts) == 2 else cnts[1]
+    for c in cnts:
+        area = cv2.contourArea(c)
+        if area > area_min:
+            img_cnts.append(c)
+
+    return Image.fromarray(255 - cv2.drawContours(new_img, img_cnts, -1, (255,255,255), -1))