app.py

import torch
from PIL import Image, ImageOps, ImageSequence
import numpy as np

import comfy.sample
import comfy.sd


def vencode(vae, pth):
    pilimg = pth
    pixels = np.array(pilimg).astype(np.float32) / 255.0
    pixels = torch.from_numpy(pixels)[None,]
    t = vae.encode(pixels[:,:,:,:3])
    return {"samples":t}
from pathlib import Path
if not Path("model.safetensors").exists():
    import requests
    with open("model.safetensors", "wb") as f:
        f.write(requests.get("https://huggingface.co/parsee-mizuhashi/mangaka/resolve/main/mangaka.safetensors?download=true").content)
MODEL_FILE = "model.safetensors"
with torch.no_grad():
    unet, clip, vae = comfy.sd.load_checkpoint_guess_config(MODEL_FILE, output_vae=True, output_clip=True)[:3]# :3
BASE_NEG = "(low-quality worst-quality:1.4 (bad-anatomy (inaccurate-limb:1.2 bad-composition inaccurate-eyes extra-digit fewer-digits (extra-arms:1.2)"
DEVICE = "cpu" if not torch.cuda.is_available() else "cuda"

def common_ksampler(model, seed, steps, cfg, sampler_name, scheduler, positive, negative, latent, denoise=1.0):

    noise_mask = None
    if "noise_mask" in latent:
        noise_mask = latent["noise_mask"]
    latnt = latent["samples"]
    noise = comfy.sample.prepare_noise(latnt, seed, None)
    disable_pbar = True
    samples = comfy.sample.sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latnt,
                                denoise=denoise, noise_mask=noise_mask, disable_pbar=disable_pbar, seed=seed)
    out = samples
    return out
def set_mask(samples, mask):
    s = samples.copy()
    s["noise_mask"] = mask.reshape((-1, 1, mask.shape[-2], mask.shape[-1]))
    return s
def load_image_mask(image):
    image_path = image
    i = Image.open(image_path)
    i = ImageOps.exif_transpose(i)
    if i.getbands() != ("R", "G", "B", "A"):
        if i.mode == 'I':
            i = i.point(lambda i: i * (1 / 255))
        i = i.convert("RGBA")
    mask = None
    c = "A"
    if c in i.getbands():
        mask = np.array(i.getchannel(c)).astype(np.float32) / 255.0
        mask = torch.from_numpy(mask)
    else:
        mask = torch.zeros((64,64), dtype=torch.float32, device="cpu")
    return mask.unsqueeze(0)
@torch.no_grad()
def main(img, variant, positive, negative, pilimg):
    variant = min(int(variant), limits[img])
    
    global unet, clip, vae 
    mask = load_image_mask(f"./mangaka-d/{img}/i{variant}.png")
    
    tkns = clip.tokenize("(greyscale monochrome black-and-white:1.3)" + positive)
    cond, c = clip.encode_from_tokens(tkns, return_pooled=True) 
    
    uncond_tkns = clip.tokenize(BASE_NEG + negative)
    uncond, uc = clip.encode_from_tokens(uncond_tkns, return_pooled=True)
    cn = [[cond, {"pooled_output": c}]]
    un = [[uncond, {"pooled_output": uc}]]

    latent = vencode(vae, pilimg)
    latent = set_mask(latent, mask)
    
    denoised = common_ksampler(unet, 0, 20, 7, 'ddpm', 'karras', cn, un, latent, denoise=1)
    decoded = vae.decode(denoised)
    i = 255. * decoded[0].cpu().numpy()
    img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8))
    return img

limits = {
    "1": 4,
    "2": 4,
    "3": 5,
    "4": 6,
    "5": 4,
    "6": 6,
    "7": 8,
    "8": 5,
    "9": 5,
    "s1": 4,
    "s2": 6,
    "s3": 5,
    "s4": 5,
    "s5": 4,
    "s6": 4
}
import gradio as gr
def visualize_fn(page, panel):
    base = f"./mangaka-d/{page}/base.png"
    base = Image.open(base)
    if panel == "none":
        return base
    panel = min(int(panel), limits[page])
    mask = f"./mangaka-d/{page}/i{panel}.png"
    base = base.convert("RGBA")
    mask = Image.open(mask)
    #remove all green and blue from the mask
    mask = mask.convert("RGBA")
    data = mask.getdata()
    data = [
    (255, 0, 0, 255) if pixel[:3] == (255, 255, 255) else pixel
    for pixel in mask.getdata()
    ]
    mask.putdata(data)
    #overlay the mask on the base
    base.paste(mask, (0,0), mask)
    return base
def reset_fn(page):
    base = f"./mangaka-d/{page}/base.png"
    base = Image.open(base)
    return base
with gr.Blocks() as demo:
    with gr.Tab("Mangaka"):
        with gr.Row():
            with gr.Column():
                positive = gr.Textbox(label="Positive prompt", lines=2)
                negative = gr.Textbox(label="Negative prompt")
                with gr.Accordion("Page Settings"):
                    with gr.Row():
                        with gr.Column():
                            page = gr.Dropdown(label="Page", choices=["1", "2", "3", "4", "5", "6", "7", "8", "9", "s1", "s2", "s3", "s4", "s5", "s6"], value="s1")
                            panel = gr.Dropdown(label="Panel", choices=["1", "2", "3", "4", "5", "6", "7", "8", "none"], value="1")
                            visualize = gr.Button("Visualize")
                        with gr.Column():
                            visualize_output = gr.Image(interactive=False)
                    visualize.click(visualize_fn, inputs=[page, panel], outputs=visualize_output)
            with gr.Column():
                with gr.Row():
                    with gr.Column():
                        generate = gr.Button("Generate", variant="primary")
                    with gr.Column():
                        reset = gr.Button("Reset", variant="stop")
                current_panel = gr.Image(interactive=False)
                reset.click(reset_fn, inputs=[page], outputs=current_panel)
                generate.click(main, inputs=[page, panel, positive, negative, current_panel], outputs=current_panel)

demo.launch()