app.py

import os
import gradio as gr
import argparse
import numpy as np
import torch
import einops
import copy
import math
import time
import random
import spaces
import re
import uuid

from gradio_imageslider import ImageSlider
from PIL import Image
from SUPIR.util import HWC3, upscale_image, fix_resize, convert_dtype, create_SUPIR_model, load_QF_ckpt
from huggingface_hub import hf_hub_download
from pillow_heif import register_heif_opener

register_heif_opener()

max_64_bit_int = np.iinfo(np.int32).max

hf_hub_download(repo_id="laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", filename="open_clip_pytorch_model.bin", local_dir="laion_CLIP-ViT-bigG-14-laion2B-39B-b160k")
hf_hub_download(repo_id="camenduru/SUPIR", filename="sd_xl_base_1.0_0.9vae.safetensors", local_dir="yushan777_SUPIR")
hf_hub_download(repo_id="camenduru/SUPIR", filename="SUPIR-v0F.ckpt", local_dir="yushan777_SUPIR")
hf_hub_download(repo_id="camenduru/SUPIR", filename="SUPIR-v0Q.ckpt", local_dir="yushan777_SUPIR")
hf_hub_download(repo_id="RunDiffusion/Juggernaut-XL-Lightning", filename="Juggernaut_RunDiffusionPhoto2_Lightning_4Steps.safetensors", local_dir="RunDiffusion_Juggernaut-XL-Lightning")

parser = argparse.ArgumentParser()
parser.add_argument("--opt", type=str, default='options/SUPIR_v0.yaml')
parser.add_argument("--ip", type=str, default='127.0.0.1')
parser.add_argument("--port", type=int, default='6688')
parser.add_argument("--no_llava", action='store_true', default=True)
parser.add_argument("--use_image_slider", action='store_true', default=False)
parser.add_argument("--log_history", action='store_true', default=False)
parser.add_argument("--loading_half_params", action='store_true', default=False)
parser.add_argument("--use_tile_vae", action='store_true', default=True)
parser.add_argument("--encoder_tile_size", type=int, default=512)
parser.add_argument("--decoder_tile_size", type=int, default=64)
parser.add_argument("--load_8bit_llava", action='store_true', default=False)
args = parser.parse_args()

if torch.cuda.device_count() > 0:
    SUPIR_device = 'cuda:0'

    # Load SUPIR
    model, default_setting = create_SUPIR_model(args.opt, SUPIR_sign='Q', load_default_setting=True)
    if args.loading_half_params:
        model = model.half()
    if args.use_tile_vae:
        model.init_tile_vae(encoder_tile_size=args.encoder_tile_size, decoder_tile_size=args.decoder_tile_size)
    model = model.to(SUPIR_device)
    model.first_stage_model.denoise_encoder_s1 = copy.deepcopy(model.first_stage_model.denoise_encoder)
    model.current_model = 'v0-Q'
    ckpt_Q, ckpt_F = load_QF_ckpt(args.opt)

def check_upload(input_image):
    if input_image is None:
        raise gr.Error("Please provide an image to restore.")
    return gr.update(visible=True)

def process_uploaded_image(image_path):
    image = Image.open(image_path)
    width, height = image.size
    max_dim = max(width, height)
    if max_dim > 1024:
        if width > height:
            new_width = 1024
            new_height = int((1024 / width) * height)
        else:
            new_height = 1024
            new_width = int((1024 / height) * width)
        image = image.resize((new_width, new_height), Image.ANTIALIAS)
        image.save(image_path)
    return image_path

def update_seed(is_randomize_seed, seed):
    if is_randomize_seed:
        return random.randint(0, max_64_bit_int)
    return seed

def reset():
    return [
        None,  # input_image
        "",  # prompt
        'Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore detailing, hyper sharpness, perfect without deformations.',  # a_prompt
        'painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, unsharp, weird textures, ugly, dirty, messy, worst quality, low quality, frames, watermark, signature, jpeg artifacts, deformed, lowres, over-smooth',  # n_prompt
        1,  # num_samples
        1024,  # min_size
        1,  # downscale
        2,  # upscale
        default_setting.edm_steps if torch.cuda.device_count() > 0 else 1,  # edm_steps
        -1.0,  # s_stage1
        1.0,  # s_stage2
        default_setting.s_cfg_Quality if torch.cuda.device_count() > 0 else 1.0,  # s_cfg
        random.randint(0, max_64_bit_int),  # seed
        5,  # s_churn
        1.003,  # s_noise
        'Wavelet',  # color_fix_type
        'fp32',  # diff_dtype
        'fp32',  # ae_dtype
        1.0,  # gamma_correction
        True,  # linear_CFG
        False,  # linear_s_stage2
        default_setting.spt_linear_CFG_Quality if torch.cuda.device_count() > 0 else 1.0,  # spt_linear_CFG
        0.0,  # spt_linear_s_stage2
        'v0-Q',  # model_select
        4  # allocation
    ]

def check(input_image):
    if input_image is None:
        raise gr.Error("Please provide an image to restore.")

def stage2_process(
    input_image,
    prompt,
    a_prompt,
    n_prompt,
    num_samples,
    min_size,
    downscale,
    upscale,
    edm_steps,
    s_stage1,
    s_stage2,
    s_cfg,
    seed,
    s_churn,
    s_noise,
    color_fix_type,
    diff_dtype,
    ae_dtype,
    gamma_correction,
    linear_CFG,
    linear_s_stage2,
    spt_linear_CFG,
    spt_linear_s_stage2,
    model_select,
    allocation
):
    try:
        return restore_in_Xmin(
            input_image,
            prompt,
            a_prompt,
            n_prompt,
            num_samples,
            min_size,
            downscale,
            upscale,
            edm_steps,
            s_stage1,
            s_stage2,
            s_cfg,
            seed,
            s_churn,
            s_noise,
            color_fix_type,
            diff_dtype,
            ae_dtype,
            gamma_correction,
            linear_CFG,
            linear_s_stage2,
            spt_linear_CFG,
            spt_linear_s_stage2,
            model_select,
            allocation
        )
    except Exception as e:
        print(f"Exception occurred: {str(e)}")
        raise e

def restore_in_Xmin(
    input_image_path,
    prompt,
    a_prompt,
    n_prompt,
    num_samples,
    min_size,
    downscale,
    upscale,
    edm_steps,
    s_stage1,
    s_stage2,
    s_cfg,
    seed,
    s_churn,
    s_noise,
    color_fix_type,
    diff_dtype,
    ae_dtype,
    gamma_correction,
    linear_CFG,
    linear_s_stage2,
    spt_linear_CFG,
    spt_linear_s_stage2,
    model_select,
    allocation
):
    print("Starting image restoration process...")
    input_format = re.sub(r"^.*\.([^\.]+)$", r"\1", input_image_path)

    if input_format.lower() not in ['png', 'webp', 'jpg', 'jpeg', 'gif', 'bmp', 'heic']:
        gr.Warning('Invalid image format. Please use a supported image format.')
        return None, None, None

    if prompt is None:
        prompt = ""

    if a_prompt is None:
        a_prompt = ""

    if n_prompt is None:
        n_prompt = ""

    if prompt != "" and a_prompt != "":
        a_prompt = prompt + ", " + a_prompt
    else:
        a_prompt = prompt + a_prompt
    print("Final prompt: " + str(a_prompt))

    denoise_image = np.array(Image.open(input_image_path))

    if downscale > 1:
        input_height, input_width, input_channel = denoise_image.shape
        denoise_image = np.array(Image.fromarray(denoise_image).resize((input_width // downscale, input_height // downscale), Image.LANCZOS))

    denoise_image = HWC3(denoise_image)

    if torch.cuda.device_count() == 0:
        gr.Warning('Set this space to GPU config to make it work.')
        return [input_image_path, denoise_image], gr.update(value="GPU not available."), gr.update(visible=True)

    if model_select != model.current_model:
        print('Loading model: ' + model_select)
        if model_select == 'v0-Q':
            model.load_state_dict(ckpt_Q, strict=False)
        elif model_select == 'v0-F':
            model.load_state_dict(ckpt_F, strict=False)
        model.current_model = model_select

    model.ae_dtype = convert_dtype(ae_dtype)
    model.model.dtype = convert_dtype(diff_dtype)

    # Allocation
    allocation_functions = {
        1: restore_in_1min,
        2: restore_in_2min,
        3: restore_in_3min,
        4: restore_in_4min,
        5: restore_in_5min,
        6: restore_in_6min,
        7: restore_in_7min,
        8: restore_in_8min,
        9: restore_in_9min,
        10: restore_in_10min,
    }

    restore_function = allocation_functions.get(allocation, restore_in_4min)
    return restore_function(
        input_image_path, prompt, a_prompt, n_prompt, num_samples, min_size, downscale, upscale,
        edm_steps, s_stage1, s_stage2, s_cfg, seed, s_churn, s_noise, color_fix_type,
        diff_dtype, ae_dtype, gamma_correction, linear_CFG, linear_s_stage2, spt_linear_CFG,
        spt_linear_s_stage2, model_select, allocation
    )

@spaces.GPU(duration=59)
def restore_in_1min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

@spaces.GPU(duration=119)
def restore_in_2min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

@spaces.GPU(duration=179)
def restore_in_3min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

@spaces.GPU(duration=239)
def restore_in_4min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

@spaces.GPU(duration=299)
def restore_in_5min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

@spaces.GPU(duration=359)
def restore_in_6min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

@spaces.GPU(duration=419)
def restore_in_7min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

@spaces.GPU(duration=479)
def restore_in_8min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

@spaces.GPU(duration=539)
def restore_in_9min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

@spaces.GPU(duration=599)
def restore_in_10min(*args, **kwargs):
    return restore_on_gpu(*args, **kwargs)

def restore_on_gpu(
    input_image_path,
    prompt,
    a_prompt,
    n_prompt,
    num_samples,
    min_size,
    downscale,
    upscale,
    edm_steps,
    s_stage1,
    s_stage2,
    s_cfg,
    seed,
    s_churn,
    s_noise,
    color_fix_type,
    diff_dtype,
    ae_dtype,
    gamma_correction,
    linear_CFG,
    linear_s_stage2,
    spt_linear_CFG,
    spt_linear_s_stage2,
    model_select,
    allocation
):
    start = time.time()
    print('Starting GPU restoration...')

    torch.cuda.set_device(SUPIR_device)

    with torch.no_grad():
        # Convert input image to NumPy array and ensure it has 3 channels
        input_image = HWC3(np.array(Image.open(input_image_path)))
        input_image = upscale_image(input_image, upscale, unit_resolution=32, min_size=min_size)
        LQ = input_image / 255.0
        LQ = np.power(LQ, gamma_correction)
        LQ *= 255.0
        LQ = LQ.round().clip(0, 255).astype(np.uint8)
        LQ = LQ / 255 * 2 - 1
        LQ = torch.tensor(LQ, dtype=torch.float32).permute(2, 0, 1).unsqueeze(0).to(SUPIR_device)[:, :3, :, :]
        captions = ['']

        samples = model.batchify_sample(
            LQ, captions, num_steps=edm_steps, restoration_scale=s_stage1, s_churn=s_churn,
            s_noise=s_noise, cfg_scale=s_cfg, control_scale=s_stage2, seed=seed,
            num_samples=num_samples, p_p=a_prompt, n_p=n_prompt, color_fix_type=color_fix_type,
            use_linear_CFG=linear_CFG, use_linear_control_scale=linear_s_stage2,
            cfg_scale_start=spt_linear_CFG, control_scale_start=spt_linear_s_stage2
        )

        x_samples = (einops.rearrange(samples, 'b c h w -> b h w c') * 127.5 + 127.5).cpu().numpy().round().clip(
            0, 255).astype(np.uint8)
        results = [x_samples[i] for i in range(num_samples)]
    torch.cuda.empty_cache()

    input_height, input_width, input_channel = input_image.shape
    result_height, result_width, result_channel = results[0].shape

    print('Restoration completed.')
    end = time.time()
    secondes = int(end - start)
    minutes = math.floor(secondes / 60)
    secondes = secondes - (minutes * 60)
    hours = math.floor(minutes / 60)
    minutes = minutes - (hours * 60)
    information = ("Start the process again if you want a different result. " if seed is not None else "") + \
        "The image has been enhanced successfully."

    # Save the result image to a temporary file for downloading
    result_image = Image.fromarray(results[0])
    result_image_path = f"result_{uuid.uuid4().hex}.png"
    result_image.save(result_image_path)

    # Update the result slider with the before and after images
    return [input_image_path, result_image_path], gr.update(value=information, visible=True), gr.update(visible=True)

def load_and_reset(param_setting):
    print('Resetting parameters...')
    if torch.cuda.device_count() == 0:
        gr.Warning('Set this space to GPU config to make it work.')
        return None, None, None, None, None, None, None, None, None, None, None, None, None
    edm_steps = default_setting.edm_steps
    s_stage2 = 1.0
    s_stage1 = -1.0
    s_churn = 5
    s_noise = 1.003
    a_prompt = 'Cinematic, High Contrast, highly detailed, taken using a Canon EOS R camera, hyper detailed photo - ' \
               'realistic maximum detail, 32k, Color Grading, ultra HD, extreme meticulous detailing, skin pore ' \
               'detailing, hyper sharpness, perfect without deformations.'
    n_prompt = 'painting, oil painting, illustration, drawing, art, sketch, anime, cartoon, CG Style, ' \
               '3D render, unreal engine, blurring, dirty, messy, worst quality, low quality, frames, watermark, ' \
               'signature, jpeg artifacts, deformed, lowres, over-smooth'
    color_fix_type = 'Wavelet'
    spt_linear_s_stage2 = 0.0
    linear_s_stage2 = False
    linear_CFG = True
    if param_setting == "Quality":
        s_cfg = default_setting.s_cfg_Quality
        spt_linear_CFG = default_setting.spt_linear_CFG_Quality
        model_select = "v0-Q"
    elif param_setting == "Fidelity":
        s_cfg = default_setting.s_cfg_Fidelity
        spt_linear_CFG = default_setting.spt_linear_CFG_Fidelity
        model_select = "v0-F"
    else:
        raise NotImplementedError
    gr.Info('The parameters are reset.')
    print('Parameters reset completed.')
    return edm_steps, s_cfg, s_stage2, s_stage1, s_churn, s_noise, a_prompt, n_prompt, color_fix_type, linear_CFG, \
        linear_s_stage2, spt_linear_CFG, spt_linear_s_stage2, model_select

def log_information(result_slider):
    print('Logging information...')
    if result_slider is not None:
        print(result_slider)

title_html = """
    <h1><center>Maree's Magical Photo Tool</center></h1>
    """

# Gradio interface
with gr.Blocks() as interface:
    if torch.cuda.device_count() == 0:
        with gr.Row():
            gr.HTML("""
    <p style="background-color: red;"><big><big><big><b>⚠️To use this tool, set a GPU with sufficient VRAM.</b></big></big></big></p>
    """)
    gr.HTML(title_html)

    input_image = gr.Image(label="Upload your photo", show_label=True, type="filepath", height=400, elem_id="image-input")
    with gr.Group():
        prompt = gr.Textbox(
            label="Describe your photo",
            info="Tell me about your photo so I can make it better.",
            value="",
            placeholder="Type a description...",
            lines=3
        )
        upscale = gr.Radio(
            [["x1", 1], ["x2", 2], ["x3", 3], ["x4", 4]],
            label="Upscale factor",
            info="Choose how much to enlarge the photo",
            value=2,
            interactive=True
        )
        allocation = gr.Radio(
            [["1 min", 1], ["2 min", 2], ["3 min", 3], ["4 min", 4], ["5 min", 5]],
            label="GPU allocation time (for Jon)",
            info="You can ignore this setting",
            value=4,
            interactive=True
        )

    gamma_correction = gr.Number(value=1.0, visible=False)  # Hidden component with default value 1.0

    with gr.Accordion("Advanced options", open=False):
        a_prompt = gr.Textbox(
            label="Additional image description",
            info="Completes the main image description",
            value='Cinematic, High Contrast, highly detailed, taken using a Canon EOS R '
                  'camera, hyper detailed photo - realistic maximum detail, 32k, Color '
                  'Grading, ultra HD, extreme meticulous detailing, skin pore detailing, '
                  'hyper sharpness, perfect without deformations.',
            lines=3
        )
        n_prompt = gr.Textbox(
            label="Negative image description",
            info="Disambiguate by listing what the image does NOT represent",
            value='painting, oil painting, illustration, drawing, art, sketch, anime, '
                  'cartoon, CG Style, 3D render, unreal engine, blurring, aliasing, unsharp, weird textures, ugly, dirty, messy, '
                  'worst quality, low quality, frames, watermark, signature, jpeg artifacts, '
                  'deformed, lowres, over-smooth',
            lines=3
        )
        edm_steps = gr.Slider(
            label="Steps",
            info="Lower=faster, higher=more details",
            minimum=1,
            maximum=200,
            value=default_setting.edm_steps if torch.cuda.device_count() > 0 else 1,
            step=1
        )
        num_samples = gr.Slider(
            label="Num Samples",
            info="Number of generated results",
            minimum=1,
            maximum=4 if not args.use_image_slider else 1,
            value=1,
            step=1
        )
        min_size = gr.Slider(
            label="Minimum size",
            info="Minimum height, minimum width of the result",
            minimum=32,
            maximum=4096,
            value=1024,
            step=32
        )
        downscale = gr.Radio(
            [["/1", 1], ["/2", 2], ["/3", 3], ["/4", 4]],
            label="Pre-downscale factor",
            info="Reducing blurred image reduces the process time",
            value=1,
            interactive=True
        )
        with gr.Row():
            with gr.Column():
                model_select = gr.Radio(
                    [["💃 Quality (v0-Q)", "v0-Q"], ["🎯 Fidelity (v0-F)", "v0-F"]],
                    label="Model Selection",
                    info="Pretrained model",
                    value="v0-Q",
                    interactive=True
                )
            with gr.Column():
                color_fix_type = gr.Radio(
                    [["None", "None"], ["AdaIn (improve as a photo)", "AdaIn"], ["Wavelet (for JPEG artifacts)", "Wavelet"]],
                    label="Color-Fix Type",
                    info="AdaIn=Improve following a style, Wavelet=For JPEG artifacts",
                    value="AdaIn",
                    interactive=True
                )
        s_cfg = gr.Slider(
            label="Text Guidance Scale",
            info="Lower=follow the image, higher=follow the prompt",
            minimum=1.0,
            maximum=15.0,
            value=default_setting.s_cfg_Quality if torch.cuda.device_count() > 0 else 1.0,
            step=0.1
        )
        s_stage2 = gr.Slider(
            label="Restoring Guidance Strength",
            minimum=0.,
            maximum=1.,
            value=1.,
            step=0.05
        )
        s_stage1 = gr.Slider(
            label="Pre-denoising Guidance Strength",
            minimum=-1.0,
            maximum=6.0,
            value=-1.0,
            step=1.0
        )
        s_churn = gr.Slider(
            label="S-Churn",
            minimum=0,
            maximum=40,
            value=5,
            step=1
        )
        s_noise = gr.Slider(
            label="S-Noise",
            minimum=1.0,
            maximum=1.1,
            value=1.003,
            step=0.001
        )
        with gr.Row():
            with gr.Column():
                linear_CFG = gr.Checkbox(label="Linear CFG", value=True)
                spt_linear_CFG = gr.Slider(
                    label="CFG Start",
                    minimum=1.0,
                    maximum=9.0,
                    value=default_setting.spt_linear_CFG_Quality if torch.cuda.device_count() > 0 else 1.0,
                    step=0.5
                )
            with gr.Column():
                linear_s_stage2 = gr.Checkbox(label="Linear Restoring Guidance", value=False)
                spt_linear_s_stage2 = gr.Slider(
                    label="Guidance Start",
                    minimum=0.,
                    maximum=1.,
                    value=0.,
                    step=0.05
                )
            with gr.Column():
                diff_dtype = gr.Radio(
                    [["fp32 (precision)", "fp32"], ["fp16 (medium)", "fp16"], ["bf16 (speed)", "bf16"]],
                    label="Diffusion Data Type",
                    value="fp32",
                    interactive=True
                )
            with gr.Column():
                ae_dtype = gr.Radio(
                    [["fp32 (precision)", "fp32"], ["bf16 (speed)", "bf16"]],
                    label="Auto-Encoder Data Type",
                    value="fp32",
                    interactive=True
                )
        randomize_seed = gr.Checkbox(
            label="\U0001F3B2 Randomize seed",
            value=True,
            info="If checked, result is always different"
        )
        seed = gr.Slider(
            label="Seed",
            minimum=0,
            maximum=max_64_bit_int,
            step=1,
            randomize=True
        )
        with gr.Group():
            param_setting = gr.Radio(
                ["Quality", "Fidelity"],
                interactive=True,
                label="Presetting",
                value="Quality"
            )
            restart_button = gr.Button(value="Apply presetting")

    with gr.Column():
        diffusion_button = gr.Button(
            value="🚀 Enhance Photo",
            variant="primary",
            elem_id="process_button"
        )
        reset_btn = gr.Button(
            value="🧹 Reset",
            variant="stop",
            elem_id="reset_button",
            visible=False
        )

        restore_information = gr.HTML(
            value="Start the process again if you want a different result.",
            visible=False
        )
        result_slider = ImageSlider(
            label='Result',
            show_label=False,
            interactive=False,
            elem_id="slider1",
            show_download_button=True  # Enable the download button
        )

    input_image.upload(
        fn=process_uploaded_image,
        inputs=input_image,
        outputs=input_image,
        queue=False
    )

    input_image.upload(
        fn=check_upload,
        inputs=input_image,
        outputs=[],
        queue=False,
        show_progress=False
    )

    diffusion_button.click(
        fn=update_seed,
        inputs=[randomize_seed, seed],
        outputs=[seed],
        queue=False,
        show_progress=False
    ).then(
        fn=check,
        inputs=[input_image],
        outputs=[],
        queue=False,
        show_progress=False
    ).success(
        fn=stage2_process,
        inputs=[
            input_image,
            prompt,
            a_prompt,
            n_prompt,
            num_samples,
            min_size,
            downscale,
            upscale,
            edm_steps,
            s_stage1,
            s_stage2,
            s_cfg,
            seed,
            s_churn,
            s_noise,
            color_fix_type,
            diff_dtype,
            ae_dtype,
            gamma_correction,  # Use the hidden gamma_correction component
            linear_CFG,
            linear_s_stage2,
            spt_linear_CFG,
            spt_linear_s_stage2,
            model_select,
            allocation
        ],
        outputs=[
            result_slider,
            restore_information,
            reset_btn
        ]
    ).success(
        fn=log_information,
        inputs=[result_slider],
        outputs=[],
        queue=False,
        show_progress=False
    )

    restart_button.click(
        fn=load_and_reset,
        inputs=[param_setting],
        outputs=[
            edm_steps,
            s_cfg,
            s_stage2,
            s_stage1,
            s_churn,
            s_noise,
            a_prompt,
            n_prompt,
            color_fix_type,
            linear_CFG,
            linear_s_stage2,
            spt_linear_CFG,
            spt_linear_s_stage2,
            model_select
        ]
    )

    reset_btn.click(
        fn=reset,
        inputs=[],
        outputs=[
            input_image,
            prompt,
            a_prompt,
            n_prompt,
            num_samples,
            min_size,
            downscale,
            upscale,
            edm_steps,
            s_stage1,
            s_stage2,
            s_cfg,
            seed,
            s_churn,
            s_noise,
            color_fix_type,
            diff_dtype,
            ae_dtype,
            gamma_correction,  # Use the hidden gamma_correction component
            linear_CFG,
            linear_s_stage2,
            spt_linear_CFG,
            spt_linear_s_stage2,
            model_select,
            allocation
        ],
        queue=False,
        show_progress=False
    )

    interface.queue(10).launch()