#!/usr/bin/env python3 import io import json import logging import multiprocessing import os import random import time import imghdr from pathlib import Path from typing import Union from PIL import Image import cv2 import torch import numpy as np from loguru import logger from watchdog.events import FileSystemEventHandler from interactive_seg import InteractiveSeg, Click from make_gif import make_compare_gif from model_manager import ModelManager from schema import Config from file_manager import FileManager from ext import ImageWatermarkHandler try: torch._C._jit_override_can_fuse_on_cpu(False) torch._C._jit_override_can_fuse_on_gpu(False) torch._C._jit_set_texpr_fuser_enabled(False) torch._C._jit_set_nvfuser_enabled(False) except: pass from flask import ( Flask, request, send_file, cli, make_response, send_from_directory, jsonify, config, ) # Disable ability for Flask to display warning about using a development server in a production environment. # https://gist.github.com/jerblack/735b9953ba1ab6234abb43174210d356 cli.show_server_banner = lambda *_: None from flask_cors import CORS from helper import ( load_img, numpy_to_bytes, resize_max_size, pil_to_bytes, ) NUM_THREADS = str(multiprocessing.cpu_count()) # fix libomp problem on windows https://github.com/Sanster/lama-cleaner/issues/56 os.environ["KMP_DUPLICATE_LIB_OK"] = "True" os.environ["OMP_NUM_THREADS"] = NUM_THREADS os.environ["OPENBLAS_NUM_THREADS"] = NUM_THREADS os.environ["MKL_NUM_THREADS"] = NUM_THREADS os.environ["VECLIB_MAXIMUM_THREADS"] = NUM_THREADS os.environ["NUMEXPR_NUM_THREADS"] = NUM_THREADS if os.environ.get("CACHE_DIR"): os.environ["TORCH_HOME"] = os.environ["CACHE_DIR"] BUILD_DIR = os.environ.get("LAMA_CLEANER_BUILD_DIR", "app/build") class NoFlaskwebgui(logging.Filter): def filter(self, record): return "flaskwebgui-keep-server-alive" not in record.getMessage() logging.getLogger("werkzeug").addFilter(NoFlaskwebgui()) app = Flask(__name__, static_folder=os.path.join(BUILD_DIR, "static")) app.config["JSON_AS_ASCII"] = False CORS(app, expose_headers=["Content-Disposition"]) # variable model: ModelManager = None thumb: FileManager = None output_dir: str = None interactive_seg_model: InteractiveSeg = None device = None input_image_path: str = None is_disable_model_switch: bool = False is_enable_file_manager: bool = False is_enable_auto_saving: bool = False is_desktop: bool = False def get_image_ext(img_bytes): w = imghdr.what("", img_bytes) if w is None: w = "jpeg" return w def diffuser_callback(i, t, latents): pass # socketio.emit('diffusion_step', {'diffusion_step': step}) @app.route("/make_gif", methods=["POST"]) def make_gif(): input = request.files filename = request.form["filename"] origin_image_bytes = input["origin_img"].read() clean_image_bytes = input["clean_img"].read() origin_image, _ = load_img(origin_image_bytes) clean_image, _ = load_img(clean_image_bytes) gif_bytes = make_compare_gif( Image.fromarray(origin_image), Image.fromarray(clean_image) ) return send_file( io.BytesIO(gif_bytes), mimetype="image/gif", as_attachment=True, attachment_filename=filename, ) @app.route("/medias/") def medias(tab): if tab == "image": response = make_response(jsonify(thumb.media_names), 200) else: response = make_response(jsonify(thumb.output_media_names), 200) # response.last_modified = thumb.modified_time[tab] # response.cache_control.no_cache = True # response.cache_control.max_age = 0 # response.make_conditional(request) return response @app.route("/media//") def media_file(tab, filename): if tab == "image": return send_from_directory(thumb.root_directory, filename) return send_from_directory(thumb.output_dir, filename) @app.route("/media_thumbnail//") def media_thumbnail_file(tab, filename): args = request.args width = args.get("width") height = args.get("height") if width is None and height is None: width = 256 if width: width = int(float(width)) if height: height = int(float(height)) directory = thumb.root_directory if tab == "output": directory = thumb.output_dir thumb_filename, (width, height) = thumb.get_thumbnail( directory, filename, width, height ) thumb_filepath = f"{app.config['THUMBNAIL_MEDIA_THUMBNAIL_ROOT']}{thumb_filename}" response = make_response(send_file(thumb_filepath)) response.headers["X-Width"] = str(width) response.headers["X-Height"] = str(height) return response @app.route("/inpaint", methods=["POST"]) def process(): input = request.files # RGB origin_image_bytes = input["image"].read() image, alpha_channel, exif = load_img(origin_image_bytes, return_exif=True) mask, _ = load_img(input["mask"].read(), gray=True) mask = cv2.threshold(mask, 127, 255, cv2.THRESH_BINARY)[1] if image.shape[:2] != mask.shape[:2]: return ( f"Mask shape{mask.shape[:2]} not queal to Image shape{image.shape[:2]}", 400, ) original_shape = image.shape interpolation = cv2.INTER_CUBIC form = request.form size_limit: Union[int, str] = form.get("sizeLimit", "1080") if size_limit == "Original": size_limit = max(image.shape) else: size_limit = int(size_limit) if "paintByExampleImage" in input: paint_by_example_example_image, _ = load_img( input["paintByExampleImage"].read() ) paint_by_example_example_image = Image.fromarray(paint_by_example_example_image) else: paint_by_example_example_image = None config = Config( ldm_steps=form["ldmSteps"], ldm_sampler=form["ldmSampler"], hd_strategy=form["hdStrategy"], zits_wireframe=form["zitsWireframe"], hd_strategy_crop_margin=form["hdStrategyCropMargin"], hd_strategy_crop_trigger_size=form["hdStrategyCropTrigerSize"], hd_strategy_resize_limit=form["hdStrategyResizeLimit"], prompt=form["prompt"], negative_prompt=form["negativePrompt"], use_croper=form["useCroper"], croper_x=form["croperX"], croper_y=form["croperY"], croper_height=form["croperHeight"], croper_width=form["croperWidth"], sd_scale=form["sdScale"], sd_mask_blur=form["sdMaskBlur"], sd_strength=form["sdStrength"], sd_steps=form["sdSteps"], sd_guidance_scale=form["sdGuidanceScale"], sd_sampler=form["sdSampler"], sd_seed=form["sdSeed"], sd_match_histograms=form["sdMatchHistograms"], cv2_flag=form["cv2Flag"], cv2_radius=form["cv2Radius"], paint_by_example_steps=form["paintByExampleSteps"], paint_by_example_guidance_scale=form["paintByExampleGuidanceScale"], paint_by_example_mask_blur=form["paintByExampleMaskBlur"], paint_by_example_seed=form["paintByExampleSeed"], paint_by_example_match_histograms=form["paintByExampleMatchHistograms"], paint_by_example_example_image=paint_by_example_example_image, p2p_steps=form["p2pSteps"], p2p_image_guidance_scale=form["p2pImageGuidanceScale"], p2p_guidance_scale=form["p2pGuidanceScale"], ) if config.sd_seed == -1: config.sd_seed = random.randint(1, 999999999) if config.paint_by_example_seed == -1: config.paint_by_example_seed = random.randint(1, 999999999) logger.info(f"Origin image shape: {original_shape}") image = resize_max_size(image, size_limit=size_limit, interpolation=interpolation) logger.info(f"Resized image shape: {image.shape}") mask = resize_max_size(mask, size_limit=size_limit, interpolation=interpolation) start = time.time() try: res_np_img = model(image, mask, config) except RuntimeError as e: torch.cuda.empty_cache() if "CUDA out of memory. " in str(e): # NOTE: the string may change? return "CUDA out of memory", 500 else: logger.exception(e) return "Internal Server Error", 500 finally: logger.info(f"process time: {(time.time() - start) * 1000}ms") torch.cuda.empty_cache() res_np_img = cv2.cvtColor(res_np_img.astype(np.uint8), cv2.COLOR_BGR2RGB) if alpha_channel is not None: if alpha_channel.shape[:2] != res_np_img.shape[:2]: alpha_channel = cv2.resize( alpha_channel, dsize=(res_np_img.shape[1], res_np_img.shape[0]) ) res_np_img = np.concatenate( (res_np_img, alpha_channel[:, :, np.newaxis]), axis=-1 ) ext = get_image_ext(origin_image_bytes) if exif is not None: bytes_io = io.BytesIO(pil_to_bytes(Image.fromarray(res_np_img), ext, exif=exif)) else: bytes_io = io.BytesIO(pil_to_bytes(Image.fromarray(res_np_img), ext)) response = make_response( send_file( # io.BytesIO(numpy_to_bytes(res_np_img, ext)), bytes_io, mimetype=f"image/{ext}", ) ) response.headers["X-Seed"] = str(config.sd_seed) return response @app.route("/save_image", methods=["POST"]) def save_image(): """ 保存原始图片 :return: """ if output_dir is None: return "--output-dir is None", 500 input = request.files filename = request.form["filename"] origin_image_bytes = input["image"].read() # RGB image, _ = load_img(origin_image_bytes) # 根据深度depth判断 if image.shape[2] == 3: image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) elif image.shape[2] == 4: image = cv2.cvtColor(image, cv2.COLOR_RGBA2BGRA) print("image:{}".format(image)) cv2.imwrite(os.path.join(output_dir, filename), image) return "ok", 200 # sichaolong extend override @app.route("/mysave_image", methods=["POST"]) def strengthen_save_image(): if output_dir is None: return "--output-dir is None", 500 input = request.files # mask config block_size = int(request.form["blokSize"]) step_len = int(request.form["stepLen"]) temp_output_path = request.form["tempOutputPath"] threshold = int(request.form["threshold"]) image_name = request.form['imageName'] mask_image_name = request.form["maskImageName"] # img origin_image_bytes = input["image"].read() # RGB image, _ = load_img(origin_image_bytes) if image.shape[2] == 3: image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) elif image.shape[2] == 4: image = cv2.cvtColor(image, cv2.COLOR_RGBA2BGRA) cv2.imwrite(os.path.join(output_dir, image_name), image) # calculate mask image_path = os.path.join(output_dir, image_name) mask_image_path = os.path.join(output_dir, mask_image_name) iwh = ImageWatermarkHandler() list = iwh.index_watermark(img_path=image_path, block_size=block_size, step_len=step_len, threshold=threshold, temp_output_path=temp_output_path) # get and save mask image mask_image = iwh.get_mask(img_path=image_path, list=list, block_size=block_size, mask_img_path=mask_image_path) # return mask ext = get_image_ext(origin_image_bytes) mask_image_bytes_io = io.BytesIO(pil_to_bytes(Image.fromarray(mask_image), ext)) response = make_response( send_file( # io.BytesIO(numpy_to_bytes(res_np_img, ext)), mask_image_bytes_io, mimetype=f"image/{ext}" ) ) response.headers["X-Seed"] = str() return response # sichaolong extend override @app.route("/myinpaint", methods=["POST"]) def strengthen_process(): # request data form = request.form image_name = form['imageName'] mask_image_name = form["maskImageName"] # 1、首先访问/save_image上传图片,然后将图片字节读取出来 if image_name is None or mask_image_name is None: print("image or mask_image not exist!=>imageName:{},maskImageName:{}".format(image_name, mask_image_name)) return # RGB # image image_path = os.path.join(output_dir, image_name) image_bytes = open(image_path, 'rb').read() image, alpha_channel, exif = load_img(image_bytes, return_exif=True) # mask mask_image_path = os.path.join(output_dir, mask_image_name) mask_image_bytes = open(mask_image_path, 'rb').read() mask, _ = load_img(mask_image_bytes, gray=True) mask = cv2.threshold(mask, 127, 255, cv2.THRESH_BINARY)[1] if image.shape[:2] != mask.shape[:2]: return ( f"Mask shape{mask.shape[:2]} not queal to Image shape{image.shape[:2]}", 400, ) original_shape = image.shape interpolation = cv2.INTER_CUBIC form = request.form size_limit: Union[int, str] = form.get("sizeLimit", "1080") if size_limit == "Original": size_limit = max(image.shape) else: size_limit = int(size_limit) if "paintByExampleImage" in request.files: paint_by_example_example_image, _ = load_img( request.files["paintByExampleImage"].read() ) paint_by_example_example_image = Image.fromarray(paint_by_example_example_image) else: paint_by_example_example_image = None """ 支持模型: models = { "lama": LaMa, "ldm": LDM, "zits": ZITS, "mat": MAT, "fcf": FcF, ===》 FcF only support fixed size(512x512) image input. Lama Cleaner will take care of resize and crop process, it still recommended applies to small defects. "sd1.5": SD15, Anything4.name: Anything4, RealisticVision14.name: RealisticVision14, "cv2": OpenCV2, "manga": Manga, "sd2": SD2, "paint_by_example": PaintByExample, "instruct_pix2pix": InstructPix2Pix, } """ config = Config( # 支持的策略有:Crop、Origin、Resize。Crop masking area from the original image to do inpainting.对GPU友好 hd_strategy=form.get("hdStrategy", type=str, default='Crop'), # ldm 模型 ldm_steps=form.get("ldmSteps", type=str, default=25), hd_strategy_crop_margin=form.get("hdStrategyCropMargin", type=str, default=196), hd_strategy_crop_trigger_size=form.get("hdStrategyCropTrigerSize", type=str, default=800), hd_strategy_resize_limit=form.get("hdStrategyResizeLimit", type=str, default=2048), # # 是否应用裁剪 # use_croper=form.get("useCroper", type=bool, default=False), # # 裁剪参数 # croper_x=form.get("croperX", type=str, default=None), # croper_y=form.get("croperY", type=str, default=None), # croper_height=form.get("croperHeight", type=str, default=None), # croper_width=form.get("croperWidth", type=str, default=None), # # 支持的策略有plms、ddim # ldm_sampler=form.get("ldmSampler", type=str, default=None), # # zits 模型 # # 线框 # zits_wireframe=form.get("zitsWireframe", type=bool, default=True), # prompt=form.get("prompt", type=str, default=None), # negative_prompt=form.get("negativePrompt", type=str, default=None), # # sd 模型 # sd_scale=form.get("sdScale", type=str, default=None), # sd_mask_blur=form.get("sdMaskBlur", type=str, default=None), # sd_strength=form.get("sdStrength", type=str, default=None), # sd_steps=form.get("sdSteps", type=str, default=None), # sd_guidance_scale=form.get("sdGuidanceScale", type=str, default=None), # sd_sampler=form.get("sdSampler", type=str, default=None), # sd_seed=form.get("sdSeed", type=str, default=None), # sd_match_histograms=form.get("sdMatchHistograms", type=bool, default=False), # # cv2 模型 # cv2_flag=form.get("cv2Flag", type=str, default=None), # cv2_radius=form.get("cv2Radius", type=str, default=None), # # paint_by_example模型 # paint_by_example_steps=form.get("paintByExampleSteps", type=str, default=None), # paint_by_example_guidance_scale=form.get("paintByExampleGuidanceScale", type=str, default=None), # paint_by_example_mask_blur=form.get("paintByExampleMaskBlur", type=str, default=None), # paint_by_example_seed=form.get("paintByExampleSeed", type=str, default=None), # paint_by_example_match_histograms=form.get("paintByExampleMatchHistograms", type=bool, default=False), # paint_by_example_example_image=paint_by_example_example_image, # # instruct_pix2pix模型 # p2p_steps=form.get("p2pSteps", type=str, default=None), # p2p_image_guidance_scale=form.get("p2pImageGuidanceScale", type=str, default=None), # p2p_guidance_scale=form.get("p2pGuidanceScale", type=str, default=None), ) if config.sd_seed == -1: config.sd_seed = random.randint(1, 999999999) if config.paint_by_example_seed == -1: config.paint_by_example_seed = random.randint(1, 999999999) logger.info(f"Origin image shape: {original_shape}") image = resize_max_size(image, size_limit=size_limit, interpolation=interpolation) logger.info(f"Resized image shape: {image.shape}") mask = resize_max_size(mask, size_limit=size_limit, interpolation=interpolation) start = time.time() try: res_np_img = model(image, mask, config) except RuntimeError as e: torch.cuda.empty_cache() if "CUDA out of memory. " in str(e): # NOTE: the string may change? return "CUDA out of memory", 500 else: logger.exception(e) return "Internal Server Error", 500 finally: logger.info(f"process time: {(time.time() - start) * 1000}ms") torch.cuda.empty_cache() res_np_img = cv2.cvtColor(res_np_img.astype(np.uint8), cv2.COLOR_BGR2RGB) if alpha_channel is not None: if alpha_channel.shape[:2] != res_np_img.shape[:2]: alpha_channel = cv2.resize( alpha_channel, dsize=(res_np_img.shape[1], res_np_img.shape[0]) ) res_np_img = np.concatenate( (res_np_img, alpha_channel[:, :, np.newaxis]), axis=-1 ) ext = get_image_ext(image_bytes) if exif is not None: bytes_io = io.BytesIO(pil_to_bytes(Image.fromarray(res_np_img), ext, exif=exif)) else: bytes_io = io.BytesIO(pil_to_bytes(Image.fromarray(res_np_img), ext)) response = make_response( send_file( # io.BytesIO(numpy_to_bytes(res_np_img, ext)), bytes_io, mimetype=f"image/{ext}", ) ) response.headers["X-Seed"] = str(config.sd_seed) return response @app.route("/interactive_seg", methods=["POST"]) def interactive_seg(): input = request.files origin_image_bytes = input["image"].read() # RGB image, _ = load_img(origin_image_bytes) if "mask" in input: mask, _ = load_img(input["mask"].read(), gray=True) else: mask = None _clicks = json.loads(request.form["clicks"]) clicks = [] for i, click in enumerate(_clicks): clicks.append( Click(coords=(click[1], click[0]), indx=i, is_positive=click[2] == 1) ) start = time.time() new_mask = interactive_seg_model(image, clicks=clicks, prev_mask=mask) logger.info(f"interactive seg process time: {(time.time() - start) * 1000}ms") response = make_response( send_file( io.BytesIO(numpy_to_bytes(new_mask, "png")), mimetype=f"image/png", ) ) return response @app.route("/model") def current_model(): return model.name, 200 @app.route("/is_disable_model_switch") def get_is_disable_model_switch(): res = "true" if is_disable_model_switch else "false" return res, 200 @app.route("/is_enable_file_manager") def get_is_enable_file_manager(): res = "true" if is_enable_file_manager else "false" return res, 200 @app.route("/is_enable_auto_saving") def get_is_enable_auto_saving(): res = "true" if is_enable_auto_saving else "false" return res, 200 @app.route("/model_downloaded/") def model_downloaded(name): return str(model.is_downloaded(name)), 200 @app.route("/is_desktop") def get_is_desktop(): return str(is_desktop), 200 @app.route("/model", methods=["POST"]) def switch_model(): if is_disable_model_switch: return "Switch model is disabled", 400 new_name = request.form.get("name") if new_name == model.name: return "Same model", 200 try: model.switch(new_name) except NotImplementedError: return f"{new_name} not implemented", 403 return f"ok, switch to {new_name}", 200 @app.route("/") def index(): return send_file(os.path.join(BUILD_DIR, "index.html"), cache_timeout=0) @app.route("/inputimage") def set_input_photo(): if input_image_path: with open(input_image_path, "rb") as f: image_in_bytes = f.read() return send_file( input_image_path, as_attachment=True, attachment_filename=Path(input_image_path).name, mimetype=f"image/{get_image_ext(image_in_bytes)}", ) else: return "No Input Image" class FSHandler(FileSystemEventHandler): def on_modified(self, event): print("File modified: %s" % event.src_path) def main(args): global model global interactive_seg_model global device global input_image_path global is_disable_model_switch global is_enable_file_manager global is_desktop global thumb global output_dir global is_enable_auto_saving output_dir = args.output_dir if output_dir is not None: is_enable_auto_saving = True device = torch.device(args.device) is_disable_model_switch = args.disable_model_switch is_desktop = args.gui if is_disable_model_switch: logger.info( f"Start with --disable-model-switch, model switch on frontend is disable" ) if args.input and os.path.isdir(args.input): logger.info(f"Initialize file manager") thumb = FileManager(app) is_enable_file_manager = True app.config["THUMBNAIL_MEDIA_ROOT"] = args.input app.config["THUMBNAIL_MEDIA_THUMBNAIL_ROOT"] = os.path.join( args.output_dir, "lama_cleaner_thumbnails" ) thumb.output_dir = Path(args.output_dir) # thumb.start() # try: # while True: # time.sleep(1) # finally: # thumb.image_dir_observer.stop() # thumb.image_dir_observer.join() # thumb.output_dir_observer.stop() # thumb.output_dir_observer.join() else: input_image_path = args.input model = ModelManager( name=args.model, device=device, no_half=args.no_half, hf_access_token=args.hf_access_token, disable_nsfw=args.sd_disable_nsfw or args.disable_nsfw, sd_cpu_textencoder=args.sd_cpu_textencoder, sd_run_local=args.sd_run_local, local_files_only=args.local_files_only, cpu_offload=args.cpu_offload, enable_xformers=args.sd_enable_xformers or args.enable_xformers, callback=diffuser_callback, ) interactive_seg_model = InteractiveSeg() if args.gui: app_width, app_height = args.gui_size from flaskwebgui import FlaskUI ui = FlaskUI( app, width=app_width, height=app_height, host=args.host, port=args.port, close_server_on_exit=not args.no_gui_auto_close, ) ui.run() else: app.run(host=args.host, port=args.port, debug=args.debug)