c1a010c78919f8efef07b18490f90b54ce38605a218a48907aa972358dd0fbc1

extensions/adetailer/controlnet_ext/restore.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+
+from modules import img2img, processing, shared
+
+
+def cn_restore_unet_hook(p, cn_latest_network):
+    if cn_latest_network is not None:
+        unet = p.sd_model.model.diffusion_model
+        cn_latest_network.restore(unet)
+
+
+class CNHijackRestore:
+    def __init__(self):
+        self.process = hasattr(processing, "__controlnet_original_process_images_inner")
+        self.img2img = hasattr(img2img, "__controlnet_original_process_batch")
+
+    def __enter__(self):
+        if self.process:
+            self.orig_process = processing.process_images_inner
+            processing.process_images_inner = getattr(
+                processing, "__controlnet_original_process_images_inner"
+            )
+        if self.img2img:
+            self.orig_img2img = img2img.process_batch
+            img2img.process_batch = getattr(
+                img2img, "__controlnet_original_process_batch"
+            )
+
+    def __exit__(self, *args, **kwargs):
+        if self.process:
+            processing.process_images_inner = self.orig_process
+        if self.img2img:
+            img2img.process_batch = self.orig_img2img
+
+
+@contextmanager
+def cn_allow_script_control():
+    orig = False
+    if "control_net_allow_script_control" in shared.opts.data:
+        try:
+            orig = shared.opts.data["control_net_allow_script_control"]
+            shared.opts.data["control_net_allow_script_control"] = True
+            yield
+        finally:
+            shared.opts.data["control_net_allow_script_control"] = orig
+    else:
+        yield

extensions/adetailer/install.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import importlib.util
+import subprocess
+import sys
+from importlib.metadata import version  # python >= 3.8
+
+from packaging.version import parse
+
+
+def is_installed(
+    package: str, min_version: str | None = None, max_version: str | None = None
+):
+    try:
+        spec = importlib.util.find_spec(package)
+    except ModuleNotFoundError:
+        return False
+
+    if spec is None:
+        return False
+
+    if not min_version and not max_version:
+        return True
+
+    if not min_version:
+        min_version = "0.0.0"
+    if not max_version:
+        max_version = "99999999.99999999.99999999"
+
+    if package == "google.protobuf":
+        package = "protobuf"
+
+    try:
+        pkg_version = version(package)
+        return parse(min_version) <= parse(pkg_version) <= parse(max_version)
+    except Exception:
+        return False
+
+
+def run_pip(*args):
+    subprocess.run([sys.executable, "-m", "pip", "install", *args])
+
+
+def install():
+    deps = [
+        # requirements
+        ("ultralytics", "8.0.97", None),
+        ("mediapipe", "0.10.0", None),
+        ("huggingface_hub", None, None),
+        ("pydantic", None, None),
+        # mediapipe
+        ("protobuf", "3.20.0", "3.20.9999"),
+    ]
+
+    for pkg, low, high in deps:
+        # https://github.com/protocolbuffers/protobuf/tree/main/python
+        name = "google.protobuf" if pkg == "protobuf" else pkg
+
+        if not is_installed(name, low, high):
+            if low and high:
+                cmd = f"{pkg}>={low},<={high}"
+            elif low:
+                cmd = f"{pkg}>={low}"
+            elif high:
+                cmd = f"{pkg}<={high}"
+            else:
+                cmd = pkg
+
+            run_pip("-U", cmd)
+
+
+try:
+    import launch
+
+    skip_install = launch.args.skip_install
+except Exception:
+    skip_install = False
+
+if not skip_install:
+    install()

extensions/adetailer/preload.py

@@ -0,0 +1,9 @@
+import argparse
+
+
+def preload(parser: argparse.ArgumentParser):
+    parser.add_argument(
+        "--ad-no-huggingface",
+        action="store_true",
+        help="Don't use adetailer models from huggingface",
+    )

extensions/adetailer/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "adetailer"
+description = "An object detection and auto-mask extension for stable diffusion webui."
+authors = [
+    {name = "dowon", email = "ks2515@naver.com"},
+]
+requires-python = ">=3.8,<3.12"
+readme = "README.md"
+license = {text = "AGPL-3.0"}
+
+[project.urls]
+repository = "https://github.com/Bing-su/adetailer"
+
+[tool.isort]
+profile = "black"
+known_first_party = ["launch", "modules"]
+
+[tool.ruff]
+select = ["A", "B", "C4", "E", "F", "I001", "ISC", "N", "PIE", "PT", "RET", "SIM", "UP", "W"]
+ignore = ["B008", "B905", "E501", "F401", "UP007"]
+
+[tool.ruff.isort]
+known-first-party = ["launch", "modules"]
+
+[tool.ruff.per-file-ignores]
+"sd_webui/*.py" = ["B027", "F403"]

extensions/adetailer/scripts/!adetailer.py

@@ -0,0 +1,634 @@
+from __future__ import annotations
+
+import os
+import platform
+import re
+import sys
+import traceback
+from contextlib import contextmanager, suppress
+from copy import copy, deepcopy
+from pathlib import Path
+from textwrap import dedent
+from typing import Any
+
+import gradio as gr
+import torch
+
+import modules  # noqa: F401
+from adetailer import (
+    AFTER_DETAILER,
+    __version__,
+    get_models,
+    mediapipe_predict,
+    ultralytics_predict,
+)
+from adetailer.args import ALL_ARGS, BBOX_SORTBY, ADetailerArgs, EnableChecker
+from adetailer.common import PredictOutput
+from adetailer.mask import filter_by_ratio, mask_preprocess, sort_bboxes
+from adetailer.ui import adui, ordinal, suffix
+from controlnet_ext import ControlNetExt, controlnet_exists
+from controlnet_ext.restore import (
+    CNHijackRestore,
+    cn_allow_script_control,
+    cn_restore_unet_hook,
+)
+from sd_webui import images, safe, script_callbacks, scripts, shared
+from sd_webui.paths import data_path, models_path
+from sd_webui.processing import (
+    StableDiffusionProcessingImg2Img,
+    create_infotext,
+    process_images,
+)
+from sd_webui.shared import cmd_opts, opts, state
+
+with suppress(ImportError):
+    from rich import print
+
+
+no_huggingface = getattr(cmd_opts, "ad_no_huggingface", False)
+adetailer_dir = Path(models_path, "adetailer")
+model_mapping = get_models(adetailer_dir, huggingface=not no_huggingface)
+txt2img_submit_button = img2img_submit_button = None
+SCRIPT_DEFAULT = "dynamic_prompting,dynamic_thresholding,wildcard_recursive,wildcards"
+
+if (
+    not adetailer_dir.exists()
+    and adetailer_dir.parent.exists()
+    and os.access(adetailer_dir.parent, os.W_OK)
+):
+    adetailer_dir.mkdir()
+
+print(
+    f"[-] ADetailer initialized. version: {__version__}, num models: {len(model_mapping)}"
+)
+
+
+@contextmanager
+def change_torch_load():
+    orig = torch.load
+    try:
+        torch.load = safe.unsafe_torch_load
+        yield
+    finally:
+        torch.load = orig
+
+
+@contextmanager
+def pause_total_tqdm():
+    orig = opts.data.get("multiple_tqdm", True)
+    try:
+        opts.data["multiple_tqdm"] = False
+        yield
+    finally:
+        opts.data["multiple_tqdm"] = orig
+
+
+class AfterDetailerScript(scripts.Script):
+    def __init__(self):
+        super().__init__()
+        self.ultralytics_device = self.get_ultralytics_device()
+
+        self.controlnet_ext = None
+        self.cn_script = None
+        self.cn_latest_network = None
+
+    def title(self):
+        return AFTER_DETAILER
+
+    def show(self, is_img2img):
+        return scripts.AlwaysVisible
+
+    def ui(self, is_img2img):
+        num_models = opts.data.get("ad_max_models", 2)
+        model_list = list(model_mapping.keys())
+
+        components, infotext_fields = adui(
+            num_models,
+            is_img2img,
+            model_list,
+            txt2img_submit_button,
+            img2img_submit_button,
+        )
+
+        self.infotext_fields = infotext_fields
+        return components
+
+    def init_controlnet_ext(self) -> None:
+        if self.controlnet_ext is not None:
+            return
+        self.controlnet_ext = ControlNetExt()
+
+        if controlnet_exists:
+            try:
+                self.controlnet_ext.init_controlnet()
+            except ImportError:
+                error = traceback.format_exc()
+                print(
+                    f"[-] ADetailer: ControlNetExt init failed:\n{error}",
+                    file=sys.stderr,
+                )
+
+    def update_controlnet_args(self, p, args: ADetailerArgs) -> None:
+        if self.controlnet_ext is None:
+            self.init_controlnet_ext()
+
+        if (
+            self.controlnet_ext is not None
+            and self.controlnet_ext.cn_available
+            and args.ad_controlnet_model != "None"
+        ):
+            self.controlnet_ext.update_scripts_args(
+                p,
+                model=args.ad_controlnet_model,
+                weight=args.ad_controlnet_weight,
+                guidance_start=args.ad_controlnet_guidance_start,
+                guidance_end=args.ad_controlnet_guidance_end,
+            )
+
+    def is_ad_enabled(self, *args_) -> bool:
+        if len(args_) == 0 or (len(args_) == 1 and isinstance(args_[0], bool)):
+            message = f"""
+                       [-] ADetailer: Not enough arguments passed to ADetailer.
+                           input: {args_!r}
+                       """
+            raise ValueError(dedent(message))
+        a0 = args_[0]
+        a1 = args_[1] if len(args_) > 1 else None
+        checker = EnableChecker(a0=a0, a1=a1)
+        return checker.is_enabled()
+
+    def get_args(self, *args_) -> list[ADetailerArgs]:
+        """
+        `args_` is at least 1 in length by `is_ad_enabled` immediately above
+        """
+        args = [arg for arg in args_ if isinstance(arg, dict)]
+
+        if not args:
+            message = f"[-] ADetailer: Invalid arguments passed to ADetailer: {args_!r}"
+            raise ValueError(message)
+
+        all_inputs = []
+
+        for n, arg_dict in enumerate(args, 1):
+            try:
+                inp = ADetailerArgs(**arg_dict)
+            except ValueError as e:
+                msgs = [
+                    f"[-] ADetailer: ValidationError when validating {ordinal(n)} arguments: {e}\n"
+                ]
+                for attr in ALL_ARGS.attrs:
+                    arg = arg_dict.get(attr)
+                    dtype = type(arg)
+                    arg = "DEFAULT" if arg is None else repr(arg)
+                    msgs.append(f"    {attr}: {arg} ({dtype})")
+                raise ValueError("\n".join(msgs)) from e
+
+            all_inputs.append(inp)
+
+        return all_inputs
+
+    def extra_params(self, arg_list: list[ADetailerArgs]) -> dict:
+        params = {}
+        for n, args in enumerate(arg_list):
+            params.update(args.extra_params(suffix=suffix(n)))
+        params["ADetailer version"] = __version__
+        return params
+
+    @staticmethod
+    def get_ultralytics_device() -> str:
+        '`device = ""` means autodetect'
+        device = ""
+        if platform.system() == "Darwin":
+            return device
+
+        if any(getattr(cmd_opts, vram, False) for vram in ["lowvram", "medvram"]):
+            device = "cpu"
+
+        return device
+
+    def prompt_blank_replacement(
+        self, all_prompts: list[str], i: int, default: str
+    ) -> str:
+        if not all_prompts:
+            return default
+        if i < len(all_prompts):
+            return all_prompts[i]
+        j = i % len(all_prompts)
+        return all_prompts[j]
+
+    def _get_prompt(
+        self, ad_prompt: str, all_prompts: list[str], i: int, default: str
+    ) -> list[str]:
+        prompts = re.split(r"\s*\[SEP\]\s*", ad_prompt)
+        blank_replacement = self.prompt_blank_replacement(all_prompts, i, default)
+        for n in range(len(prompts)):
+            if not prompts[n]:
+                prompts[n] = blank_replacement
+        return prompts
+
+    def get_prompt(self, p, args: ADetailerArgs) -> tuple[list[str], list[str]]:
+        i = p._idx
+
+        prompt = self._get_prompt(args.ad_prompt, p.all_prompts, i, p.prompt)
+        negative_prompt = self._get_prompt(
+            args.ad_negative_prompt, p.all_negative_prompts, i, p.negative_prompt
+        )
+
+        return prompt, negative_prompt
+
+    def get_seed(self, p) -> tuple[int, int]:
+        i = p._idx
+
+        if not p.all_seeds:
+            seed = p.seed
+        elif i < len(p.all_seeds):
+            seed = p.all_seeds[i]
+        else:
+            j = i % len(p.all_seeds)
+            seed = p.all_seeds[j]
+
+        if not p.all_subseeds:
+            subseed = p.subseed
+        elif i < len(p.all_subseeds):
+            subseed = p.all_subseeds[i]
+        else:
+            j = i % len(p.all_subseeds)
+            subseed = p.all_subseeds[j]
+
+        return seed, subseed
+
+    def get_width_height(self, p, args: ADetailerArgs) -> tuple[int, int]:
+        if args.ad_use_inpaint_width_height:
+            width = args.ad_inpaint_width
+            height = args.ad_inpaint_height
+        else:
+            width = p.width
+            height = p.height
+
+        return width, height
+
+    def get_steps(self, p, args: ADetailerArgs) -> int:
+        if args.ad_use_steps:
+            return args.ad_steps
+        return p.steps
+
+    def get_cfg_scale(self, p, args: ADetailerArgs) -> float:
+        if args.ad_use_cfg_scale:
+            return args.ad_cfg_scale
+        return p.cfg_scale
+
+    def infotext(self, p) -> str:
+        return create_infotext(
+            p, p.all_prompts, p.all_seeds, p.all_subseeds, None, 0, 0
+        )
+
+    def write_params_txt(self, p) -> None:
+        infotext = self.infotext(p)
+        params_txt = Path(data_path, "params.txt")
+        params_txt.write_text(infotext, encoding="utf-8")
+
+    def script_filter(self, p, args: ADetailerArgs):
+        script_runner = copy(p.scripts)
+        script_args = deepcopy(p.script_args)
+        self.disable_controlnet_units(script_args)
+
+        ad_only_seleted_scripts = opts.data.get("ad_only_seleted_scripts", True)
+        if not ad_only_seleted_scripts:
+            return script_runner, script_args
+
+        ad_script_names = opts.data.get("ad_script_names", SCRIPT_DEFAULT)
+        script_names_set = {
+            name
+            for script_name in ad_script_names.split(",")
+            for name in (script_name, script_name.strip())
+        }
+
+        if args.ad_controlnet_model != "None":
+            script_names_set.add("controlnet")
+
+        filtered_alwayson = []
+        for script_object in script_runner.alwayson_scripts:
+            filepath = script_object.filename
+            filename = Path(filepath).stem
+            if filename in script_names_set:
+                filtered_alwayson.append(script_object)
+            if filename == "controlnet":
+                self.cn_script = script_object
+                self.cn_latest_network = script_object.latest_network
+
+        script_runner.alwayson_scripts = filtered_alwayson
+        return script_runner, script_args
+
+    def disable_controlnet_units(self, script_args: list[Any]) -> None:
+        for obj in script_args:
+            if "controlnet" in obj.__class__.__name__.lower():
+                if hasattr(obj, "enabled"):
+                    obj.enabled = False
+                if hasattr(obj, "input_mode"):
+                    obj.input_mode = getattr(obj.input_mode, "SIMPLE", "simple")
+
+            elif isinstance(obj, dict) and "module" in obj:
+                obj["enabled"] = False
+
+    def get_i2i_p(self, p, args: ADetailerArgs, image):
+        seed, subseed = self.get_seed(p)
+        width, height = self.get_width_height(p, args)
+        steps = self.get_steps(p, args)
+        cfg_scale = self.get_cfg_scale(p, args)
+
+        sampler_name = p.sampler_name
+        if sampler_name in ["PLMS", "UniPC"]:
+            sampler_name = "Euler"
+
+        i2i = StableDiffusionProcessingImg2Img(
+            init_images=[image],
+            resize_mode=0,
+            denoising_strength=args.ad_denoising_strength,
+            mask=None,
+            mask_blur=args.ad_mask_blur,
+            inpainting_fill=1,
+            inpaint_full_res=args.ad_inpaint_only_masked,
+            inpaint_full_res_padding=args.ad_inpaint_only_masked_padding,
+            inpainting_mask_invert=0,
+            sd_model=p.sd_model,
+            outpath_samples=p.outpath_samples,
+            outpath_grids=p.outpath_grids,
+            prompt="",  # replace later
+            negative_prompt="",
+            styles=p.styles,
+            seed=seed,
+            subseed=subseed,
+            subseed_strength=p.subseed_strength,
+            seed_resize_from_h=p.seed_resize_from_h,
+            seed_resize_from_w=p.seed_resize_from_w,
+            sampler_name=sampler_name,
+            batch_size=1,
+            n_iter=1,
+            steps=steps,
+            cfg_scale=cfg_scale,
+            width=width,
+            height=height,
+            restore_faces=args.ad_restore_face,
+            tiling=p.tiling,
+            extra_generation_params=p.extra_generation_params,
+            do_not_save_samples=True,
+            do_not_save_grid=True,
+        )
+
+        i2i.scripts, i2i.script_args = self.script_filter(p, args)
+        i2i._disable_adetailer = True
+
+        if args.ad_controlnet_model != "None":
+            self.update_controlnet_args(i2i, args)
+        else:
+            i2i.control_net_enabled = False
+
+        return i2i
+
+    def save_image(self, p, image, *, condition: str, suffix: str) -> None:
+        i = p._idx
+        seed, _ = self.get_seed(p)
+
+        if opts.data.get(condition, False):
+            images.save_image(
+                image=image,
+                path=p.outpath_samples,
+                basename="",
+                seed=seed,
+                prompt=p.all_prompts[i] if i < len(p.all_prompts) else p.prompt,
+                extension=opts.samples_format,
+                info=self.infotext(p),
+                p=p,
+                suffix=suffix,
+            )
+
+    def get_ad_model(self, name: str):
+        if name not in model_mapping:
+            msg = f"[-] ADetailer: Model {name!r} not found. Available models: {list(model_mapping.keys())}"
+            raise ValueError(msg)
+        return model_mapping[name]
+
+    def sort_bboxes(self, pred: PredictOutput) -> PredictOutput:
+        sortby = opts.data.get("ad_bbox_sortby", BBOX_SORTBY[0])
+        sortby_idx = BBOX_SORTBY.index(sortby)
+        pred = sort_bboxes(pred, sortby_idx)
+        return pred
+
+    def pred_preprocessing(self, pred: PredictOutput, args: ADetailerArgs):
+        pred = filter_by_ratio(
+            pred, low=args.ad_mask_min_ratio, high=args.ad_mask_max_ratio
+        )
+        pred = self.sort_bboxes(pred)
+        return mask_preprocess(
+            pred.masks,
+            kernel=args.ad_dilate_erode,
+            x_offset=args.ad_x_offset,
+            y_offset=args.ad_y_offset,
+            merge_invert=args.ad_mask_merge_invert,
+        )
+
+    def i2i_prompts_replace(
+        self, i2i, prompts: list[str], negative_prompts: list[str], j: int
+    ) -> None:
+        i1 = min(j, len(prompts) - 1)
+        i2 = min(j, len(negative_prompts) - 1)
+        prompt = prompts[i1]
+        negative_prompt = negative_prompts[i2]
+        i2i.prompt = prompt
+        i2i.negative_prompt = negative_prompt
+
+    def is_need_call_process(self, p) -> bool:
+        i = p._idx
+        n_iter = p.iteration
+        bs = p.batch_size
+        return (i == (n_iter + 1) * bs - 1) and (i != len(p.all_prompts) - 1)
+
+    def process(self, p, *args_):
+        if getattr(p, "_disable_adetailer", False):
+            return
+
+        if self.is_ad_enabled(*args_):
+            arg_list = self.get_args(*args_)
+            extra_params = self.extra_params(arg_list)
+            p.extra_generation_params.update(extra_params)
+
+            p._idx = -1
+
+    def _postprocess_image(self, p, pp, args: ADetailerArgs, *, n: int = 0) -> bool:
+        """
+        Returns
+        -------
+            bool
+
+            `True` if image was processed, `False` otherwise.
+        """
+        if state.interrupted:
+            return False
+
+        i = p._idx
+
+        i2i = self.get_i2i_p(p, args, pp.image)
+        seed, subseed = self.get_seed(p)
+        ad_prompts, ad_negatives = self.get_prompt(p, args)
+
+        is_mediapipe = args.ad_model.lower().startswith("mediapipe")
+
+        kwargs = {}
+        if is_mediapipe:
+            predictor = mediapipe_predict
+            ad_model = args.ad_model
+        else:
+            predictor = ultralytics_predict
+            ad_model = self.get_ad_model(args.ad_model)
+            kwargs["device"] = self.ultralytics_device
+
+        with change_torch_load():
+            pred = predictor(ad_model, pp.image, args.ad_confidence, **kwargs)
+
+        masks = self.pred_preprocessing(pred, args)
+
+        if not masks:
+            print(
+                f"[-] ADetailer: nothing detected on image {i + 1} with {ordinal(n + 1)} settings."
+            )
+            return False
+
+        self.save_image(
+            p,
+            pred.preview,
+            condition="ad_save_previews",
+            suffix="-ad-preview" + suffix(n, "-"),
+        )
+
+        steps = len(masks)
+        processed = None
+        state.job_count += steps
+
+        if is_mediapipe:
+            print(f"mediapipe: {steps} detected.")
+
+        p2 = copy(i2i)
+        for j in range(steps):
+            p2.image_mask = masks[j]
+            self.i2i_prompts_replace(p2, ad_prompts, ad_negatives, j)
+
+            if not re.match(r"^\s*\[SKIP\]\s*$", p2.prompt):
+                if args.ad_controlnet_model == "None":
+                    cn_restore_unet_hook(p2, self.cn_latest_network)
+                processed = process_images(p2)
+
+                p2 = copy(i2i)
+                p2.init_images = [processed.images[0]]
+
+            p2.seed = seed + j + 1
+            p2.subseed = subseed + j + 1
+
+        if processed is not None:
+            pp.image = processed.images[0]
+            return True
+
+        return False
+
+    def postprocess_image(self, p, pp, *args_):
+        if getattr(p, "_disable_adetailer", False):
+            return
+
+        if not self.is_ad_enabled(*args_):
+            return
+
+        p._idx = getattr(p, "_idx", -1) + 1
+        init_image = copy(pp.image)
+        arg_list = self.get_args(*args_)
+
+        is_processed = False
+        with CNHijackRestore(), pause_total_tqdm(), cn_allow_script_control():
+            for n, args in enumerate(arg_list):
+                if args.ad_model == "None":
+                    continue
+                is_processed |= self._postprocess_image(p, pp, args, n=n)
+
+        if is_processed:
+            self.save_image(
+                p, init_image, condition="ad_save_images_before", suffix="-ad-before"
+            )
+
+        if self.cn_script is not None and self.is_need_call_process(p):
+            self.cn_script.process(p)
+
+        try:
+            if p._idx == len(p.all_prompts) - 1:
+                self.write_params_txt(p)
+        except Exception:
+            pass
+
+
+def on_after_component(component, **_kwargs):
+    global txt2img_submit_button, img2img_submit_button
+    if getattr(component, "elem_id", None) == "txt2img_generate":
+        txt2img_submit_button = component
+        return
+
+    if getattr(component, "elem_id", None) == "img2img_generate":
+        img2img_submit_button = component
+
+
+def on_ui_settings():
+    section = ("ADetailer", AFTER_DETAILER)
+    shared.opts.add_option(
+        "ad_max_models",
+        shared.OptionInfo(
+            default=2,
+            label="Max models",
+            component=gr.Slider,
+            component_args={"minimum": 1, "maximum": 5, "step": 1},
+            section=section,
+        ),
+    )
+
+    shared.opts.add_option(
+        "ad_save_previews",
+        shared.OptionInfo(False, "Save mask previews", section=section),
+    )
+
+    shared.opts.add_option(
+        "ad_save_images_before",
+        shared.OptionInfo(False, "Save images before ADetailer", section=section),
+    )
+
+    shared.opts.add_option(
+        "ad_only_seleted_scripts",
+        shared.OptionInfo(
+            True, "Apply only selected scripts to ADetailer", section=section
+        ),
+    )
+
+    textbox_args = {
+        "placeholder": "comma-separated list of script names",
+        "interactive": True,
+    }
+
+    shared.opts.add_option(
+        "ad_script_names",
+        shared.OptionInfo(
+            default=SCRIPT_DEFAULT,
+            label="Script names to apply to ADetailer (separated by comma)",
+            component=gr.Textbox,
+            component_args=textbox_args,
+            section=section,
+        ),
+    )
+
+    shared.opts.add_option(
+        "ad_bbox_sortby",
+        shared.OptionInfo(
+            default="None",
+            label="Sort bounding boxes by",
+            component=gr.Radio,
+            component_args={"choices": BBOX_SORTBY},
+            section=section,
+        ),
+    )
+
+
+script_callbacks.on_ui_settings(on_ui_settings)
+script_callbacks.on_after_component(on_after_component)

extensions/adetailer/sd_webui/images.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from PIL import Image, PngImagePlugin
+
+    from sd_webui.processing import StableDiffusionProcessing
+
+    def save_image(
+        image: Image.Image,
+        path: str,
+        basename: str,
+        seed: int | None = None,
+        prompt: str = "",
+        extension: str = "png",
+        info: str | PngImagePlugin.iTXt = "",
+        short_filename: bool = False,
+        no_prompt: bool = False,
+        grid: bool = False,
+        pnginfo_section_name: str = "parameters",
+        p: StableDiffusionProcessing | None = None,
+        existing_info: dict | None = None,
+        forced_filename: str | None = None,
+        suffix: str = "",
+        save_to_dirs: bool = False,
+    ) -> tuple[str, str | None]:
+        """Save an image.
+
+        Args:
+            image (`PIL.Image`):
+                The image to be saved.
+            path (`str`):
+                The directory to save the image. Note, the option `save_to_dirs` will make the image to be saved into a sub directory.
+            basename (`str`):
+                The base filename which will be applied to `filename pattern`.
+            seed, prompt, short_filename,
+            extension (`str`):
+                Image file extension, default is `png`.
+            pngsectionname (`str`):
+                Specify the name of the section which `info` will be saved in.
+            info (`str` or `PngImagePlugin.iTXt`):
+                PNG info chunks.
+            existing_info (`dict`):
+                Additional PNG info. `existing_info == {pngsectionname: info, ...}`
+            no_prompt:
+                TODO I don't know its meaning.
+            p (`StableDiffusionProcessing`)
+            forced_filename (`str`):
+                If specified, `basename` and filename pattern will be ignored.
+            save_to_dirs (bool):
+                If true, the image will be saved into a subdirectory of `path`.
+
+        Returns: (fullfn, txt_fullfn)
+            fullfn (`str`):
+                The full path of the saved imaged.
+            txt_fullfn (`str` or None):
+                If a text file is saved for this image, this will be its full path. Otherwise None.
+        """
+
+else:
+    from modules.images import save_image

extensions/adetailer/sd_webui/paths.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import os
+
+    models_path = os.path.join(os.path.dirname(__file__), "1")
+    script_path = os.path.join(os.path.dirname(__file__), "2")
+    data_path = os.path.join(os.path.dirname(__file__), "3")
+    extensions_dir = os.path.join(os.path.dirname(__file__), "4")
+    extensions_builtin_dir = os.path.join(os.path.dirname(__file__), "5")
+else:
+    from modules.paths import data_path, models_path, script_path

extensions/adetailer/sd_webui/processing.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from dataclasses import dataclass, field
+    from typing import Any, Callable
+
+    import numpy as np
+    import torch
+    from PIL import Image
+
+    def _image():
+        return Image.new("L", (512, 512))
+
+    @dataclass
+    class StableDiffusionProcessing:
+        sd_model: torch.nn.Module = field(default_factory=lambda: torch.nn.Linear(1, 1))
+        outpath_samples: str = ""
+        outpath_grids: str = ""
+        prompt: str = ""
+        prompt_for_display: str = ""
+        negative_prompt: str = ""
+        styles: list[str] = field(default_factory=list)
+        seed: int = -1
+        subseed: int = -1
+        subseed_strength: float = 0.0
+        seed_resize_from_h: int = -1
+        seed_resize_from_w: int = -1
+        sampler_name: str | None = None
+        batch_size: int = 1
+        n_iter: int = 1
+        steps: int = 50
+        cfg_scale: float = 7.0
+        width: int = 512
+        height: int = 512
+        restore_faces: bool = False
+        tiling: bool = False
+        do_not_save_samples: bool = False
+        do_not_save_grid: bool = False
+        extra_generation_params: dict[str, Any] = field(default_factory=dict)
+        overlay_images: list[Image.Image] = field(default_factory=list)
+        eta: float = 0.0
+        do_not_reload_embeddings: bool = False
+        paste_to: tuple[int | float, ...] = (0, 0, 0, 0)
+        color_corrections: list[np.ndarray] = field(default_factory=list)
+        denoising_strength: float = 0.0
+        sampler_noise_scheduler_override: Callable | None = None
+        ddim_discretize: str = ""
+        s_min_uncond: float = 0.0
+        s_churn: float = 0.0
+        s_tmin: float = 0.0
+        s_tmax: float = 0.0
+        s_noise: float = 0.0
+        override_settings: dict[str, Any] = field(default_factory=dict)
+        override_settings_restore_afterwards: bool = False
+        is_using_inpainting_conditioning: bool = False
+        disable_extra_networks: bool = False
+        scripts: Any = None
+        script_args: list[Any] = field(default_factory=list)
+        all_prompts: list[str] = field(default_factory=list)
+        all_negative_prompts: list[str] = field(default_factory=list)
+        all_seeds: list[int] = field(default_factory=list)
+        all_subseeds: list[int] = field(default_factory=list)
+        iteration: int = 1
+        is_hr_pass: bool = False
+
+    @dataclass
+    class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing):
+        sampler: Callable | None = None
+        enable_hr: bool = False
+        denoising_strength: float = 0.75
+        hr_scale: float = 2.0
+        hr_upscaler: str = ""
+        hr_second_pass_steps: int = 0
+        hr_resize_x: int = 0
+        hr_resize_y: int = 0
+        hr_upscale_to_x: int = 0
+        hr_upscale_to_y: int = 0
+        width: int = 512
+        height: int = 512
+        truncate_x: int = 512
+        truncate_y: int = 512
+        applied_old_hires_behavior_to: tuple[int, int] = (512, 512)
+
+    @dataclass
+    class StableDiffusionProcessingImg2Img(StableDiffusionProcessing):
+        sampler: Callable | None = None
+        init_images: list[Image.Image] = field(default_factory=list)
+        resize_mode: int = 0
+        denoising_strength: float = 0.75
+        image_cfg_scale: float | None = None
+        init_latent: torch.Tensor | None = None
+        image_mask: Image.Image = field(default_factory=_image)
+        latent_mask: Image.Image = field(default_factory=_image)
+        mask_for_overlay: Image.Image = field(default_factory=_image)
+        mask_blur: int = 4
+        inpainting_fill: int = 0
+        inpaint_full_res: bool = True
+        inpaint_full_res_padding: int = 0
+        inpainting_mask_invert: int | bool = 0
+        initial_noise_multiplier: float = 1.0
+        mask: torch.Tensor | None = None
+        nmask: torch.Tensor | None = None
+        image_conditioning: torch.Tensor | None = None
+
+    @dataclass
+    class Processed:
+        images: list[Image.Image] = field(default_factory=list)
+        prompt: list[str] = field(default_factory=list)
+        negative_prompt: list[str] = field(default_factory=list)
+        seed: list[int] = field(default_factory=list)
+        subseed: list[int] = field(default_factory=list)
+        subseed_strength: float = 0.0
+        info: str = ""
+        comments: str = ""
+        width: int = 512
+        height: int = 512
+        sampler_name: str = ""
+        cfg_scale: float = 7.0
+        image_cfg_scale: float | None = None
+        steps: int = 50
+        batch_size: int = 1
+        restore_faces: bool = False
+        face_restoration_model: str | None = None
+        sd_model_hash: str = ""
+        seed_resize_from_w: int = -1
+        seed_resize_from_h: int = -1
+        denoising_strength: float = 0.0
+        extra_generation_params: dict[str, Any] = field(default_factory=dict)
+        index_of_first_image: int = 0
+        styles: list[str] = field(default_factory=list)
+        job_timestamp: str = ""
+        clip_skip: int = 1
+        eta: float = 0.0
+        ddim_discretize: str = ""
+        s_churn: float = 0.0
+        s_tmin: float = 0.0
+        s_tmax: float = 0.0
+        s_noise: float = 0.0
+        sampler_noise_scheduler_override: Callable | None = None
+        is_using_inpainting_conditioning: bool = False
+        all_prompts: list[str] = field(default_factory=list)
+        all_negative_prompts: list[str] = field(default_factory=list)
+        all_seeds: list[int] = field(default_factory=list)
+        all_subseeds: list[int] = field(default_factory=list)
+        infotexts: list[str] = field(default_factory=list)
+
+    def create_infotext(
+        p: StableDiffusionProcessingTxt2Img | StableDiffusionProcessingImg2Img,
+        all_prompts: list[str],
+        all_seeds: list[int],
+        all_subseeds: list[int],
+        comments: Any,
+        iteration: int = 0,
+        position_in_batch: int = 0,
+    ) -> str:
+        pass
+
+    def process_images(
+        p: StableDiffusionProcessingTxt2Img | StableDiffusionProcessingImg2Img,
+    ) -> Processed:
+        pass
+
+else:
+    from modules.processing import (
+        StableDiffusionProcessing,
+        StableDiffusionProcessingImg2Img,
+        StableDiffusionProcessingTxt2Img,
+        create_infotext,
+        process_images,
+    )

extensions/adetailer/sd_webui/safe.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import torch
+
+    unsafe_torch_load = torch.load
+else:
+    from modules.safe import unsafe_torch_load

extensions/adetailer/sd_webui/script_callbacks.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Callable
+
+    def on_ui_settings(callback: Callable):
+        pass
+
+    def on_after_component(callback: Callable):
+        pass
+
+else:
+    from modules.script_callbacks import on_after_component, on_ui_settings

extensions/adetailer/sd_webui/scripts.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from abc import ABC, abstractmethod
+    from dataclasses import dataclass
+    from typing import Any
+
+    import gradio as gr
+    from PIL import Image
+
+    from sd_webui.processing import (
+        Processed,
+        StableDiffusionProcessingImg2Img,
+        StableDiffusionProcessingTxt2Img,
+    )
+
+    SDPType = StableDiffusionProcessingImg2Img | StableDiffusionProcessingTxt2Img
+    AlwaysVisible = object()
+
+    @dataclass
+    class PostprocessImageArgs:
+        image: Image.Image
+
+    class Script(ABC):
+        filename: str
+        args_from: int
+        args_to: int
+        alwayson: bool
+
+        is_txt2img: bool
+        is_img2img: bool
+
+        group: gr.Group
+        infotext_fields: list[tuple[str, str]]
+        paste_field_names: list[str]
+
+        @abstractmethod
+        def title(self):
+            raise NotImplementedError
+
+        def ui(self, is_img2img: bool):
+            pass
+
+        def show(self, is_img2img: bool):
+            return True
+
+        def run(self, p: SDPType, *args):
+            pass
+
+        def process(self, p: SDPType, *args):
+            pass
+
+        def before_process_batch(self, p: SDPType, *args, **kwargs):
+            pass
+
+        def process_batch(self, p: SDPType, *args, **kwargs):
+            pass
+
+        def postprocess_batch(self, p: SDPType, *args, **kwargs):
+            pass
+
+        def postprocess_image(self, p: SDPType, pp: PostprocessImageArgs, *args):
+            pass
+
+        def postprocess(self, p: SDPType, processed: Processed, *args):
+            pass
+
+        def before_component(self, component, **kwargs):
+            pass
+
+        def after_component(self, component, **kwargs):
+            pass
+
+        def describe(self):
+            return ""
+
+        def elem_id(self, item_id: Any) -> str:
+            pass
+
+else:
+    from modules.scripts import AlwaysVisible, PostprocessImageArgs, Script

extensions/adetailer/sd_webui/shared.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import argparse
+    from dataclasses import dataclass
+    from typing import Any, Callable
+
+    import torch
+    from PIL import Image
+
+    @dataclass
+    class State:
+        skipped: bool = False
+        interrupted: bool = False
+        job: str = ""
+        job_no: int = 0
+        job_count: int = 0
+        processing_has_refined_job_count: bool = False
+        job_timestamp: str = "0"
+        sampling_step: int = 0
+        sampling_steps: int = 0
+        current_latent: torch.Tensor | None = None
+        current_image: Image.Image | None = None
+        current_image_sampling_step: int = 0
+        id_live_preview: int = 0
+        textinfo: str | None = None
+        time_start: float | None = None
+        need_restart: bool = False
+        server_start: float | None = None
+
+    @dataclass
+    class OptionInfo:
+        default: Any = None
+        label: str = ""
+        component: Any = None
+        component_args: Callable[[], dict] | dict[str, Any] | None = None
+        onchange: Callable[[], None] | None = None
+        section: tuple[str, str] | None = None
+        refresh: Callable[[], None] | None = None
+
+    class Option:
+        data_labels: dict[str, OptionInfo]
+
+        def __init__(self):
+            self.data: dict[str, Any] = {}
+
+        def add_option(self, key: str, info: OptionInfo):
+            pass
+
+        def __getattr__(self, item: str):
+            if self.data is not None and item in self.data:
+                return self.data[item]
+
+            if item in self.data_labels:
+                return self.data_labels[item].default
+
+            return super().__getattribute__(item)
+
+    opts = Option()
+    cmd_opts = argparse.Namespace()
+    state = State()
+
+else:
+    from modules.shared import OptionInfo, cmd_opts, opts, state

extensions/microsoftexcel-controlnet/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,91 @@
+name: Bug Report
+description: Create a report
+title: "[Bug]: "
+labels: ["bug-report"]
+
+body:
+  - type: checkboxes
+    attributes:
+      label: Is there an existing issue for this?
+      description: Please search to see if an issue already exists for the bug you encountered, and that it hasn't been fixed in a recent build/commit.
+      options:
+        - label: I have searched the existing issues and checked the recent builds/commits of both this extension and the webui
+          required: true
+  - type: markdown
+    attributes:
+      value: |
+        *Please fill this form with as much information as possible, don't forget to fill "What OS..." and "What browsers" and *provide screenshots if possible**
+  - type: textarea
+    id: what-did
+    attributes:
+      label: What happened?
+      description: Tell us what happened in a very clear and simple way
+    validations:
+      required: true
+  - type: textarea
+    id: steps
+    attributes:
+      label: Steps to reproduce the problem
+      description: Please provide us with precise step by step information on how to reproduce the bug
+      value: |
+        1. Go to .... 
+        2. Press ....
+        3. ...
+    validations:
+      required: true
+  - type: textarea
+    id: what-should
+    attributes:
+      label: What should have happened?
+      description: Tell what you think the normal behavior should be
+    validations:
+      required: true
+  - type: textarea
+    id: commits
+    attributes:
+      label: Commit where the problem happens
+      description: Which commit of the extension are you running on? Please include the commit of both the extension and the webui (Do not write *Latest version/repo/commit*, as this means nothing and will have changed by the time we read your issue. Rather, copy the **Commit** link at the bottom of the UI, or from the cmd/terminal if you can't launch it.)
+      value: |
+        webui: 
+        controlnet: 
+    validations:
+      required: true
+  - type: dropdown
+    id: browsers
+    attributes:
+      label: What browsers do you use to access the UI ?
+      multiple: true
+      options:
+        - Mozilla Firefox
+        - Google Chrome
+        - Brave
+        - Apple Safari
+        - Microsoft Edge
+  - type: textarea
+    id: cmdargs
+    attributes:
+      label: Command Line Arguments
+      description: Are you using any launching parameters/command line arguments (modified webui-user .bat/.sh) ? If yes, please write them below. Write "No" otherwise.
+      render: Shell
+    validations:
+      required: true
+  - type: textarea
+    id: extensions
+    attributes:
+      label: List of enabled extensions
+      description: Please provide a full list of enabled extensions or screenshots of your "Extensions" tab.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Console logs
+      description: Please provide full cmd/terminal logs from the moment you started UI to the end of it, after your bug happened. If it's very long, provide a link to pastebin or similar service.
+      render: Shell
+    validations:
+      required: true
+  - type: textarea
+    id: misc
+    attributes:
+      label: Additional information
+      description: Please provide us with any relevant additional info or context.

extensions/microsoftexcel-controlnet/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: true

extensions/microsoftexcel-controlnet/.github/workflows/tests.yml

@@ -0,0 +1,37 @@
+name: Run basic features tests on CPU
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+        with: 
+          repository: 'AUTOMATIC1111/stable-diffusion-webui'
+          path: 'stable-diffusion-webui'
+          ref: '5ab7f213bec2f816f9c5644becb32eb72c8ffb89'
+
+      - name: Checkout Code
+        uses: actions/checkout@v3
+        with:
+          repository: 'Mikubill/sd-webui-controlnet'
+          path: 'stable-diffusion-webui/extensions/sd-webui-controlnet'
+          
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.10.6
+          cache: pip
+          cache-dependency-path: |
+            **/requirements*txt
+            stable-diffusion-webui/requirements*txt
+            
+      - run: |
+          pip install torch torchvision
+          curl -Lo stable-diffusion-webui/extensions/sd-webui-controlnet/models/control_canny-fp16.safetensors https://huggingface.co/webui/ControlNet-modules-safetensors/resolve/main/control_canny-fp16.safetensors
+          cd stable-diffusion-webui && python launch.py --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test --api --tests ./extensions/sd-webui-controlnet/tests
+          rm -fr stable-diffusion-webui/extensions/sd-webui-controlnet/models/control_canny-fp16.safetensors

extensions/microsoftexcel-controlnet/.gitignore

@@ -0,0 +1,171 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea
+*.pt
+*.pth
+*.ckpt
+*.bin
+*.safetensors
+
+# Editor setting metadata
+.idea/
+.vscode/
+detected_maps/
+annotator/downloads/

extensions/microsoftexcel-controlnet/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Kakigōri Maker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

extensions/microsoftexcel-controlnet/README.md

@@ -0,0 +1,246 @@
+# ControlNet for Stable Diffusion WebUI
+
+The WebUI extension for ControlNet and other injection-based SD controls.
+
+![image](https://user-images.githubusercontent.com/19834515/235606305-229b3d1e-5bfc-467f-9d55-0976eab71652.png)
+
+This extension is for AUTOMATIC1111's [Stable Diffusion web UI](https://github.com/AUTOMATIC1111/stable-diffusion-webui), allows the Web UI to add [ControlNet](https://github.com/lllyasviel/ControlNet) to the original Stable Diffusion model to generate images. The addition is on-the-fly, the merging is not required.
+
+# Installation
+
+1. Open "Extensions" tab.
+2. Open "Install from URL" tab in the tab.
+3. Enter `https://github.com/Mikubill/sd-webui-controlnet.git` to "URL for extension's git repository".
+4. Press "Install" button.
+5. Wait for 5 seconds, and you will see the message "Installed into stable-diffusion-webui\extensions\sd-webui-controlnet. Use Installed tab to restart".
+6. Go to "Installed" tab, click "Check for updates", and then click "Apply and restart UI". (The next time you can also use these buttons to update ControlNet.)
+7. Completely restart A1111 webui including your terminal. (If you do not know what is a "terminal", you can reboot your computer to achieve the same effect.)
+8. Download models (see below).
+9. After you put models in the correct folder, you may need to refresh to see the models. The refresh button is right to your "Model" dropdown.
+
+
+**Update from ControlNet 1.0 to 1.1:**
+
+* If you are not sure, you can back up and remove the folder "stable-diffusion-webui\extensions\sd-webui-controlnet", and then start from the step 1 in the above Installation section. 
+
+* Or you can start from the step 6 in the above Install section.
+
+# Download Models
+
+Right now all the 14 models of ControlNet 1.1 are in the beta test.
+
+Download the models from ControlNet 1.1: https://huggingface.co/lllyasviel/ControlNet-v1-1/tree/main
+
+You need to download model files ending with ".pth" .
+
+Put models in your "stable-diffusion-webui\extensions\sd-webui-controlnet\models". Now we have already included all "yaml" files. You only need to download "pth" files.
+
+Do not right-click the filenames in HuggingFace website to download. Some users right-clicked those HuggingFace HTML websites and saved those HTML pages as PTH/YAML files. They are not downloading correct files. Instead, please click the small download arrow “↓” icon in HuggingFace to download.
+
+Note: If you download models elsewhere, please make sure that yaml file names and model files names are same. Please manually rename all yaml files if you download from other sources. (Some models like "shuffle" needs the yaml file so that we know the outputs of ControlNet should pass a global average pooling before injecting to SD U-Nets.)
+
+# New Features in ControlNet 1.1
+
+### Perfect Support for All ControlNet 1.0/1.1 and T2I Adapter Models.
+
+Now we have perfect support all available models and preprocessors, including perfect support for T2I style adapter and ControlNet 1.1 Shuffle. (Make sure that your YAML file names and model file names are same, see also YAML files in "stable-diffusion-webui\extensions\sd-webui-controlnet\models".)
+
+### Perfect Support for A1111 High-Res. Fix
+
+Now if you turn on High-Res Fix in A1111, each controlnet will output two different control images: a small one and a large one. The small one is for your basic generating, and the big one is for your High-Res Fix generating. The two control images are computed by a smart algorithm called "super high-quality control image resampling". This is turned on by default, and you do not need to change any setting.
+
+### Perfect Support for All A1111 Img2Img or Inpaint Settings and All Mask Types
+
+Now ControlNet is extensively tested with A1111's different types of masks, including "Inpaint masked"/"Inpaint not masked", and "Whole picture"/"Only masked", and "Only masked padding"&"Mask blur". The resizing perfectly matches A1111's "Just resize"/"Crop and resize"/"Resize and fill". This means you can use ControlNet in nearly everywhere in your A1111 UI without difficulty!
+
+### The New "Pixel-Perfect" Mode
+
+Now if you turn on pixel-perfect mode, you do not need to set preprocessor (annotator) resolutions manually. The ControlNet will automatically compute the best annotator resolution for you so that each pixel perfectly matches Stable Diffusion.
+
+### User-Friendly GUI and Preprocessor Preview
+
+We reorganized some previously confusing UI like "canvas width/height for new canvas" and it is in the 📝 button now. Now the preview GUI is controlled by the "allow preview" option and the trigger button 💥. The preview image size is better than before, and you do not need to scroll up and down - your a1111 GUI will not be messed up anymore!
+
+### Support for Almost All Upscaling Scripts
+
+Now ControlNet 1.1 can support almost all Upscaling/Tile methods. ControlNet 1.1 support the script "Ultimate SD upscale" and almost all other tile-based extensions. Please do not confuse ["Ultimate SD upscale"](https://github.com/Coyote-A/ultimate-upscale-for-automatic1111) with "SD upscale" - they are different scripts. Note that the most recommended upscaling method is ["Tiled VAE/Diffusion"](https://github.com/pkuliyi2015/multidiffusion-upscaler-for-automatic1111) but we test as many methods/extensions as possible. Note that "SD upscale" is supported since 1.1.117, and if you use it, you need to leave all ControlNet images as blank (We do not recommend "SD upscale" since it is somewhat buggy and cannot be maintained - use the "Ultimate SD upscale" instead).
+
+### More Control Modes (previously called Guess Mode)
+
+We have fixed many bugs in previous 1.0’s Guess Mode and now it is called Control Mode
+
+![image](https://user-images.githubusercontent.com/19834515/236641759-6c44ddf6-c7ad-4bda-92be-e90a52911d75.png)
+
+Now you can control which aspect is more important (your prompt or your ControlNet)：
+
+* "Balanced": ControlNet on both sides of CFG scale, same as turning off "Guess Mode" in ControlNet 1.0
+
+* "My prompt is more important": ControlNet on both sides of CFG scale, with progressively reduced SD U-Net injections (layer_weight*=0.825**I, where 0<=I <13, and the 13 means ControlNet injected SD 13 times). In this way, you can make sure that your prompts are perfectly displayed in your generated images.
+
+* "ControlNet is more important": ControlNet only on the Conditional Side of CFG scale (the cond in A1111's batch-cond-uncond). This means the ControlNet will be X times stronger if your cfg-scale is X. For example, if your cfg-scale is 7, then ControlNet is 7 times stronger. Note that here the X times stronger is different from "Control Weights" since your weights are not modified. This "stronger" effect usually has less artifact and give ControlNet more room to guess what is missing from your prompts (and in the previous 1.0, it is called "Guess Mode"). 
+
+<table width="100%">
+<tr>
+<td width="25%" style="text-align: center">Input (depth+canny+hed)</td>
+<td width="25%" style="text-align: center">"Balanced"</td>
+<td width="25%" style="text-align: center">"My prompt is more important"</td>
+<td width="25%" style="text-align: center">"ControlNet is more important"</td>
+</tr>
+<tr>
+<td width="25%" style="text-align: center"><img src="samples/cm1.png"></td>
+<td width="25%" style="text-align: center"><img src="samples/cm2.png"></td>
+<td width="25%" style="text-align: center"><img src="samples/cm3.png"></td>
+<td width="25%" style="text-align: center"><img src="samples/cm4.png"></td>
+</tr>
+</table>
+
+### Reference-Only Control
+
+Now we have a `reference-only` preprocessor that does not require any control models. It can guide the diffusion directly using images as references.
+
+(Prompt "a dog running on grassland, best quality, ...")
+
+![image](samples/ref.png)
+
+This method is similar to inpaint-based reference but it does not make your image disordered. 
+
+Many professional A1111 users know a trick to diffuse image with references by inpaint. For example, if you have a 512x512 image of a dog, and want to generate another 512x512 image with the same dog, some users will connect the 512x512 dog image and a 512x512 blank image into a 1024x512 image, send to inpaint, and mask out the blank 512x512 part to diffuse a dog with similar appearance. However, that method is usually not very satisfying since images are connected and many distortions will appear.
+
+This `reference-only` ControlNet can directly link the attention layers of your SD to any independent images, so that your SD will read arbitary images for reference. You need at least ControlNet 1.1.153 to use it.
+
+To use, just select `reference-only` as preprocessor and put an image. Your SD will just use the image as reference.
+
+*Note that this method is as "non-opinioned" as possible. It only contains very basic connection codes, without any personal preferences, to connect the attention layers with your reference images. However, even if we tried best to not include any opinioned codes, we still need to write some subjective implementations to deal with weighting, cfg-scale, etc - tech report is on the way.*
+
+More examples [here](https://github.com/Mikubill/sd-webui-controlnet/discussions/1236).
+
+# Technical Documents
+
+See also the documents of ControlNet 1.1: 
+
+https://github.com/lllyasviel/ControlNet-v1-1-nightly#model-specification
+
+# Default Setting
+
+This is my setting. If you run into any problem, you can use this setting as a sanity check
+
+![image](https://user-images.githubusercontent.com/19834515/235620638-17937171-8ac1-45bc-a3cb-3aebf605b4ef.png)
+
+# Use Previous Models
+
+### Use ControlNet 1.0 Models
+
+https://huggingface.co/lllyasviel/ControlNet/tree/main/models
+
+You can still use all previous models in the previous ControlNet 1.0. Now, the previous "depth" is now called "depth_midas", the previous "normal" is called "normal_midas", the previous "hed" is called "softedge_hed". And starting from 1.1, all line maps, edge maps, lineart maps, boundary maps will have black background and white lines.
+
+### Use T2I-Adapter Models
+
+(From TencentARC/T2I-Adapter)
+
+To use T2I-Adapter models:
+
+1. Download files from https://huggingface.co/TencentARC/T2I-Adapter/tree/main/models
+2. Put them in "stable-diffusion-webui\extensions\sd-webui-controlnet\models".
+3. Make sure that the file names of pth files and yaml files are consistent.
+
+*Note that "CoAdapter" is not implemented yet.*
+
+# Gallery
+
+The below results are from ControlNet 1.0.
+
+| Source | Input | Output |
+|:-------------------------:|:-------------------------:|:-------------------------:|
+| (no preprocessor) |  <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/bal-source.png?raw=true"> | <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/bal-gen.png?raw=true"> |
+| (no preprocessor) |  <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/dog_rel.jpg?raw=true"> | <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/dog_rel.png?raw=true"> |
+|<img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/mahiro_input.png?raw=true">  |  <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/mahiro_canny.png?raw=true"> | <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/mahiro-out.png?raw=true"> |
+|<img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/evt_source.jpg?raw=true">  |  <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/evt_hed.png?raw=true"> | <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/evt_gen.png?raw=true"> |
+|<img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/an-source.jpg?raw=true">  |  <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/an-pose.png?raw=true"> | <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/an-gen.png?raw=true"> |
+|<img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/sk-b-src.png?raw=true">  |  <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/sk-b-dep.png?raw=true"> | <img width="256" alt="" src="https://github.com/Mikubill/sd-webui-controlnet/blob/main/samples/sk-b-out.png?raw=true"> |
+
+The below examples are from T2I-Adapter.
+
+From `t2iadapter_color_sd14v1.pth` :
+
+| Source | Input | Output |
+|:-------------------------:|:-------------------------:|:-------------------------:|
+| <img width="256" alt="" src="https://user-images.githubusercontent.com/31246794/222947416-ec9e52a4-a1d0-48d8-bb81-736bf636145e.jpeg"> | <img width="256" alt="" src="https://user-images.githubusercontent.com/31246794/222947435-1164e7d8-d857-42f9-ab10-2d4a4b25f33a.png"> | <img width="256" alt="" src="https://user-images.githubusercontent.com/31246794/222947557-5520d5f8-88b4-474d-a576-5c9cd3acac3a.png"> |
+
+From `t2iadapter_style_sd14v1.pth` :
+
+| Source | Input | Output |
+|:-------------------------:|:-------------------------:|:-------------------------:|
+| <img width="256" alt="" src="https://user-images.githubusercontent.com/31246794/222947416-ec9e52a4-a1d0-48d8-bb81-736bf636145e.jpeg"> | (clip, non-image) | <img width="256" alt="" src="https://user-images.githubusercontent.com/31246794/222965711-7b884c9e-7095-45cb-a91c-e50d296ba3a2.png"> |
+
+# Minimum Requirements
+
+* (Windows) (NVIDIA: Ampere) 4gb - with `--xformers` enabled, and `Low VRAM` mode ticked in the UI, goes up to 768x832
+
+# Multi-ControlNet
+
+This option allows multiple ControlNet inputs for a single generation. To enable this option, change `Multi ControlNet: Max models amount (requires restart)` in the settings. Note that you will need to restart the WebUI for changes to take effect.
+
+<table width="100%">
+<tr>
+<td width="25%" style="text-align: center">Source A</td>
+<td width="25%" style="text-align: center">Source B</td>
+<td width="25%" style="text-align: center">Output</td>
+</tr>
+<tr>
+<td width="25%" style="text-align: center"><img src="https://user-images.githubusercontent.com/31246794/220448620-cd3ede92-8d3f-43d5-b771-32dd8417618f.png"></td>
+<td width="25%" style="text-align: center"><img src="https://user-images.githubusercontent.com/31246794/220448619-beed9bdb-f6bb-41c2-a7df-aa3ef1f653c5.png"></td>
+<td width="25%" style="text-align: center"><img src="https://user-images.githubusercontent.com/31246794/220448613-c99a9e04-0450-40fd-bc73-a9122cefaa2c.png"></td>
+</tr>
+</table>
+
+# Control Weight/Start/End
+
+Weight is the weight of the controlnet "influence". It's analogous to prompt attention/emphasis. E.g. (myprompt: 1.2). Technically, it's the factor by which to multiply the ControlNet outputs before merging them with original SD Unet.
+
+Guidance Start/End is the percentage of total steps the controlnet applies (guidance strength = guidance end). It's analogous to prompt editing/shifting. E.g. \[myprompt::0.8\] (It applies from the beginning until 80% of total steps)
+
+# Batch Mode
+
+Put any unit into batch mode to activate batch mode for all units. Specify a batch directory for each unit, or use the new textbox in the img2img batch tab as a fallback. Although the textbox is located in the img2img batch tab, you can use it to generate images in the txt2img tab as well.
+
+Note that this feature is only available in the gradio user interface. Call the APIs as many times as you want for custom batch scheduling.
+
+# API and Script Access
+
+This extension can accept txt2img or img2img tasks via API or external extension call. Note that you may need to enable `Allow other scripts to control this extension` in settings for external calls.
+
+To use the API: start WebUI with argument `--api` and go to `http://webui-address/docs` for documents or checkout [examples](https://github.com/Mikubill/sd-webui-controlnet/blob/main/example/api_txt2img.ipynb).
+
+To use external call: Checkout [Wiki](https://github.com/Mikubill/sd-webui-controlnet/wiki/API)
+
+# Command Line Arguments
+
+This extension adds these command line arguments to the webui:
+
+```
+    --controlnet-dir <path to directory with controlnet models>                                ADD a controlnet models directory
+    --controlnet-annotator-models-path <path to directory with annotator model directories>    SET the directory for annotator models
+    --no-half-controlnet                                                                       load controlnet models in full precision
+    --controlnet-preprocessor-cache-size                                                       Cache size for controlnet preprocessor results
+```
+
+# MacOS Support
+
+Tested with pytorch nightly: https://github.com/Mikubill/sd-webui-controlnet/pull/143#issuecomment-1435058285
+
+To use this extension with mps and normal pytorch, currently you may need to start WebUI with `--no-half`.
+
+# Archive of Deprecated Versions
+
+The previous version (sd-webui-controlnet 1.0) is archived in 
+
+https://github.com/lllyasviel/webui-controlnet-v1-archived
+
+Using this version is not a temporary stop of updates. You will stop all updates forever.
+
+Please consider this version if you work with professional studios that requires 100% reproducing of all previous results pixel by pixel.
+
+# Thanks
+
+This implementation is inspired by kohya-ss/sd-webui-additional-networks

extensions/microsoftexcel-controlnet/annotator/annotator_path.py

@@ -0,0 +1,22 @@
+import os
+from modules import shared
+
+models_path = shared.opts.data.get('control_net_modules_path', None)
+if not models_path:
+    models_path = getattr(shared.cmd_opts, 'controlnet_annotator_models_path', None)
+if not models_path:
+    models_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'downloads')
+
+if not os.path.isabs(models_path):
+    models_path = os.path.join(shared.data_path, models_path)
+
+clip_vision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'clip_vision')
+# clip vision is always inside controlnet "extensions\sd-webui-controlnet"
+# and any problem can be solved by removing controlnet and reinstall
+
+models_path = os.path.realpath(models_path)
+os.makedirs(models_path, exist_ok=True)
+print(f'ControlNet preprocessor location: {models_path}')
+# Make sure that the default location is inside controlnet "extensions\sd-webui-controlnet"
+# so that any problem can be solved by removing controlnet and reinstall
+# if users do not change configs on their own (otherwise users will know what is wrong)

extensions/microsoftexcel-controlnet/annotator/binary/__init__.py

@@ -0,0 +1,14 @@
+import cv2
+
+
+def apply_binary(img, bin_threshold):
+    img_gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+
+    if bin_threshold == 0 or bin_threshold == 255:
+        # Otsu's threshold
+        otsu_threshold, img_bin = cv2.threshold(img_gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
+        print("Otsu threshold:", otsu_threshold)
+    else:
+        _, img_bin = cv2.threshold(img_gray, bin_threshold, 255, cv2.THRESH_BINARY_INV)
+
+    return cv2.cvtColor(img_bin, cv2.COLOR_GRAY2RGB)

extensions/microsoftexcel-controlnet/annotator/canny/__init__.py

@@ -0,0 +1,5 @@
+import cv2
+
+
+def apply_canny(img, low_threshold, high_threshold):
+    return cv2.Canny(img, low_threshold, high_threshold)

extensions/microsoftexcel-controlnet/annotator/clip/__init__.py

@@ -0,0 +1,39 @@
+import torch
+from transformers import CLIPProcessor, CLIPVisionModel
+from modules import devices
+import os
+from annotator.annotator_path import clip_vision_path
+
+
+remote_model_path = "https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/pytorch_model.bin"
+clip_path = clip_vision_path
+print(f'ControlNet ClipVision location: {clip_path}')
+
+clip_proc = None
+clip_vision_model = None
+
+
+def apply_clip(img):
+    global clip_proc, clip_vision_model
+    
+    if clip_vision_model is None:
+        modelpath = os.path.join(clip_path, 'pytorch_model.bin')
+        if not os.path.exists(modelpath):
+            from basicsr.utils.download_util import load_file_from_url
+            load_file_from_url(remote_model_path, model_dir=clip_path)
+
+        clip_proc = CLIPProcessor.from_pretrained(clip_path)
+        clip_vision_model = CLIPVisionModel.from_pretrained(clip_path)
+
+    with torch.no_grad():
+        clip_vision_model = clip_vision_model.to(devices.get_device_for("controlnet"))
+        style_for_clip = clip_proc(images=img, return_tensors="pt")['pixel_values']
+        style_feat = clip_vision_model(style_for_clip.to(devices.get_device_for("controlnet")))['last_hidden_state']
+
+    return style_feat
+
+
+def unload_clip_model():
+    global clip_proc, clip_vision_model
+    if clip_vision_model is not None:
+        clip_vision_model.cpu()

extensions/microsoftexcel-controlnet/annotator/clip_vision/config.json

@@ -0,0 +1,171 @@
+{
+  "_name_or_path": "clip-vit-large-patch14/",
+  "architectures": [
+    "CLIPModel"
+  ],
+  "initializer_factor": 1.0,
+  "logit_scale_init_value": 2.6592,
+  "model_type": "clip",
+  "projection_dim": 768,
+  "text_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "bos_token_id": 0,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0.0,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "quick_gelu",
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_factor": 1.0,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 77,
+    "min_length": 0,
+    "model_type": "clip_text_model",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 1,
+    "prefix": null,
+    "problem_type": null,
+    "projection_dim" : 768,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.16.0.dev0",
+    "use_bfloat16": false,
+    "vocab_size": 49408
+  },
+  "text_config_dict": {
+    "hidden_size": 768,
+    "intermediate_size": 3072,
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "projection_dim": 768
+  },
+  "torch_dtype": "float32",
+  "transformers_version": null,
+  "vision_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0.0,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "quick_gelu",
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 224,
+    "initializer_factor": 1.0,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "clip_vision_model",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 24,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "projection_dim" : 768,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.16.0.dev0",
+    "use_bfloat16": false
+  },
+  "vision_config_dict": {
+    "hidden_size": 1024,
+    "intermediate_size": 4096,
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "patch_size": 14,
+    "projection_dim": 768
+  }
+}

extensions/microsoftexcel-controlnet/annotator/clip_vision/preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "crop_size": 224,
+  "do_center_crop": true,
+  "do_normalize": true,
+  "do_resize": true,
+  "feature_extractor_type": "CLIPFeatureExtractor",
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "size": 224
+}

extensions/microsoftexcel-controlnet/annotator/clip_vision/tokenizer_config.json

@@ -0,0 +1,34 @@
+{
+    "unk_token": {
+        "content": "<|endoftext|>",
+        "single_word": false,
+        "lstrip": false,
+        "rstrip": false,
+        "normalized": true,
+        "__type": "AddedToken"
+    },
+    "bos_token": {
+        "content": "<|startoftext|>",
+        "single_word": false,
+        "lstrip": false,
+        "rstrip": false,
+        "normalized": true,
+        "__type": "AddedToken"
+    },
+    "eos_token": {
+        "content": "<|endoftext|>",
+        "single_word": false,
+        "lstrip": false,
+        "rstrip": false,
+        "normalized": true,
+        "__type": "AddedToken"
+    },
+    "pad_token": "<|endoftext|>",
+    "add_prefix_space": false,
+    "errors": "replace",
+    "do_lower_case": true,
+    "name_or_path": "openai/clip-vit-base-patch32",
+    "model_max_length": 77,
+    "special_tokens_map_file": "./special_tokens_map.json",
+    "tokenizer_class": "CLIPTokenizer"
+}

extensions/microsoftexcel-controlnet/annotator/color/__init__.py

@@ -0,0 +1,20 @@
+import cv2
+
+def cv2_resize_shortest_edge(image, size):
+    h, w = image.shape[:2]
+    if h < w:
+        new_h = size
+        new_w = int(round(w / h * size))
+    else:
+        new_w = size
+        new_h = int(round(h / w * size))
+    resized_image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA)
+    return resized_image
+
+def apply_color(img, res=512):
+    img = cv2_resize_shortest_edge(img, res)
+    h, w = img.shape[:2]
+
+    input_img_color = cv2.resize(img, (w//64, h//64), interpolation=cv2.INTER_CUBIC)  
+    input_img_color = cv2.resize(input_img_color, (w, h), interpolation=cv2.INTER_NEAREST)
+    return input_img_color

extensions/microsoftexcel-controlnet/annotator/downloads/openpose/body_pose_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25a948c16078b0f08e236bda51a385d855ef4c153598947c28c0d47ed94bb746
+size 209267595

extensions/microsoftexcel-controlnet/annotator/downloads/openpose/facenet.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8beb52e548624ffcc4aed12af7aee7dcbfaeea420c75609fee999fe7add79d43
+size 153718792

extensions/microsoftexcel-controlnet/annotator/downloads/openpose/hand_pose_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b76b00d1750901abd07b9f9d8c98cc3385b8fe834a26d4b4f0aad439e75fc600
+size 147341049

extensions/microsoftexcel-controlnet/annotator/hed/__init__.py

@@ -0,0 +1,98 @@
+# This is an improved version and model of HED edge detection with Apache License, Version 2.0.
+# Please use this implementation in your products
+# This implementation may produce slightly different results from Saining Xie's official implementations,
+# but it generates smoother edges and is more suitable for ControlNet as well as other image-to-image translations.
+# Different from official models and other implementations, this is an RGB-input model (rather than BGR)
+# and in this way it works better for gradio's RGB protocol
+
+import os
+import cv2
+import torch
+import numpy as np
+
+from einops import rearrange
+import os
+from modules import devices
+from annotator.annotator_path import models_path
+from annotator.util import safe_step, nms
+
+
+class DoubleConvBlock(torch.nn.Module):
+    def __init__(self, input_channel, output_channel, layer_number):
+        super().__init__()
+        self.convs = torch.nn.Sequential()
+        self.convs.append(torch.nn.Conv2d(in_channels=input_channel, out_channels=output_channel, kernel_size=(3, 3), stride=(1, 1), padding=1))
+        for i in range(1, layer_number):
+            self.convs.append(torch.nn.Conv2d(in_channels=output_channel, out_channels=output_channel, kernel_size=(3, 3), stride=(1, 1), padding=1))
+        self.projection = torch.nn.Conv2d(in_channels=output_channel, out_channels=1, kernel_size=(1, 1), stride=(1, 1), padding=0)
+
+    def __call__(self, x, down_sampling=False):
+        h = x
+        if down_sampling:
+            h = torch.nn.functional.max_pool2d(h, kernel_size=(2, 2), stride=(2, 2))
+        for conv in self.convs:
+            h = conv(h)
+            h = torch.nn.functional.relu(h)
+        return h, self.projection(h)
+
+
+class ControlNetHED_Apache2(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.norm = torch.nn.Parameter(torch.zeros(size=(1, 3, 1, 1)))
+        self.block1 = DoubleConvBlock(input_channel=3, output_channel=64, layer_number=2)
+        self.block2 = DoubleConvBlock(input_channel=64, output_channel=128, layer_number=2)
+        self.block3 = DoubleConvBlock(input_channel=128, output_channel=256, layer_number=3)
+        self.block4 = DoubleConvBlock(input_channel=256, output_channel=512, layer_number=3)
+        self.block5 = DoubleConvBlock(input_channel=512, output_channel=512, layer_number=3)
+
+    def __call__(self, x):
+        h = x - self.norm
+        h, projection1 = self.block1(h)
+        h, projection2 = self.block2(h, down_sampling=True)
+        h, projection3 = self.block3(h, down_sampling=True)
+        h, projection4 = self.block4(h, down_sampling=True)
+        h, projection5 = self.block5(h, down_sampling=True)
+        return projection1, projection2, projection3, projection4, projection5
+
+
+netNetwork = None
+remote_model_path = "https://huggingface.co/lllyasviel/Annotators/resolve/main/ControlNetHED.pth"
+modeldir = os.path.join(models_path, "hed")
+old_modeldir = os.path.dirname(os.path.realpath(__file__))
+
+
+def apply_hed(input_image, is_safe=False):
+    global netNetwork
+    if netNetwork is None:
+        modelpath = os.path.join(modeldir, "ControlNetHED.pth")
+        old_modelpath = os.path.join(old_modeldir, "ControlNetHED.pth")
+        if os.path.exists(old_modelpath):
+            modelpath = old_modelpath
+        elif not os.path.exists(modelpath):
+            from basicsr.utils.download_util import load_file_from_url
+            load_file_from_url(remote_model_path, model_dir=modeldir)
+        netNetwork = ControlNetHED_Apache2().to(devices.get_device_for("controlnet"))
+        netNetwork.load_state_dict(torch.load(modelpath, map_location='cpu'))
+    netNetwork.to(devices.get_device_for("controlnet")).float().eval()
+
+    assert input_image.ndim == 3
+    H, W, C = input_image.shape
+    with torch.no_grad():
+        image_hed = torch.from_numpy(input_image.copy()).float().to(devices.get_device_for("controlnet"))
+        image_hed = rearrange(image_hed, 'h w c -> 1 c h w')
+        edges = netNetwork(image_hed)
+        edges = [e.detach().cpu().numpy().astype(np.float32)[0, 0] for e in edges]
+        edges = [cv2.resize(e, (W, H), interpolation=cv2.INTER_LINEAR) for e in edges]
+        edges = np.stack(edges, axis=2)
+        edge = 1 / (1 + np.exp(-np.mean(edges, axis=2).astype(np.float64)))
+        if is_safe:
+            edge = safe_step(edge)
+        edge = (edge * 255.0).clip(0, 255).astype(np.uint8)
+        return edge
+
+    
+def unload_hed_model():
+    global netNetwork
+    if netNetwork is not None:
+        netNetwork.cpu()

extensions/microsoftexcel-controlnet/annotator/keypose/__init__.py

@@ -0,0 +1,212 @@
+import numpy as np
+import cv2
+import torch
+
+import os
+from modules import devices
+from annotator.annotator_path import models_path
+
+import mmcv
+from mmdet.apis import inference_detector, init_detector
+from mmpose.apis import inference_top_down_pose_model
+from mmpose.apis import init_pose_model, process_mmdet_results, vis_pose_result
+
+
+def preprocessing(image, device):
+    # Resize
+    scale = 640 / max(image.shape[:2])
+    image = cv2.resize(image, dsize=None, fx=scale, fy=scale)
+    raw_image = image.astype(np.uint8)
+
+    # Subtract mean values
+    image = image.astype(np.float32)
+    image -= np.array(
+        [
+            float(104.008),
+            float(116.669),
+            float(122.675),
+        ]
+    )
+
+    # Convert to torch.Tensor and add "batch" axis
+    image = torch.from_numpy(image.transpose(2, 0, 1)).float().unsqueeze(0)
+    image = image.to(device)
+
+    return image, raw_image
+
+
+def imshow_keypoints(img,
+                     pose_result,
+                     skeleton=None,
+                     kpt_score_thr=0.1,
+                     pose_kpt_color=None,
+                     pose_link_color=None,
+                     radius=4,
+                     thickness=1):
+    """Draw keypoints and links on an image.
+    Args:
+            img (ndarry): The image to draw poses on.
+            pose_result (list[kpts]): The poses to draw. Each element kpts is
+                a set of K keypoints as an Kx3 numpy.ndarray, where each
+                keypoint is represented as x, y, score.
+            kpt_score_thr (float, optional): Minimum score of keypoints
+                to be shown. Default: 0.3.
+            pose_kpt_color (np.array[Nx3]`): Color of N keypoints. If None,
+                the keypoint will not be drawn.
+            pose_link_color (np.array[Mx3]): Color of M links. If None, the
+                links will not be drawn.
+            thickness (int): Thickness of lines.
+    """
+
+    img_h, img_w, _ = img.shape
+    img = np.zeros(img.shape)
+
+    for idx, kpts in enumerate(pose_result):
+        if idx > 1:
+            continue
+        kpts = kpts['keypoints']
+        # print(kpts)
+        kpts = np.array(kpts, copy=False)
+
+        # draw each point on image
+        if pose_kpt_color is not None:
+            assert len(pose_kpt_color) == len(kpts)
+
+            for kid, kpt in enumerate(kpts):
+                x_coord, y_coord, kpt_score = int(kpt[0]), int(kpt[1]), kpt[2]
+
+                if kpt_score < kpt_score_thr or pose_kpt_color[kid] is None:
+                    # skip the point that should not be drawn
+                    continue
+
+                color = tuple(int(c) for c in pose_kpt_color[kid])
+                cv2.circle(img, (int(x_coord), int(y_coord)),
+                           radius, color, -1)
+
+        # draw links
+        if skeleton is not None and pose_link_color is not None:
+            assert len(pose_link_color) == len(skeleton)
+
+            for sk_id, sk in enumerate(skeleton):
+                pos1 = (int(kpts[sk[0], 0]), int(kpts[sk[0], 1]))
+                pos2 = (int(kpts[sk[1], 0]), int(kpts[sk[1], 1]))
+
+                if (pos1[0] <= 0 or pos1[0] >= img_w or pos1[1] <= 0 or pos1[1] >= img_h or pos2[0] <= 0
+                        or pos2[0] >= img_w or pos2[1] <= 0 or pos2[1] >= img_h or kpts[sk[0], 2] < kpt_score_thr
+                        or kpts[sk[1], 2] < kpt_score_thr or pose_link_color[sk_id] is None):
+                    # skip the link that should not be drawn
+                    continue
+                color = tuple(int(c) for c in pose_link_color[sk_id])
+                cv2.line(img, pos1, pos2, color, thickness=thickness)
+
+    return img
+
+
+human_det, pose_model = None, None
+det_model_path = "https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth"
+pose_model_path = "https://download.openmmlab.com/mmpose/top_down/hrnet/hrnet_w48_coco_256x192-b9e0b3ab_20200708.pth"
+
+modeldir = os.path.join(models_path, "keypose")
+old_modeldir = os.path.dirname(os.path.realpath(__file__))
+
+det_config = 'faster_rcnn_r50_fpn_coco.py'
+pose_config = 'hrnet_w48_coco_256x192.py'
+
+det_checkpoint = 'faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth'
+pose_checkpoint = 'hrnet_w48_coco_256x192-b9e0b3ab_20200708.pth'
+det_cat_id = 1
+bbox_thr = 0.2
+
+skeleton = [
+    [15, 13], [13, 11], [16, 14], [14, 12], [11, 12], [5, 11], [6, 12], [5, 6], [5, 7], [6, 8],
+    [7, 9], [8, 10],
+    [1, 2], [0, 1], [0, 2], [1, 3], [2, 4], [3, 5], [4, 6]
+]
+
+pose_kpt_color = [
+    [51, 153, 255], [51, 153, 255], [51, 153, 255], [51, 153, 255], [51, 153, 255],
+    [0, 255, 0],
+    [255, 128, 0], [0, 255, 0], [255, 128, 0], [0, 255, 0], [255, 128, 0], [0, 255, 0],
+    [255, 128, 0],
+    [0, 255, 0], [255, 128, 0], [0, 255, 0], [255, 128, 0]
+]
+
+pose_link_color = [
+    [0, 255, 0], [0, 255, 0], [255, 128, 0], [255, 128, 0],
+    [51, 153, 255], [51, 153, 255], [51, 153, 255], [51, 153, 255], [0, 255, 0],
+    [255, 128, 0],
+    [0, 255, 0], [255, 128, 0], [51, 153, 255], [51, 153, 255], [51, 153, 255],
+    [51, 153, 255],
+    [51, 153, 255], [51, 153, 255], [51, 153, 255]
+]
+
+def find_download_model(checkpoint, remote_path):
+    modelpath = os.path.join(modeldir, checkpoint)
+    old_modelpath = os.path.join(old_modeldir, checkpoint)
+        
+    if os.path.exists(old_modelpath):
+        modelpath = old_modelpath
+    elif not os.path.exists(modelpath):
+        from basicsr.utils.download_util import load_file_from_url
+        load_file_from_url(remote_path, model_dir=modeldir)
+        
+    return modelpath
+
+def apply_keypose(input_image):
+    global human_det, pose_model
+    if netNetwork is None:
+        det_model_local = find_download_model(det_checkpoint, det_model_path)
+        hrnet_model_local = find_download_model(pose_checkpoint, pose_model_path)
+        det_config_mmcv = mmcv.Config.fromfile(det_config)
+        pose_config_mmcv = mmcv.Config.fromfile(pose_config)
+        human_det = init_detector(det_config_mmcv, det_model_local, device=devices.get_device_for("controlnet"))
+        pose_model = init_pose_model(pose_config_mmcv, hrnet_model_local, device=devices.get_device_for("controlnet"))
+
+    assert input_image.ndim == 3
+    input_image = input_image.copy()
+    with torch.no_grad():
+        image = torch.from_numpy(input_image).float().to(devices.get_device_for("controlnet"))
+        image = image / 255.0
+        mmdet_results = inference_detector(human_det, image)
+        
+        # keep the person class bounding boxes.
+        person_results = process_mmdet_results(mmdet_results, det_cat_id)
+        
+        return_heatmap = False
+        dataset = pose_model.cfg.data['test']['type']
+        
+        # e.g. use ('backbone', ) to return backbone feature
+        output_layer_names = None
+        pose_results, _ = inference_top_down_pose_model(
+            pose_model,
+            image,
+            person_results,
+            bbox_thr=bbox_thr,
+            format='xyxy',
+            dataset=dataset,
+            dataset_info=None,
+            return_heatmap=return_heatmap,
+            outputs=output_layer_names
+        )
+        
+        im_keypose_out = imshow_keypoints(
+            image,
+            pose_results,
+            skeleton=skeleton,
+            pose_kpt_color=pose_kpt_color,
+            pose_link_color=pose_link_color,
+            radius=2,
+            thickness=2
+        )
+        im_keypose_out = im_keypose_out.astype(np.uint8)
+
+        # image_hed = rearrange(image_hed, 'h w c -> 1 c h w')
+        # edge = netNetwork(image_hed)[0]
+        # edge = (edge.cpu().numpy() * 255.0).clip(0, 255).astype(np.uint8)
+        return im_keypose_out
+
+
+def unload_hed_model():
+    global netNetwork
+    if netNetwork is not None:
+        netNetwork.cpu()

extensions/microsoftexcel-controlnet/annotator/keypose/faster_rcnn_r50_fpn_coco.py

@@ -0,0 +1,182 @@
+checkpoint_config = dict(interval=1)
+# yapf:disable
+log_config = dict(
+    interval=50,
+    hooks=[
+        dict(type='TextLoggerHook'),
+        # dict(type='TensorboardLoggerHook')
+    ])
+# yapf:enable
+dist_params = dict(backend='nccl')
+log_level = 'INFO'
+load_from = None
+resume_from = None
+workflow = [('train', 1)]
+# optimizer
+optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001)
+optimizer_config = dict(grad_clip=None)
+# learning policy
+lr_config = dict(
+    policy='step',
+    warmup='linear',
+    warmup_iters=500,
+    warmup_ratio=0.001,
+    step=[8, 11])
+total_epochs = 12
+
+model = dict(
+    type='FasterRCNN',
+    pretrained='torchvision://resnet50',
+    backbone=dict(
+        type='ResNet',
+        depth=50,
+        num_stages=4,
+        out_indices=(0, 1, 2, 3),
+        frozen_stages=1,
+        norm_cfg=dict(type='BN', requires_grad=True),
+        norm_eval=True,
+        style='pytorch'),
+    neck=dict(
+        type='FPN',
+        in_channels=[256, 512, 1024, 2048],
+        out_channels=256,
+        num_outs=5),
+    rpn_head=dict(
+        type='RPNHead',
+        in_channels=256,
+        feat_channels=256,
+        anchor_generator=dict(
+            type='AnchorGenerator',
+            scales=[8],
+            ratios=[0.5, 1.0, 2.0],
+            strides=[4, 8, 16, 32, 64]),
+        bbox_coder=dict(
+            type='DeltaXYWHBBoxCoder',
+            target_means=[.0, .0, .0, .0],
+            target_stds=[1.0, 1.0, 1.0, 1.0]),
+        loss_cls=dict(
+            type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0),
+        loss_bbox=dict(type='L1Loss', loss_weight=1.0)),
+    roi_head=dict(
+        type='StandardRoIHead',
+        bbox_roi_extractor=dict(
+            type='SingleRoIExtractor',
+            roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0),
+            out_channels=256,
+            featmap_strides=[4, 8, 16, 32]),
+        bbox_head=dict(
+            type='Shared2FCBBoxHead',
+            in_channels=256,
+            fc_out_channels=1024,
+            roi_feat_size=7,
+            num_classes=80,
+            bbox_coder=dict(
+                type='DeltaXYWHBBoxCoder',
+                target_means=[0., 0., 0., 0.],
+                target_stds=[0.1, 0.1, 0.2, 0.2]),
+            reg_class_agnostic=False,
+            loss_cls=dict(
+                type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0),
+            loss_bbox=dict(type='L1Loss', loss_weight=1.0))),
+    # model training and testing settings
+    train_cfg=dict(
+        rpn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.7,
+                neg_iou_thr=0.3,
+                min_pos_iou=0.3,
+                match_low_quality=True,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=256,
+                pos_fraction=0.5,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=False),
+            allowed_border=-1,
+            pos_weight=-1,
+            debug=False),
+        rpn_proposal=dict(
+            nms_pre=2000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            assigner=dict(
+                type='MaxIoUAssigner',
+                pos_iou_thr=0.5,
+                neg_iou_thr=0.5,
+                min_pos_iou=0.5,
+                match_low_quality=False,
+                ignore_iof_thr=-1),
+            sampler=dict(
+                type='RandomSampler',
+                num=512,
+                pos_fraction=0.25,
+                neg_pos_ub=-1,
+                add_gt_as_proposals=True),
+            pos_weight=-1,
+            debug=False)),
+    test_cfg=dict(
+        rpn=dict(
+            nms_pre=1000,
+            max_per_img=1000,
+            nms=dict(type='nms', iou_threshold=0.7),
+            min_bbox_size=0),
+        rcnn=dict(
+            score_thr=0.05,
+            nms=dict(type='nms', iou_threshold=0.5),
+            max_per_img=100)
+        # soft-nms is also supported for rcnn testing
+        # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05)
+    ))
+
+dataset_type = 'CocoDataset'
+data_root = 'data/coco'
+img_norm_cfg = dict(
+    mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
+train_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='LoadAnnotations', with_bbox=True),
+    dict(type='Resize', img_scale=(1333, 800), keep_ratio=True),
+    dict(type='RandomFlip', flip_ratio=0.5),
+    dict(type='Normalize', **img_norm_cfg),
+    dict(type='Pad', size_divisor=32),
+    dict(type='DefaultFormatBundle'),
+    dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']),
+]
+test_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(
+        type='MultiScaleFlipAug',
+        img_scale=(1333, 800),
+        flip=False,
+        transforms=[
+            dict(type='Resize', keep_ratio=True),
+            dict(type='RandomFlip'),
+            dict(type='Normalize', **img_norm_cfg),
+            dict(type='Pad', size_divisor=32),
+            dict(type='DefaultFormatBundle'),
+            dict(type='Collect', keys=['img']),
+        ])
+]
+data = dict(
+    samples_per_gpu=2,
+    workers_per_gpu=2,
+    train=dict(
+        type=dataset_type,
+        ann_file=f'{data_root}/annotations/instances_train2017.json',
+        img_prefix=f'{data_root}/train2017/',
+        pipeline=train_pipeline),
+    val=dict(
+        type=dataset_type,
+        ann_file=f'{data_root}/annotations/instances_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        pipeline=test_pipeline),
+    test=dict(
+        type=dataset_type,
+        ann_file=f'{data_root}/annotations/instances_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        pipeline=test_pipeline))
+evaluation = dict(interval=1, metric='bbox')

extensions/microsoftexcel-controlnet/annotator/keypose/hrnet_w48_coco_256x192.py

@@ -0,0 +1,169 @@
+# _base_ = [
+#     '../../../../_base_/default_runtime.py',
+#     '../../../../_base_/datasets/coco.py'
+# ]
+evaluation = dict(interval=10, metric='mAP', save_best='AP')
+
+optimizer = dict(
+    type='Adam',
+    lr=5e-4,
+)
+optimizer_config = dict(grad_clip=None)
+# learning policy
+lr_config = dict(
+    policy='step',
+    warmup='linear',
+    warmup_iters=500,
+    warmup_ratio=0.001,
+    step=[170, 200])
+total_epochs = 210
+channel_cfg = dict(
+    num_output_channels=17,
+    dataset_joints=17,
+    dataset_channel=[
+        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
+    ],
+    inference_channel=[
+        0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
+    ])
+
+# model settings
+model = dict(
+    type='TopDown',
+    pretrained='https://download.openmmlab.com/mmpose/'
+    'pretrain_models/hrnet_w48-8ef0771d.pth',
+    backbone=dict(
+        type='HRNet',
+        in_channels=3,
+        extra=dict(
+            stage1=dict(
+                num_modules=1,
+                num_branches=1,
+                block='BOTTLENECK',
+                num_blocks=(4, ),
+                num_channels=(64, )),
+            stage2=dict(
+                num_modules=1,
+                num_branches=2,
+                block='BASIC',
+                num_blocks=(4, 4),
+                num_channels=(48, 96)),
+            stage3=dict(
+                num_modules=4,
+                num_branches=3,
+                block='BASIC',
+                num_blocks=(4, 4, 4),
+                num_channels=(48, 96, 192)),
+            stage4=dict(
+                num_modules=3,
+                num_branches=4,
+                block='BASIC',
+                num_blocks=(4, 4, 4, 4),
+                num_channels=(48, 96, 192, 384))),
+    ),
+    keypoint_head=dict(
+        type='TopdownHeatmapSimpleHead',
+        in_channels=48,
+        out_channels=channel_cfg['num_output_channels'],
+        num_deconv_layers=0,
+        extra=dict(final_conv_kernel=1, ),
+        loss_keypoint=dict(type='JointsMSELoss', use_target_weight=True)),
+    train_cfg=dict(),
+    test_cfg=dict(
+        flip_test=True,
+        post_process='default',
+        shift_heatmap=True,
+        modulate_kernel=11))
+
+data_cfg = dict(
+    image_size=[192, 256],
+    heatmap_size=[48, 64],
+    num_output_channels=channel_cfg['num_output_channels'],
+    num_joints=channel_cfg['dataset_joints'],
+    dataset_channel=channel_cfg['dataset_channel'],
+    inference_channel=channel_cfg['inference_channel'],
+    soft_nms=False,
+    nms_thr=1.0,
+    oks_thr=0.9,
+    vis_thr=0.2,
+    use_gt_bbox=False,
+    det_bbox_thr=0.0,
+    bbox_file='data/coco/person_detection_results/'
+    'COCO_val2017_detections_AP_H_56_person.json',
+)
+
+train_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='TopDownGetBboxCenterScale', padding=1.25),
+    dict(type='TopDownRandomShiftBboxCenter', shift_factor=0.16, prob=0.3),
+    dict(type='TopDownRandomFlip', flip_prob=0.5),
+    dict(
+        type='TopDownHalfBodyTransform',
+        num_joints_half_body=8,
+        prob_half_body=0.3),
+    dict(
+        type='TopDownGetRandomScaleRotation', rot_factor=40, scale_factor=0.5),
+    dict(type='TopDownAffine'),
+    dict(type='ToTensor'),
+    dict(
+        type='NormalizeTensor',
+        mean=[0.485, 0.456, 0.406],
+        std=[0.229, 0.224, 0.225]),
+    dict(type='TopDownGenerateTarget', sigma=2),
+    dict(
+        type='Collect',
+        keys=['img', 'target', 'target_weight'],
+        meta_keys=[
+            'image_file', 'joints_3d', 'joints_3d_visible', 'center', 'scale',
+            'rotation', 'bbox_score', 'flip_pairs'
+        ]),
+]
+
+val_pipeline = [
+    dict(type='LoadImageFromFile'),
+    dict(type='TopDownGetBboxCenterScale', padding=1.25),
+    dict(type='TopDownAffine'),
+    dict(type='ToTensor'),
+    dict(
+        type='NormalizeTensor',
+        mean=[0.485, 0.456, 0.406],
+        std=[0.229, 0.224, 0.225]),
+    dict(
+        type='Collect',
+        keys=['img'],
+        meta_keys=[
+            'image_file', 'center', 'scale', 'rotation', 'bbox_score',
+            'flip_pairs'
+        ]),
+]
+
+test_pipeline = val_pipeline
+
+data_root = 'data/coco'
+data = dict(
+    samples_per_gpu=32,
+    workers_per_gpu=2,
+    val_dataloader=dict(samples_per_gpu=32),
+    test_dataloader=dict(samples_per_gpu=32),
+    train=dict(
+        type='TopDownCocoDataset',
+        ann_file=f'{data_root}/annotations/person_keypoints_train2017.json',
+        img_prefix=f'{data_root}/train2017/',
+        data_cfg=data_cfg,
+        pipeline=train_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+    val=dict(
+        type='TopDownCocoDataset',
+        ann_file=f'{data_root}/annotations/person_keypoints_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        data_cfg=data_cfg,
+        pipeline=val_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+    test=dict(
+        type='TopDownCocoDataset',
+        ann_file=f'{data_root}/annotations/person_keypoints_val2017.json',
+        img_prefix=f'{data_root}/val2017/',
+        data_cfg=data_cfg,
+        pipeline=test_pipeline,
+        dataset_info={{_base_.dataset_info}}),
+)

extensions/microsoftexcel-controlnet/annotator/leres/__init__.py

@@ -0,0 +1,113 @@
+import cv2
+import numpy as np
+import torch
+import os
+from modules import devices, shared
+from annotator.annotator_path import models_path
+from torchvision.transforms import transforms
+
+# AdelaiDepth/LeReS imports
+from .leres.depthmap import estimateleres, estimateboost
+from .leres.multi_depth_model_woauxi import RelDepthModel
+from .leres.net_tools import strip_prefix_if_present
+
+# pix2pix/merge net imports
+from .pix2pix.options.test_options import TestOptions
+from .pix2pix.models.pix2pix4depth_model import Pix2Pix4DepthModel
+
+base_model_path = os.path.join(models_path, "leres")
+old_modeldir = os.path.dirname(os.path.realpath(__file__))
+
+remote_model_path_leres = "https://huggingface.co/lllyasviel/Annotators/resolve/main/res101.pth"
+remote_model_path_pix2pix = "https://huggingface.co/lllyasviel/Annotators/resolve/main/latest_net_G.pth"
+
+model = None
+pix2pixmodel = None
+
+def unload_leres_model():
+    global model, pix2pixmodel
+    if model is not None:
+        model = model.cpu()
+    if pix2pixmodel is not None:
+        pix2pixmodel = pix2pixmodel.unload_network('G')
+
+
+def apply_leres(input_image, thr_a, thr_b, boost=False):
+    global model, pix2pixmodel
+    if model is None:
+        model_path = os.path.join(base_model_path, "res101.pth")
+        old_model_path = os.path.join(old_modeldir, "res101.pth")
+        
+        if os.path.exists(old_model_path):
+            model_path = old_model_path
+        elif not os.path.exists(model_path):
+            from basicsr.utils.download_util import load_file_from_url
+            load_file_from_url(remote_model_path_leres, model_dir=base_model_path)
+
+        if torch.cuda.is_available():
+            checkpoint = torch.load(model_path)
+        else:
+            checkpoint = torch.load(model_path, map_location=torch.device('cpu'))
+
+        model = RelDepthModel(backbone='resnext101')
+        model.load_state_dict(strip_prefix_if_present(checkpoint['depth_model'], "module."), strict=True)
+        del checkpoint
+
+    if boost and pix2pixmodel is None:
+        pix2pixmodel_path = os.path.join(base_model_path, "latest_net_G.pth")
+        if not os.path.exists(pix2pixmodel_path):
+            from basicsr.utils.download_util import load_file_from_url
+            load_file_from_url(remote_model_path_pix2pix, model_dir=base_model_path)
+
+        opt = TestOptions().parse()
+        if not torch.cuda.is_available():
+            opt.gpu_ids = []  # cpu mode
+        pix2pixmodel = Pix2Pix4DepthModel(opt)
+        pix2pixmodel.save_dir = base_model_path
+        pix2pixmodel.load_networks('latest')
+        pix2pixmodel.eval()
+    
+    if devices.get_device_for("controlnet").type != 'mps':
+        model = model.to(devices.get_device_for("controlnet"))
+
+    assert input_image.ndim == 3
+    height, width, dim = input_image.shape
+
+    with torch.no_grad():
+
+        if boost:
+            depth = estimateboost(input_image, model, 0, pix2pixmodel, max(width, height))
+        else:
+            depth = estimateleres(input_image, model, width, height)
+
+        numbytes=2
+        depth_min = depth.min()
+        depth_max = depth.max()
+        max_val = (2**(8*numbytes))-1
+
+        # check output before normalizing and mapping to 16 bit
+        if depth_max - depth_min > np.finfo("float").eps:
+            out = max_val * (depth - depth_min) / (depth_max - depth_min)
+        else:
+            out = np.zeros(depth.shape)
+        
+        # single channel, 16 bit image
+        depth_image = out.astype("uint16")
+
+        # convert to uint8
+        depth_image = cv2.convertScaleAbs(depth_image, alpha=(255.0/65535.0))
+
+        # remove near
+        if thr_a != 0:
+            thr_a = ((thr_a/100)*255) 
+            depth_image = cv2.threshold(depth_image, thr_a, 255, cv2.THRESH_TOZERO)[1]
+
+        # invert image
+        depth_image = cv2.bitwise_not(depth_image)
+
+        # remove bg
+        if thr_b != 0:
+            thr_b = ((thr_b/100)*255)
+            depth_image = cv2.threshold(depth_image, thr_b, 255, cv2.THRESH_TOZERO)[1]
+
+        return depth_image