Repository: Mikubill/sd-webui-controlnet (language: python)

Issue:
[DevTask] Support InstantStyle
Project link: https://github.com/InstantStyle/InstantStyle


Discussion / hints:

Fix patch:
diff --git a/internal_controlnet/external_code.py b/internal_controlnet/external_code.py
index df678a495..4e33a3e20 100644
--- a/internal_controlnet/external_code.py
+++ b/internal_controlnet/external_code.py
@@ -195,6 +195,9 @@ class ControlNetUnit:
     # For T2IAdapter
     # - SD1.5: 5 weights (4 encoder block + 1 middle block)
     # - SDXL: 4 weights (3 encoder block + 1 middle block)
+    # For IPAdapter
+    # - SD15: 16 (6 input blocks + 9 output blocks + 1 middle block)
+    # - SDXL: 11 weights (4 input blocks + 6 output blocks + 1 middle block)
     # Note1: Setting advanced weighting will disable `soft_injection`, i.e.
     # It is recommended to set ControlMode = BALANCED when using `advanced_weighting`.
     # Note2: The field `weight` is still used in some places, e.g. reference_only,
diff --git a/requirements.txt b/requirements.txt
index f48dab3d9..5fb5e2e2b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,4 +4,5 @@ opencv-python>=4.8.0
 svglib
 addict
 yapf
-albumentations==1.4.3
\ No newline at end of file
+albumentations==1.4.3
+matplotlib
diff --git a/scripts/controlnet.py b/scripts/controlnet.py
index 00af0798e..f28e41631 100644
--- a/scripts/controlnet.py
+++ b/scripts/controlnet.py
@@ -927,7 +927,7 @@ def controlnet_main_entry(self, p):
         logger.info(f"unit_separate = {batch_option_uint_separate}, style_align = {batch_option_style_align}")
 
         detected_maps = []
-        forward_params = []
+        forward_params: List[ControlParams] = []
         post_processors = []
 
         # cache stuff
@@ -1177,10 +1177,21 @@ def recolor_intensity_post_processing(x, i):
 
         for i, param in enumerate(forward_params):
             if param.control_model_type == ControlModelType.IPAdapter:
+                if param.advanced_weighting is not None:
+                    logger.info(f"IP-Adapter using advanced weighting {param.advanced_weighting}")
+                    assert len(param.advanced_weighting) == global_state.get_sd_version().transformer_block_num
+                    # Convert advanced weighting list to dict
+                    weight = {
+                        i: w
+                        for i, w in enumerate(param.advanced_weighting)
+                        if w > 0
+                    }
+                else:
+                    weight = param.weight
                 param.control_model.hook(
                     model=unet,
                     preprocessor_outputs=param.hint_cond,
-                    weight=param.weight,
+                    weight=weight,
                     dtype=torch.float32,
                     start=param.start_guidance_percent,
                     end=param.stop_guidance_percent
diff --git a/scripts/controlnet_ui/advanced_weight_control.py b/scripts/controlnet_ui/advanced_weight_control.py
new file mode 100644
index 000000000..b65192a7d
--- /dev/null
+++ b/scripts/controlnet_ui/advanced_weight_control.py
@@ -0,0 +1,230 @@
+import gradio as gr
+import matplotlib.pyplot as plt
+from matplotlib.patches import Patch
+import io
+import json
+from PIL import Image
+from typing import List
+
+from scripts.enums import StableDiffusionVersion
+from scripts.global_state import get_sd_version
+from scripts.ipadapter.weight import calc_weights
+
+
+INPUT_BLOCK_COLOR = "#61bdee"
+MIDDLE_BLOCK_COLOR = "#e2e2e2"
+OUTPUT_BLOCK_COLOR = "#dc6e55"
+
+
+def get_bar_colors(
+    sd_version: StableDiffusionVersion, input_color, middle_color, output_color
+):
+    middle_block_idx = 4 if sd_version == StableDiffusionVersion.SDXL else 6
+
+    def get_color(idx):
+        if idx < middle_block_idx:
+            return input_color
+        elif idx == middle_block_idx:
+            return middle_color
+        else:
+            return output_color
+
+    return [get_color(i) for i in range(sd_version.transformer_block_num)]
+
+
+def plot_weights(
+    numbers: List[float],
+    colors: List[str],
+):
+    # Create a bar chart
+    plt.figure(figsize=(8, 4))
+    plt.bar(range(len(numbers)), numbers, color=colors)
+    plt.xlabel("Transformer Index")
+    plt.ylabel("Weight")
+    plt.legend(
+        handles=[
+            Patch(color=color, label=label)
+            for color, label in (
+                (INPUT_BLOCK_COLOR, "Input Block"),
+                (MIDDLE_BLOCK_COLOR, "Middle Block"),
+                (OUTPUT_BLOCK_COLOR, "Output Block"),
+            )
+        ],
+        loc="best",
+    )
+
+    # Save the plot to a BytesIO buffer
+    buffer = io.BytesIO()
+    plt.savefig(buffer, format="png")
+    plt.close()
+    buffer.seek(0)
+
+    # Convert the buffer to a PIL image and return it
+    image = Image.open(buffer)
+    return image
+
+
+class AdvancedWeightControl:
+    def __init__(self):
+        self.group = None
+        self.weight_type = None
+        self.weight_plot = None
+        self.weight_editor = None
+        self.weight_composition = None
+
+    def render(self):
+        with gr.Group(visible=False) as self.group:
+            with gr.Row():
+                self.weight_type = gr.Dropdown(
+                    choices=[
+                        "normal",
+                        "ease in",
+                        "ease out",
+                        "ease in-out",
+                        "reverse in-out",
+                        "weak input",
+                        "weak output",
+                        "weak middle",
+                        "strong middle",
+                        "style transfer",
+                        "composition",
+                        "strong style transfer",
+                        "style and composition",
+                        "strong style and composition",
+                    ],
+                    label="Weight Type",
+                    value="normal",
+                )
+                self.weight_composition = gr.Slider(
+                    label="Composition Weight",
+                    minimum=0,
+                    maximum=2.0,
+                    value=1.0,
+                    step=0.01,
+                    visible=False,
+                )
+                self.weight_editor = gr.Textbox(label="Weights", visible=False)
+
+            self.weight_plot = gr.Image(
+                value=None,
+                label="Weight Plot",
+                interactive=False,
+                visible=False,
+            )
+
+    def register_callbacks(
+        self,
+        weight_input: gr.Slider,
+        advanced_weighting: gr.State,
+        control_type: gr.Radio,
+        update_unit_counter: gr.Number,
+    ):
+        def advanced_weighting_supported(control_type: str) -> bool:
+            return control_type in ("IP-Adapter", "Instant-ID")
+
+        self.weight_type.change(
+            fn=lambda weight_type: gr.update(
+                visible=weight_type
+                in ("style and composition", "strong style and composition")
+            ),
+            inputs=[self.weight_type],
+            outputs=[self.weight_composition],
+        )
+
+        def update_weight_textbox(
+            control_type: str,
+            weight_type: str,
+            weight: float,
+            weight_composition: float,
+        ):
+            if not advanced_weighting_supported(control_type):
+                return gr.update()
+
+            sd_version = get_sd_version()
+            weights = calc_weights(weight_type, weight, sd_version, weight_composition)
+            return gr.update(value=str([round(w, 2) for w in weights]), visible=True)
+
+        trigger_inputs = [self.weight_type, weight_input, self.weight_composition]
+        for trigger_input in trigger_inputs:
+            trigger_input.change(
+                fn=update_weight_textbox,
+                inputs=[
+                    control_type,
+                    self.weight_type,
+                    weight_input,
+                    self.weight_composition,
+                ],
+                outputs=[self.weight_editor],
+            )
+
+        def update_plot(weights_string: str):
+            try:
+                weights = json.loads(weights_string)
+                assert isinstance(weights, list)
+            except Exception:
+                return gr.update(visible=False)
+
+            sd_version = get_sd_version()
+            weight_plot = plot_weights(
+                weights,
+                get_bar_colors(
+                    sd_version,
+                    input_color=INPUT_BLOCK_COLOR,
+                    middle_color=MIDDLE_BLOCK_COLOR,
+                    output_color=OUTPUT_BLOCK_COLOR,
+                ),
+            )
+            return gr.update(value=weight_plot, visible=True)
+
+        def update_advanced_weighting(weights_string: str):
+            try:
+                weights = json.loads(weights_string)
+                assert isinstance(weights, list)
+            except Exception:
+                return None
+            return weights
+
+        self.weight_editor.change(
+            fn=update_plot,
+            inputs=[self.weight_editor],
+            outputs=[self.weight_plot],
+        )
+
+        self.weight_editor.change(
+            fn=update_advanced_weighting,
+            inputs=[self.weight_editor],
+            outputs=[advanced_weighting],
+        ).then(
+            fn=lambda x: gr.update(value=x + 1),
+            inputs=[update_unit_counter],
+            outputs=[update_unit_counter],
+        )  # Necessary to flush gr.State change to unit state.
+
+        # TODO: Expose advanced weighting control for other control types.
+        def control_type_change(control_type: str, old_weights):
+            supported = advanced_weighting_supported(control_type)
+            if supported:
+                return (
+                    gr.update(visible=supported),
+                    old_weights,
+                    gr.update(),
+                    gr.update(),
+                )
+            else:
+                return (
+                    gr.update(visible=supported),
+                    None,
+                    gr.update(visible=False),
+                    gr.update(visible=False),
+                )
+
+        control_type.change(
+            fn=control_type_change,
+            inputs=[control_type, advanced_weighting],
+            outputs=[
+                self.group,
+                advanced_weighting,
+                self.weight_editor,
+                self.weight_plot,
+            ],
+        )
diff --git a/scripts/controlnet_ui/controlnet_ui_group.py b/scripts/controlnet_ui/controlnet_ui_group.py
index e6613db1d..46cc697d6 100644
--- a/scripts/controlnet_ui/controlnet_ui_group.py
+++ b/scripts/controlnet_ui/controlnet_ui_group.py
@@ -18,6 +18,7 @@
 from scripts.controlnet_ui.preset import ControlNetPresetUI
 from scripts.controlnet_ui.tool_button import ToolButton
 from scripts.controlnet_ui.photopea import Photopea
+from scripts.controlnet_ui.advanced_weight_control import AdvancedWeightControl
 from scripts.enums import InputMode
 from modules import shared
 from modules.ui_components import FormRow
@@ -289,6 +290,7 @@ def __init__(
         self.input_mode = gr.State(InputMode.SIMPLE)
         self.inpaint_crop_input_image = None
         self.hr_option = None
+        self.advanced_weight_control = AdvancedWeightControl()
         self.batch_image_dir_state = None
         self.output_dir_state = None
 
@@ -640,6 +642,8 @@ def render(self, tabname: str, elem_id_tabname: str) -> None:
             visible=False,
         )
 
+        self.advanced_weight_control.render()
+
         self.preset_panel = ControlNetPresetUI(
             id_prefix=f"{elem_id_tabname}_{tabname}_"
         )
@@ -675,6 +679,8 @@ def render(self, tabname: str, elem_id_tabname: str) -> None:
             self.control_mode,
             self.inpaint_crop_input_image,
             self.hr_option,
+            self.save_detected_map,
+            self.advanced_weighting,
         )
 
         unit = gr.State(self.default_unit)
@@ -1256,6 +1262,12 @@ def register_core_callbacks(self):
                 for key in vars(external_code.ControlNetUnit()).keys()
             ],
         )
+        self.advanced_weight_control.register_callbacks(
+            self.weight,
+            self.advanced_weighting,
+            self.type_filter,
+            self.update_unit_counter,
+        )
         if self.is_img2img:
             self.register_img2img_same_input()
 
diff --git a/scripts/enums.py b/scripts/enums.py
index 066d53b2c..b9e02ef34 100644
--- a/scripts/enums.py
+++ b/scripts/enums.py
@@ -1,5 +1,38 @@
 from enum import Enum
-from typing import Any
+from typing import Any, List, NamedTuple
+from functools import lru_cache
+
+
+class UnetBlockType(Enum):
+    INPUT = "input"
+    OUTPUT = "output"
+    MIDDLE = "middle"
+
+
+class TransformerID(NamedTuple):
+    block_type: UnetBlockType
+    # The id of the block the transformer is in. Not all blocks have cross attn.
+    block_id: int
+    # The index of transformer within the block.
+    # A block can have multiple transformers in SDXL.
+    block_index: int
+    # The call index of transformer if in a single step of diffusion.
+    transformer_index: int
+
+
+class TransformerIDResult(NamedTuple):
+    input_ids: List[TransformerID]
+    output_ids: List[TransformerID]
+    middle_ids: List[TransformerID]
+
+    def get(self, idx: int) -> TransformerID:
+        return self.to_list()[idx]
+
+    def to_list(self) -> List[TransformerID]:
+        return sorted(
+            self.input_ids + self.output_ids + self.middle_ids,
+            key=lambda i: i.transformer_index,
+        )
 
 
 class StableDiffusionVersion(Enum):
@@ -27,19 +60,96 @@ def detect_from_model_name(model_name: str) -> "StableDiffusionVersion":
         return StableDiffusionVersion.UNKNOWN
 
     def encoder_block_num(self) -> int:
-        if self in (StableDiffusionVersion.SD1x, StableDiffusionVersion.SD2x, StableDiffusionVersion.UNKNOWN):
+        if self in (
+            StableDiffusionVersion.SD1x,
+            StableDiffusionVersion.SD2x,
+            StableDiffusionVersion.UNKNOWN,
+        ):
             return 12
         else:
-            return 9 # SDXL
+            return 9  # SDXL
 
     def controlnet_layer_num(self) -> int:
         return self.encoder_block_num() + 1
 
+    @property
+    def transformer_block_num(self) -> int:
+        """Number of blocks that has cross attn transformers in unet."""
+        if self in (
+            StableDiffusionVersion.SD1x,
+            StableDiffusionVersion.SD2x,
+            StableDiffusionVersion.UNKNOWN,
+        ):
+            return 16
+        else:
+            return 11  # SDXL
+
+    @property
+    @lru_cache(maxsize=None)
+    def transformer_ids(self) -> List[TransformerID]:
+        """id of blocks that have cross attention"""
+        if self in (
+            StableDiffusionVersion.SD1x,
+            StableDiffusionVersion.SD2x,
+            StableDiffusionVersion.UNKNOWN,
+        ):
+            transformer_index = 0
+            input_ids = []
+            for block_id in [1, 2, 4, 5, 7, 8]:
+                input_ids.append(
+                    TransformerID(UnetBlockType.INPUT, block_id, 0, transformer_index)
+                )
+                transformer_index += 1
+            middle_id = TransformerID(UnetBlockType.MIDDLE, 0, 0, transformer_index)
+            transformer_index += 1
+            output_ids = []
+            for block_id in [3, 4, 5, 6, 7, 8, 9, 10, 11]:
+                input_ids.append(
+                    TransformerID(UnetBlockType.OUTPUT, block_id, 0, transformer_index)
+                )
+                transformer_index += 1
+            return TransformerIDResult(input_ids, output_ids, [middle_id])
+        else:
+            # SDXL
+            transformer_index = 0
+            input_ids = []
+            for block_id in [4, 5, 7, 8]:
+                block_indices = (
+                    range(2) if block_id in [4, 5] else range(10)
+                )  # transformer_depth
+                for index in block_indices:
+                    input_ids.append(
+                        TransformerID(
+                            UnetBlockType.INPUT, block_id, index, transformer_index
+                        )
+                    )
+                transformer_index += 1
+
+            middle_ids = [
+                TransformerID(UnetBlockType.MIDDLE, 0, index, transformer_index)
+                for index in range(10)
+            ]
+            transformer_index += 1
+
+            output_ids = []
+            for block_id in range(6):
+                block_indices = (
+                    range(2) if block_id in [3, 4, 5] else range(10)
+                )  # transformer_depth
+                for index in block_indices:
+                    output_ids.append(
+                        TransformerID(
+                            UnetBlockType.OUTPUT, block_id, index, transformer_index
+                        )
+                    )
+                transformer_index += 1
+            return TransformerIDResult(input_ids, output_ids, middle_ids)
+
     def is_compatible_with(self, other: "StableDiffusionVersion") -> bool:
-        """ Incompatible only when one of version is SDXL and other is not. """
+        """Incompatible only when one of version is SDXL and other is not."""
         return (
-            any(v == StableDiffusionVersion.UNKNOWN for v in [self, other]) or
-            sum(v == StableDiffusionVersion.SDXL for v in [self, other]) != 1
+            any(v == StableDiffusionVersion.UNKNOWN for v in [self, other])
+            or sum(v == StableDiffusionVersion.SDXL for v in [self, other]) != 1
         )
 
 
@@ -84,6 +194,7 @@ def allow_context_sharing(self) -> bool:
             ControlModelType.Controlllite,
         )
 
+
 # Written by Lvmin
 class AutoMachine(Enum):
     """
diff --git a/scripts/ipadapter/plugable_ipadapter.py b/scripts/ipadapter/plugable_ipadapter.py
index 5270c964d..abff06892 100644
--- a/scripts/ipadapter/plugable_ipadapter.py
+++ b/scripts/ipadapter/plugable_ipadapter.py
@@ -1,6 +1,9 @@
+import itertools
 import torch
+from typing import Union, Dict
 
 from .ipadapter_model import ImageEmbed, IPAdapterModel
+from ..enums import StableDiffusionVersion, TransformerID
 
 
 def get_block(model, flag):
@@ -61,21 +64,19 @@ def hack_blk(block, function, type):
     return
 
 
-def set_model_attn2_replace(model, function, flag, id):
-    from ldm.modules.attention import CrossAttention
-
-    block = get_block(model, flag)[id][1].transformer_blocks[0].attn2
-    hack_blk(block, function, CrossAttention)
-    return
-
-
-def set_model_patch_replace(model, function, flag, id, trans_id):
-    from sgm.modules.attention import CrossAttention
-
-    blk = get_block(model, flag)
-    block = blk[id][1].transformer_blocks[trans_id].attn2
-    hack_blk(block, function, CrossAttention)
-    return
+def set_model_attn2_replace(
+    model,
+    target_cls,
+    function,
+    transformer_id: TransformerID,
+):
+    block = get_block(model, transformer_id.block_type.value)
+    module = (
+        block[transformer_id.block_id][1]
+        .transformer_blocks[transformer_id.block_index]
+        .attn2
+    )
+    hack_blk(module, function, target_cls)
 
 
 def clear_all_ip_adapter():
@@ -94,7 +95,7 @@ def __init__(self, ipadapter: IPAdapterModel):
         self.ipadapter = ipadapter
         self.disable_memory_management = True
         self.dtype = None
-        self.weight = 1.0
+        self.weight: Union[float, Dict[int, float]] = 1.0
         self.cache = None
         self.p_start = 0.0
         self.p_end = 1.0
@@ -126,60 +127,31 @@ def hook(
         self.image_emb = ImageEmbed.average_of(
             *[self.ipadapter.get_image_emb(o) for o in preprocessor_outputs]
         )
-        # From https://github.com/laksjdjf/IPAdapter-ComfyUI
-        if not self.ipadapter.is_sdxl:
-            number = 0  # index of to_kvs
-            for id in [
-                1,
-                2,
-                4,
-                5,
-                7,
-                8,
-            ]:  # id of input_blocks that have cross attention
-                set_model_attn2_replace(model, self.patch_forward(number), "input", id)
-                number += 1
-            for id in [
-                3,
-                4,
-                5,
-                6,
-                7,
-                8,
-                9,
-                10,
-                11,
-            ]:  # id of output_blocks that have cross attention
-                set_model_attn2_replace(model, self.patch_forward(number), "output", id)
-                number += 1
-            set_model_attn2_replace(model, self.patch_forward(number), "middle", 0)
+
+        if self.ipadapter.is_sdxl:
+            sd_version = StableDiffusionVersion.SDXL
+            from sgm.modules.attention import CrossAttention
+        else:
+            sd_version = StableDiffusionVersion.SD1x
+            from ldm.modules.attention import CrossAttention
+
+        input_ids, output_ids, middle_ids = sd_version.transformer_ids
+        for i, transformer_id in enumerate(
+            itertools.chain(input_ids, output_ids, middle_ids)
+        ):
+            set_model_attn2_replace(
+                model,
+                CrossAttention,
+                self.patch_forward(i, transformer_id.transformer_index),
+                transformer_id,
+            )
+
+    def weight_on_transformer(self, transformer_index: int) -> float:
+        if isinstance(self.weight, dict):
+            return self.weight.get(transformer_index, 0.0)
         else:
-            number = 0
-            for id in [4, 5, 7, 8]:  # id of input_blocks that have cross attention
-                block_indices = (
-                    range(2) if id in [4, 5] else range(10)
-                )  # transformer_depth
-                for index in block_indices:
-                    set_model_patch_replace(
-                        model, self.patch_forward(number), "input", id, index
-                    )
-                    number += 1
-            for id in range(6):  # id of output_blocks that have cross attention
-                block_indices = (
-                    range(2) if id in [3, 4, 5] else range(10)
-                )  # transformer_depth
-                for index in block_indices:
-                    set_model_patch_replace(
-                        model, self.patch_forward(number), "output", id, index
-                    )
-                    number += 1
-            for index in range(10):
-                set_model_patch_replace(
-                    model, self.patch_forward(number), "middle", 0, index
-                )
-                number += 1
-
-        return
+            assert isinstance(self.weight, (float, int))
+            return self.weight
 
     def call_ip(self, key: str, feat, device):
         if key in self.cache:
@@ -190,12 +162,13 @@ def call_ip(self, key: str, feat, device):
             return ip
 
     @torch.no_grad()
-    def patch_forward(self, number: int):
+    def patch_forward(self, number: int, transformer_index: int):
         @torch.no_grad()
         def forward(attn_blk, x, q):
             batch_size, sequence_length, inner_dim = x.shape
             h = attn_blk.heads
             head_dim = inner_dim // h
+            weight = self.weight_on_transformer(transformer_index)
 
             current_sampling_percent = getattr(
                 current_model, "current_sampling_percent", 0.5
@@ -203,8 +176,9 @@ def forward(attn_blk, x, q):
             if (
                 current_sampling_percent < self.p_start
                 or current_sampling_percent > self.p_end
+                or weight == 0.0
             ):
-                return 0
+                return 0.0
 
             k_key = f"{number * 2 + 1}_to_k_ip"
             v_key = f"{number * 2 + 1}_to_v_ip"
@@ -229,6 +203,6 @@ def forward(attn_blk, x, q):
             )
             ip_out = ip_out.transpose(1, 2).reshape(batch_size, -1, h * head_dim)
 
-            return ip_out * self.weight
+            return ip_out * weight
 
         return forward
diff --git a/scripts/ipadapter/weight.py b/scripts/ipadapter/weight.py
new file mode 100644
index 000000000..12949c2fc
--- /dev/null
+++ b/scripts/ipadapter/weight.py
@@ -0,0 +1,170 @@
+# From https://github.com/cubiq/ComfyUI_IPAdapter_plus
+
+from ..enums import StableDiffusionVersion, TransformerIDResult
+
+
+def calc_weights(
+    weight_type: str,
+    weight: float,
+    sd_version: StableDiffusionVersion,
+    weight_composition: float = 0.0,
+):
+    layers = sd_version.transformer_block_num
+    return [
+        _calc_weight(
+            weight_type,
+            weight,
+            sd_version,
+            t_idx,
+            weight_composition,
+        )
+        for t_idx in range(layers)
+    ]
+
+
+def _calc_weight(
+    weight_type: str,
+    weight: float,
+    sd_version: StableDiffusionVersion,
+    t_idx: int,
+    weight_composition: float = 0.0,
+):
+    is_sdxl = sd_version == StableDiffusionVersion.SDXL
+    layers = sd_version.transformer_block_num
+    ids: TransformerIDResult = sd_version.transformer_ids
+    block_type = [i for i in ids.to_list() if i.transformer_index == t_idx][
+        0
+    ].block_type.value
+    if weight_type == "normal":
+        return weight
+    elif weight_type == "ease in":
+        weight = weight * (0.05 + 0.95 * (1 - t_idx / layers))
+    elif weight_type == "ease out":
+        weight = weight * (0.05 + 0.95 * (t_idx / layers))
+    elif weight_type == "ease in-out":
+        weight = weight * (0.05 + 0.95 * (1 - abs(t_idx - (layers / 2)) / (layers / 2)))
+    elif weight_type == "reverse in-out":
+        weight = weight * (0.05 + 0.95 * (abs(t_idx - (layers / 2)) / (layers / 2)))
+    elif weight_type == "weak input":
+        weight = weight * 0.2 if block_type == "input" else weight
+    elif weight_type == "weak middle":
+        weight = weight * 0.2 if block_type == "middle" else weight
+    elif weight_type == "weak output":
+        weight = weight * 0.2 if block_type == "output" else weight
+    elif weight_type == "strong middle":
+        weight = (
+            weight * 0.2
+            if (block_type == "input" or block_type == "output")
+            else weight
+        )
+    elif weight_type.startswith("style transfer"):
+        weight = (
+            {6: weight}
+            if is_sdxl
+            else {
+                0: weight,
+                1: weight,
+                2: weight,
+                3: weight,
+                9: weight,
+                10: weight,
+                11: weight,
+                12: weight,
+                13: weight,
+                14: weight,
+                15: weight,
+            }
+        )
+    elif weight_type.startswith("composition"):
+        weight = {3: weight} if is_sdxl else {4: weight * 0.25, 5: weight}
+    elif weight_type == "strong style transfer":
+        if is_sdxl:
+            weight = {
+                0: weight,
+                1: weight,
+                2: weight,
+                4: weight,
+                5: weight,
+                6: weight,
+                7: weight,
+                8: weight,
+                9: weight,
+                10: weight,
+            }
+        else:
+            weight = {
+                0: weight,
+                1: weight,
+                2: weight,
+                3: weight,
+                6: weight,
+                7: weight,
+                8: weight,
+                9: weight,
+                10: weight,
+                11: weight,
+                12: weight,
+                13: weight,
+                14: weight,
+                15: weight,
+            }
+    elif weight_type == "style and composition":
+        if is_sdxl:
+            weight = {3: weight_composition, 6: weight}
+        else:
+            weight = {
+                0: weight,
+                1: weight,
+                2: weight,
+                3: weight,
+                4: weight_composition * 0.25,
+                5: weight_composition,
+                9: weight,
+                10: weight,
+                11: weight,
+                12: weight,
+                13: weight,
+                14: weight,
+                15: weight,
+            }
+    elif weight_type == "strong style and composition":
+        if is_sdxl:
+            weight = {
+                0: weight,
+                1: weight,
+                2: weight,
+                3: weight_composition,
+                4: weight,
+                5: weight,
+                6: weight,
+                7: weight,
+                8: weight,
+                9: weight,
+                10: weight,
+            }
+        else:
+            weight = {
+                0: weight,
+                1: weight,
+                2: weight,
+                3: weight,
+                4: weight_composition,
+                5: weight_composition,
+                6: weight,
+                7: weight,
+                8: weight,
+                9: weight,
+                10: weight,
+                11: weight,
+                12: weight,
+                13: weight,
+                14: weight,
+                15: weight,
+            }
+    else:
+        raise Exception("Unrecognized weight type!")
+
+    if isinstance(weight, dict):
+        return weight.get(t_idx, 0.0)
+    else:
+        return weight


Test patch:
diff --git a/tests/web_api/ipadapter_advanced_weighting.py b/tests/web_api/ipadapter_advanced_weighting.py
new file mode 100644
index 000000000..981c97ab7
--- /dev/null
+++ b/tests/web_api/ipadapter_advanced_weighting.py
@@ -0,0 +1,43 @@
+from .template import (
+    APITestTemplate,
+    realistic_girl_face_img,
+    disable_in_cq,
+    get_model,
+)
+
+
+@disable_in_cq
+def test_ipadapter_advanced_weighting():
+    weights = [0.0] * 16  # 16 weights for SD15 / 11 weights for SDXL
+    # SD15 composition
+    weights[4] = 0.25
+    weights[5] = 1.0
+
+    APITestTemplate(
+        "test_ipadapter_advanced_weighting",
+        "txt2img",
+        payload_overrides={
+            "width": 512,
+            "height": 512,
+        },
+        unit_overrides={
+            "image": realistic_girl_face_img,
+            "module": "ip-adapter-auto",
+            "model": get_model("ip-adapter_sd15"),
+            "advanced_weighting": weights,
+        },
+    ).exec()
+
+    APITestTemplate(
+        "test_ipadapter_advanced_weighting_ref",
+        "txt2img",
+        payload_overrides={
+            "width": 512,
+            "height": 512,
+        },
+        unit_overrides={
+            "image": realistic_girl_face_img,
+            "module": "ip-adapter-auto",
+            "model": get_model("ip-adapter_sd15"),
+        },
+    ).exec()