Repository: huggingface/diffusers
File: src/diffusers/loaders/lora_conversion_utils.py (Python)

Code before:
            original_state_dict.pop(key)
            logger.debug(f"Removing {key} key from the state dict as it is a norm diff key. This is unsupported.")

        if "time_projection" in key and not has_time_projection_weight:
            # AccVideo lora has diff bias keys but not the weight keys. This causes a weird problem where
            # our lora config adds the time proj lora layers, but we don't have the weights for them.
            # CausVid lora has the weight keys and the bias keys.
            original_state_dict.pop(key)

    # For the `diff_b` keys, we treat them as lora_bias.
    # https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraConfig.lora_bias

    for i in range(min_block, max_block + 1):
        # Self-attention
        for o, c in zip(["q", "k", "v", "o"], ["to_q", "to_k", "to_v", "to_out.0"]):
            has_alpha = f"blocks.{i}.self_attn.{o}.alpha" in original_state_dict
            original_key_A = f"blocks.{i}.self_attn.{o}.{lora_down_key}.weight"
            converted_key_A = f"blocks.{i}.attn1.{c}.lora_A.weight"

            original_key_B = f"blocks.{i}.self_attn.{o}.{lora_up_key}.weight"
            converted_key_B = f"blocks.{i}.attn1.{c}.lora_B.weight"

            if has_alpha:
                down_weight = original_state_dict.pop(original_key_A)
                up_weight = original_state_dict.pop(original_key_B)
                scale_down, scale_up = get_alpha_scales(down_weight, f"blocks.{i}.self_attn.{o}.alpha")
                converted_state_dict[converted_key_A] = down_weight * scale_down
                converted_state_dict[converted_key_B] = up_weight * scale_up

            else:
                if original_key_A in original_state_dict:
                    converted_state_dict[converted_key_A] = original_state_dict.pop(original_key_A)
                if original_key_B in original_state_dict:
                    converted_state_dict[converted_key_B] = original_state_dict.pop(original_key_B)

            original_key = f"blocks.{i}.self_attn.{o}.diff_b"
            converted_key = f"blocks.{i}.attn1.{c}.lora_B.bias"
            if original_key in original_state_dict:
                converted_state_dict[converted_key] = original_state_dict.pop(original_key)

        # Cross-attention
        for o, c in zip(["q", "k", "v", "o"], ["to_q", "to_k", "to_v", "to_out.0"]):
            has_alpha = f"blocks.{i}.cross_attn.{o}.alpha" in original_state_dict
            original_key_A = f"blocks.{i}.cross_attn.{o}.{lora_down_key}.weight"
            converted_key_A = f"blocks.{i}.attn2.{c}.lora_A.weight"

            original_key_B = f"blocks.{i}.cross_attn.{o}.{lora_up_key}.weight"
            converted_key_B = f"blocks.{i}.attn2.{c}.lora_B.weight"

            if has_alpha:
                down_weight = original_state_dict.pop(original_key_A)

Review comment (suggestion):
```suggestion
                scale_down, scale_up = get_alpha_scales(down_weight, alpha_key)
```

Code after:
        if key.endswith((".diff", ".diff_b")) and "norm" in key:
            # NOTE: we don't support this because norm layer diff keys are just zeroed values. We can support it
            # in future if needed and they are not zeroed.
            original_state_dict.pop(key)
            logger.debug(f"Removing {key} key from the state dict as it is a norm diff key. This is unsupported.")

        if "time_projection" in key and not has_time_projection_weight:
            # AccVideo lora has diff bias keys but not the weight keys. This causes a weird problem where
            # our lora config adds the time proj lora layers, but we don't have the weights for them.
            # CausVid lora has the weight keys and the bias keys.
            original_state_dict.pop(key)

    # For the `diff_b` keys, we treat them as lora_bias.
    # https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraConfig.lora_bias

    for i in range(min_block, max_block + 1):
        # Self-attention
        for o, c in zip(["q", "k", "v", "o"], ["to_q", "to_k", "to_v", "to_out.0"]):
            alpha_key = f"blocks.{i}.self_attn.{o}.alpha"
            has_alpha = alpha_key in original_state_dict
            original_key_A = f"blocks.{i}.self_attn.{o}.{lora_down_key}.weight"
            converted_key_A = f"blocks.{i}.attn1.{c}.lora_A.weight"

            original_key_B = f"blocks.{i}.self_attn.{o}.{lora_up_key}.weight"
            converted_key_B = f"blocks.{i}.attn1.{c}.lora_B.weight"

            if has_alpha:
                down_weight = original_state_dict.pop(original_key_A)
                up_weight = original_state_dict.pop(original_key_B)
                scale_down, scale_up = get_alpha_scales(down_weight, alpha_key)
                converted_state_dict[converted_key_A] = down_weight * scale_down
                converted_state_dict[converted_key_B] = up_weight * scale_up

            else:
                if original_key_A in original_state_dict:
                    converted_state_dict[converted_key_A] = original_state_dict.pop(original_key_A)
                if original_key_B in original_state_dict:
                    converted_state_dict[converted_key_B] = original_state_dict.pop(original_key_B)

            original_key = f"blocks.{i}.self_attn.{o}.diff_b"
            converted_key = f"blocks.{i}.attn1.{c}.lora_B.bias"
            if original_key in original_state_dict:
                converted_state_dict[converted_key] = original_state_dict.pop(original_key)

        # Cross-attention
        for o, c in zip(["q", "k", "v", "o"], ["to_q", "to_k", "to_v", "to_out.0"]):
            alpha_key = f"blocks.{i}.cross_attn.{o}.alpha"
            has_alpha = alpha_key in original_state_dict
            original_key_A = f"blocks.{i}.cross_attn.{o}.{lora_down_key}.weight"
            converted_key_A = f"blocks.{i}.attn2.{c}.lora_A.weight"