transformer.py

# MIT License
# 
# Copyright (c) 2026 audio-embeddings contributors
# 
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# 
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from __future__ import annotations

from collections.abc import Sequence
from typing import Callable

import torch
import torch.nn as nn
from timm.layers import DropPath
from timm.layers import Mlp

from .normalization import MixedPrecisionRMSNorm
from .rope import RoPEAttention
from .rope import RotaryEmbedding1D
from .rope import RotaryEmbedding2D


class FullAttentionResidual(nn.Module):
    """Softmax attention over all preceding outputs along model depth."""

    def __init__(self, dim: int) -> None:
        super().__init__()
        self.dim = dim
        self.norm = MixedPrecisionRMSNorm(dim)
        self.query = nn.Parameter(torch.zeros(dim))

    def forward(self, values: Sequence[torch.Tensor]) -> torch.Tensor:
        if not values:
            raise ValueError("FullAttentionResidual requires at least one value")

        expected_shape = values[0].shape
        if len(expected_shape) != 3 or expected_shape[-1] != self.dim:
            raise ValueError(
                "FullAttentionResidual values must have shape [B, N, D] with "
                f"D={self.dim}, got {expected_shape}"
            )
        for index, value in enumerate(values[1:], start=1):
            if value.shape != expected_shape:
                raise ValueError(
                    "FullAttentionResidual values must have identical shapes; "
                    f"value 0 has {expected_shape}, value {index} has {value.shape}"
                )

        stacked_values = torch.stack(tuple(values), dim=0)
        keys = self.norm(stacked_values)
        logits = torch.einsum("d,l b n d->l b n", self.query, keys)
        weights = logits.softmax(dim=0)
        return torch.einsum("l b n,l b n d->b n d", weights, stacked_values)


def build_norm_layer(
    *,
    dim: int,
    norm_type: str = "layernorm",
    norm_layer: Callable[[int], nn.Module] | None = None,
    norm_eps: float | None = None,
) -> nn.Module:
    """Build a token-channel normalization layer.

    `norm_layer` is kept for backward compatibility with direct Python
    construction. Configs should prefer `norm_type` so choices are explicit in
    Hydra overrides and experiment files.
    """
    if norm_layer is not None:
        if norm_eps is not None:
            raise ValueError("Set norm_eps or a custom norm_layer, not both")
        return norm_layer(dim)

    if norm_eps is not None and norm_eps <= 0:
        raise ValueError(f"norm_eps must be positive, got {norm_eps}")
    kwargs = {} if norm_eps is None else {"eps": norm_eps}
    normalized = norm_type.strip().lower().replace("_", "")
    if normalized == "layernorm":
        return nn.LayerNorm(dim, **kwargs)
    if normalized == "rmsnorm":
        return MixedPrecisionRMSNorm(dim, **kwargs)

    raise ValueError(
        f"Unknown norm_type={norm_type!r}; expected 'layernorm' or 'rmsnorm'"
    )


def build_gelu_mlp(
    *,
    dim: int,
    mlp_ratio: float,
    act_layer: type[nn.Module],
    drop: float,
    bias: bool = True,
) -> nn.Module:
    """Build the baseline ViT feed-forward layer used before ablations."""
    return Mlp(
        in_features=dim,
        hidden_features=int(dim * mlp_ratio),
        act_layer=act_layer,
        bias=bias,
        drop=drop,
    )


class SwiGLUMlp(nn.Module):
    """SwiGLU feed-forward layer for transformer ablations.

    The hidden dimension is controlled by `mlp_ratio` in the same place as the
    baseline MLP. GLU variants normally use a smaller ratio such as 8/3 because
    they have two input projections before the output projection.
    """

    def __init__(
        self,
        *,
        dim: int,
        hidden_features: int,
        drop: float,
        bias: bool = True,
    ) -> None:
        super().__init__()
        self.gate = nn.Linear(dim, hidden_features, bias=bias)
        self.value = nn.Linear(dim, hidden_features, bias=bias)
        self.proj = nn.Linear(hidden_features, dim, bias=bias)
        self.drop = nn.Dropout(drop)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = nn.functional.silu(self.gate(x)) * self.value(x)
        x = self.drop(x)
        x = self.proj(x)
        return self.drop(x)


def build_mlp(
    *,
    dim: int,
    mlp_ratio: float,
    mlp_type: str = "gelu_mlp",
    act_layer: type[nn.Module],
    drop: float,
    bias: bool = True,
) -> nn.Module:
    """Build the transformer feed-forward layer selected by config."""
    hidden_features = int(dim * mlp_ratio)
    normalized = mlp_type.strip().lower().replace("-", "_")

    if normalized == "gelu_mlp":
        return build_gelu_mlp(
            dim=dim,
            mlp_ratio=mlp_ratio,
            act_layer=act_layer,
            drop=drop,
            bias=bias,
        )
    if normalized == "swiglu":
        return SwiGLUMlp(
            dim=dim,
            hidden_features=hidden_features,
            drop=drop,
            bias=bias,
        )

    raise ValueError(f"Unknown mlp_type={mlp_type!r}; expected 'gelu_mlp' or 'swiglu'")


class RoPEBlock(nn.Module):
    """Pre-norm transformer block with RoPE-aware attention."""

    def __init__(
        self,
        dim: int,
        num_heads: int,
        mlp_ratio: float = 4.0,
        qkv_bias: bool = False,
        proj_bias: bool = True,
        mlp_bias: bool = True,
        qk_norm: bool = False,
        qk_norm_type: str = "layernorm",
        mlp_type: str = "gelu_mlp",
        proj_drop: float = 0.0,
        attn_drop: float = 0.0,
        drop_path: float = 0.0,
        act_layer: type[nn.Module] = nn.GELU,
        norm_type: str = "layernorm",
        norm_layer: Callable[[int], nn.Module] | None = None,
        rope: RotaryEmbedding1D | RotaryEmbedding2D | None = None,
        residual_type: str = "standard",
        norm_eps: float | None = None,
    ) -> None:
        super().__init__()
        self.residual_type = residual_type.strip().lower().replace("-", "_")
        if self.residual_type not in {"standard", "full_attnres"}:
            raise ValueError(
                f"Unknown residual_type={residual_type!r}; expected 'standard' "
                "or 'full_attnres'"
            )
        self.norm1 = build_norm_layer(
            dim=dim,
            norm_type=norm_type,
            norm_layer=norm_layer,
            norm_eps=norm_eps,
        )
        self.attn = RoPEAttention(
            dim,
            num_heads=num_heads,
            qkv_bias=qkv_bias,
            proj_bias=proj_bias,
            attn_drop=attn_drop,
            proj_drop=proj_drop,
            rope=rope,
            qk_norm=qk_norm,
            qk_norm_type=qk_norm_type,
        )
        self.norm2 = build_norm_layer(
            dim=dim,
            norm_type=norm_type,
            norm_layer=norm_layer,
            norm_eps=norm_eps,
        )
        self.mlp = build_mlp(
            dim=dim,
            mlp_ratio=mlp_ratio,
            mlp_type=mlp_type,
            act_layer=act_layer,
            drop=proj_drop,
            bias=mlp_bias,
        )
        self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
        if self.residual_type == "full_attnres":
            self.attention_residual = FullAttentionResidual(dim)
            self.mlp_residual = FullAttentionResidual(dim)
        else:
            self.attention_residual = None
            self.mlp_residual = None

    def attention_output(
        self,
        x: torch.Tensor,
        pos_ids: torch.Tensor | None = None,
        grid_size: tuple[int, int] | None = None,
    ) -> torch.Tensor:
        return self.drop_path(
            self.attn(self.norm1(x), pos_ids=pos_ids, grid_size=grid_size)
        )

    def mlp_output(self, x: torch.Tensor) -> torch.Tensor:
        return self.drop_path(self.mlp(self.norm2(x)))

    def forward(
        self,
        x: torch.Tensor,
        pos_ids: torch.Tensor | None = None,
        grid_size: tuple[int, int] | None = None,
    ) -> torch.Tensor:
        x = x + self.attention_output(x, pos_ids=pos_ids, grid_size=grid_size)
        x = x + self.mlp_output(x)
        return x