Transformers v5 breaks the `_is_hf_initialized` flag
Author: ZhiyuanChen
Pull request: no
Labels: name: bug

### System Info

- `transformers` version: 5.0.0
- Platform: macOS-26.3-arm64-arm-64bit
- Python version: 3.12.9
- Huggingface_hub version: 1.3.4
- Safetensors version: 0.7.0
- Accelerate version: 1.12.0
- Accelerate config:    not found
- DeepSpeed version: not installed
- PyTorch version (accelerator?): 2.9.1 (NA)

### Who can help?

Some module, like Sinusoidal Position Embedding is stateless. 

Previously we can add a class attributes to avoid transformers auto-init. 

With the v5 release, the class level `_is_hf_initialized` is no longer working. 

I also tried to add attributes to weight tensor per the [Migration Guide](https://github.com/huggingface/transformers/blob/main/MIGRATION_GUIDE_V5.md), but they do not work either.

### Information

- [ ] The official example scripts
- [x] My own modified scripts

### Tasks

- [ ] An officially supported task in the `examples` folder (such as GLUE/SQuAD, ...)
- [ ] My own task or dataset (give details below)

### Reproduction

```
from __future__ import annotations

import math

import torch
from torch import Tensor, nn

from .registry import POSITION_EMBEDDINGS, POSITION_EMBEDDINGS_HF


@POSITION_EMBEDDINGS.register("sinusoidal")
@POSITION_EMBEDDINGS_HF.register("sinusoidal")
class SinusoidalEmbedding(nn.Embedding):
    r"""
    Sinusoidal positional embeddings for inputs with any length.

    Note: **Freezing**
        The embeddings are frozen and cannot be trained.
        They will not be saved in the model's state_dict.

    Tip: **Padding Idx**
        Padding symbols are ignored if the padding_idx is specified.

    Success: **Sequence Length**
        These embeddings get automatically extended in forward if more positions is needed.

    Args:
        num_embeddings: The number of embeddings to use.
        embedding_dim: The dimension of the embeddings.
        padding_idx: The index of the padding symbol.
        bias: The bias of the embeddings.

    Example:
        >>> embedding = SinusoidalEmbedding(num_embeddings=128, embedding_dim=64)
        >>> input_ids = torch.arange(28).repeat(4).view(4, -1)
        >>> input_embeds = torch.randn(4, 28, 64)
        >>> embeddings = embedding(input_ids)
        >>> embeddings.shape  # no batch dimension if padding_idx is None
        torch.Size([28, 64])
        >>> input_embeds = input_embeds + embeddings
        >>> input_embeds.shape
        torch.Size([4, 28, 64])
        >>> embedding = SinusoidalEmbedding(num_embeddings=128, embedding_dim=64, padding_idx=0)
        >>> embeddings = embedding(input_ids)
        >>> embeddings.shape  # batch dimension if padding_idx is not None
        torch.Size([4, 28, 64])
        >>> embedding.state_dict()  # no weight in state_dict
        OrderedDict()
    """

+    _is_hf_initialized = True

    def __init__(
        self,
        num_embeddings: int,
        embedding_dim: int,
        padding_idx: int | None = None,
        bias: int = 1,
        device: torch.device | None = None,
        dtype: torch.dtype = torch.float32,
        **kwargs,
    ):
        weight = self.get_embedding(num_embeddings, embedding_dim, padding_idx, device=device, dtype=dtype)
        super().__init__(num_embeddings, embedding_dim, padding_idx, _weight=weight.detach(), _freeze=True, **kwargs)
        del self.weight
        self.register_buffer("weight", weight, persistent=False)
+        self.weight._is_hf_initialized = True
        self.bias = bias

    def update_weight(self, num_embeddings: int):
        weight = self.get_embedding(
            num_embeddings, self.embedding_dim, self.padding_idx, dtype=self.weight.dtype, device=self.weight.device
        )
        self.register_buffer("weight", weight, persistent=False)
+        self.weight._is_hf_initialized = True

    @staticmethod
    def get_embedding(
        num_embeddings: int,
        embedding_dim: int,
        padding_idx: int | None = None,
        device: torch.device | None = None,
        dtype: torch.dtype = torch.float32,
    ) -> Tensor:
        """
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        """
        if device is None:
            device = get_default_device()
        half_dim = embedding_dim // 2
        emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -(math.log(10000) / (half_dim - 1)))
        emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0)
        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1)
        emb = emb.to(device=device, dtype=dtype)
        if embedding_dim % 2 == 1:
            emb = torch.cat([emb, torch.zeros(num_embeddings, 1, dtype=dtype, device=device)], dim=1)
        if padding_idx is not None:
            emb[padding_idx, :] = 0
        return emb.detach()

    @staticmethod
    def get_position_ids(tensor: Tensor, padding_idx: int | None = None):
        """
        Replace non-padding symbols with their position numbers.

        Position numbers begin at padding_idx+1. Padding symbols are ignored.
        """
        # The series of casts and type-conversions here are carefully
        # balanced to both work with ONNX export and XLA. In particular XLA
        # prefers ints, cumsum defaults to output longs, and ONNX doesn't know
        # how to handle the dtype kwarg in cumsum.
        if padding_idx is None:
            return torch.cumsum(tensor.new_ones(tensor.size(1), dtype=torch.long), dim=0) - 1
        mask = tensor.ne(padding_idx).long()
        return torch.cumsum(mask, dim=1, dtype=torch.long) * mask + padding_idx

    def forward(self, input_ids: Tensor) -> Tensor:
        _, seq_length = input_ids.shape[:2]
        # expand embeddings if needed
        max_position = seq_length + self.bias + 1
        if self.padding_idx is not None:
            max_position += self.padding_idx
        if max_position > self.weight.size(0):
            self.update_weight(max_position)
        # Need to shift the position ids by the padding index
        position_ids = self.get_position_ids(input_ids, self.padding_idx) + self.bias
        return super().forward(position_ids)


def get_default_device() -> torch.device:
    try:
        return torch.get_default_device()
    except ArithmeticError:
        return torch.device("cpu")
```

### Expected behavior

Transformers auto-init should not init stateless weights. 

Comments:
This is very interesting, I double checked the modelling_utils.py and find out the following function works as expected: 

https://github.com/huggingface/transformers/blob/de4958ebaba3673d98e931fd9471643ece2fb0ff/src/transformers/modeling_utils.py#L2337-L2338

Daniel, [30.01.2026 20:13]
Hi @ZhiyuanChen

I've checked _initialize_weights() doesn't check the tensor-level flag, only the module-level one.
How about adding a check for tensor-level?

```python
def _initialize_weights(self, module):
        """
        Initialize the weights if they are not already initialized.
        """
        if getattr(module, "_is_hf_initialized", False):
            return
        
+        if hasattr(module, "weight") and getattr(module.weight, "_is_hf_initialized", False):
+           return

        self._init_weights(module)
        module._is_hf_initialized = True
```

So I've verified it works with your SinusoidalEmbedding.
Just remove `_is_hf_initialized = True` from class level. flag will work now.

```
from __future__ import annotations

import math

import torch
from torch import Tensor, nn

# from .registry import POSITION_EMBEDDINGS, POSITION_EMBEDDINGS_HF
# @POSITION_EMBEDDINGS.register("sinusoidal")
# @POSITION_EMBEDDINGS_HF.register("sinusoidal")

class SinusoidalEmbedding(nn.Embedding):
    r"""
    Sinusoidal positional embeddings for inputs with any length.

    Note: **Freezing**
        The embeddings are frozen and cannot be trained.
        They will not be saved in the model's state_dict.

    Tip: **Padding Idx**
        Padding symbols are ignored if the padding_idx is specified.

    Success: **Sequence Length**
        These embeddings get automatically extended in forward if more positions is needed.

    Args:
        num_embeddings: The number of embeddings to use.
        embedding_dim: The dimension of the embeddings.
        padding_idx: The index of the padding symbol.
        bias: The bias of the embeddings.

    Example:
        >>> embedding = SinusoidalEmbedding(num_embeddings=128, embedding_dim=64)
        >>> input_ids = torch.arange(28).repeat(4).view(4, -1)
        >>> input_embeds = torch.randn(4, 28, 64)
        >>> embeddings = embedding(input_ids)
        >>> embeddings.shape  # no batch dimension if padding_idx is None
        torch.Size([28, 64])
        >>> input_embeds = input_embeds + embeddings
        >>> input_embeds.shape
        torch.Size([4, 28, 64])
        >>> embedding = SinusoidalEmbedding(num_embeddings=128, embedding_dim=64, padding_idx=0)
        >>> embeddings = embedding(input_ids)
        >>> embeddings.shape  # batch dimension if padding_idx is not None
        torch.Size([4, 28, 64])
        >>> embedding.state_dict()  # no weight in state_dict
        OrderedDict()
    """

-    _is_hf_initialized = True

    def __init__(
        self,
        num_embeddings: int,
        embedding_dim: int,
        padding_idx: int | None = None,
        bias: int = 1,
        device: torch.device | None = None,
        dtype: torch.dtype = torch.float32,
        **kwargs,
    ):
        weight = self.get_embedding(num_embeddings, embedding_dim, padding_idx, device=device, dtype=dtype)
        super().__init__(num_embeddings, embedding_dim, padding_idx, _weight=weight.detach(), _freeze=True, **kwargs)
        del self.weight
        self.register_buffer("weight", weight, persistent=False)
        self.weight._is_hf_initialized = True
        self.bias = bias

    def update_weight(self, num_embeddings: int):
        weight = self.get_embedding(
            num_embeddings, self.embedding_dim, self.padding_idx, dtype=self.weight.dtype, device=self.weight.device
        )
        self.register_buffer("weight", weight, persistent=False)
        self.weight._is_hf_initialized = True

    @staticmethod
    def get_embedding(
        num_embeddings: int,
        embedding_dim: int,
        padding_idx: int | None = None,
        device: torch.device | None = None,
        dtype: torch.dtype = torch.float32,
    ) -> Tensor:
        """
        Build sinusoidal embeddings.

Daniel, [30.01.2026 20:13]


        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        """
        if device is None:
            device = get_default_device()
        half_dim = embedding_dim // 2
        emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -(math.log(10000) / (half_dim - 1)))
        emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0)
        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1)
        emb = emb.to(device=device, dtype=dtype)
        if embedding_dim % 2 == 1:
            emb = torch.cat([emb, torch.zeros(num_embeddings, 1, dtype=dtype, device=device)], dim=1)
        if padding_idx is not None:
            emb[padding_idx, :] = 0
        return emb.detach()

    @staticmethod
    def get_position_ids(tensor: Tensor, padding_idx: int | None = None):
        """
        Replace non-padding symbols with their position numbers.

        Position numbers begin at padding_idx+1. Padding symbols are ignored.
        """
        # The series of casts and type-conversions here are carefully
        # balanced to both work with ONNX export and XLA. In particular XLA
        # prefers ints, cumsum defaults to output longs, and ONNX doesn't know
        # how to handle the dtype kwarg in cumsum.
        if padding_idx is None:
            return torch.cumsum(tensor.new_ones(tensor.size(1), dtype=torch.long), dim=0) - 1
        mask = tensor.ne(padding_idx).long()
        return torch.cumsum(mask, dim=1, dtype=torch.long) * mask + padding_idx

    def forward(self, input_ids: Tensor) -> Tensor:
        _, seq_length = input_ids.shape[:2]
        # expand embeddings if needed
        max_position = seq_length + self.bias + 1
        if self.padding_idx is not None:
            max_position += self.padding_idx
        if max_position > self.weight.size(0):
            self.update_weight(max_position)
        # Need to shift the position ids by the padding index
        position_ids = self.get_position_ids(input_ids, self.padding_idx) + self.bias
        return super().forward(position_ids)


def get_default_device() -> torch.device:
    try:
        return torch.get_default_device()
    except ArithmeticError:
        return torch.device("cpu")
    
if __name__ == "__main__":
    from transformers import PretrainedConfig, PreTrainedModel

    class TestConfig(PretrainedConfig):
        model_type = "test"

    class TestModel(PreTrainedModel):
        config_class = TestConfig
        _tied_weights_keys = []

        def __init__(self, config):
            super().__init__(config)
            self.embedding = SinusoidalEmbedding(10, 64)
            self.post_init()

        def _init_weights(self, module):
            if isinstance(module, nn.Embedding):
                module.weight.data.normal_(mean=0.0, std=0.02)

    print("Testing: tensor-level _is_hf_initialized flag")

    model = TestModel(TestConfig())
    original = model.embedding.weight[1, :5].clone()
    print(f"\nBefore init_weights(): \n{original.tolist()}")

    model.init_weights()
    after = model.embedding.weight[1, :5]
    print(f"After init_weights():  \n{after.tolist()}")

    if torch.allclose(original, after):
        print("\nPASSED: Weights preserved! Tensor-level flag works.")
    else:
        print("\nFAILED: Weights corrupted! Tensor-level flag ignored.")    
```

There are logs
```
Testing: tensor-level _is_hf_initialized flag

Before init_weights():
[0.8414709568023682, 0.6764737367630005, 0.5243873596191406, 0.398712694644928, 0.3000060021877289]
After init_weights():
[0.8414709568023682, 0.6764737367630005, 0.5243873596191406, 0.398712694644928, 0.3000060021877289]

PASSED: Weights preserved! Tensor-level flag works.
```

Thank you. I only check the weight tensor because the migration guide: 

https://github.com/huggingface/transformers/blob/78bb85146c59258a0710c8d08311d98d52303c38/MIGRATION_GUIDE_V5.md?plain=1#L369-L374

Your snippet will work only if you are not calling `XxxModel.from_pretrained`. This is likely to be an issue related to meta device handling. 

cc @cyrilvallez, seems related to initialization on the meta device

Hey @ZhiyuanChen! I'm a bit confused here, what is your exact question?
Are you saying that the class attribute 
```
class SinusoidalEmbedding(nn.Embedding):
     _is_hf_initialized = True
```
is not working?

> Hey [@ZhiyuanChen](https://github.com/ZhiyuanChen)! I'm a bit confused here, what is your exact question? Are you saying that the class attribute
> 
> ```
> class SinusoidalEmbedding(nn.Embedding):
>      _is_hf_initialized = True
> ```
> 
> is not working?

Sorry, I should have updated the issues. 

At first I thought this was an issue related to initialisation -- because the wights are not as expected after the `from_pretrained` call. I dug deeper the other day and realised this is more related to the meta device handling in the  `from_pretrained`. 

Unfortunately I was quite busy this week so I haven't have the opportunity to locate the actual issue. 

We use the following code as a workaround: 


```python
class SinusoidalEmbedding(nn.Embedding):
    r"""
    Sinusoidal positional embeddings for inputs with any length.

    Note: **Freezing**
        The embeddings are frozen and cannot be trained.
        They will not be saved in the model's state_dict.

    Tip: **Padding Idx**
        Padding symbols are ignored if the padding_idx is specified.

    Success: **Sequence Length**
        These embeddings get automatically extended in forward if more positions is needed.

    Args:
        num_embeddings: The number of embeddings to use.
        embedding_dim: The dimension of the embeddings.
        padding_idx: The index of the padding symbol.
        bias: The bias of the embeddings.

    Example:
        >>> embedding = SinusoidalEmbedding(num_embeddings=128, embedding_dim=64)
        >>> input_ids = torch.arange(28).repeat(4).view(4, -1)
        >>> input_embeds = torch.randn(4, 28, 64)
        >>> embeddings = embedding(input_ids)
        >>> embeddings.shape  # no batch dimension if padding_idx is None
        torch.Size([28, 64])
        >>> input_embeds = input_embeds + embeddings
        >>> input_embeds.shape
        torch.Size([4, 28, 64])
        >>> embedding = SinusoidalEmbedding(num_embeddings=128, embedding_dim=64, padding_idx=0)
        >>> embeddings = embedding(input_ids)
        >>> embeddings.shape  # batch dimension if padding_idx is not None
        torch.Size([4, 28, 64])
        >>> embedding.state_dict()  # no weight in state_dict
        OrderedDict()
    """

    _is_hf_initialized = True

    def __init__(
        self,
        num_embeddings: int,
        embedding_dim: int,
        padding_idx: int | None = None,
        bias: int = 1,
        device: torch.device | None = None,
        dtype: torch.dtype = torch.float32,
        **kwargs,
    ):
        weight = self.get_embedding(num_embeddings, embedding_dim, padding_idx, device=device, dtype=dtype)
        super().__init__(num_embeddings, embedding_dim, padding_idx, _weight=weight.detach(), _freeze=True, **kwargs)
        del self.weight
        self.register_buffer("weight", weight, persistent=False)
        self.bias = bias
        self._initialized = False

    def update_weight(self, num_embeddings: int):
        weight = self.get_embedding(
            num_embeddings, self.embedding_dim, self.padding_idx, dtype=self.weight.dtype, device=self.weight.device
        )
        self.register_buffer("weight", weight, persistent=False)

    @staticmethod
    def get_embedding(
        num_embeddings: int,
        embedding_dim: int,
        padding_idx: int | None = None,
        device: torch.device | None = None,
        dtype: torch.dtype = torch.float32,
    ) -> Tensor:
        """
        Build sinusoidal embeddings.

        This matches the implementation in tensor2tensor, but differs slightly from the description in Section 3.5 of
        "Attention Is All You Need".
        """
        if device is None:
            device = get_default_device()
        half_dim = embedding_dim // 2
        emb = torch.exp(torch.arange(half_dim, device=device, dtype=dtype) * -(math.log(10000) / (half_dim - 1)))
        emb = torch.arange(num_embeddings, device=device, dtype=dtype).unsqueeze(1) * emb.unsqueeze(0)
        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1)
        if embedding_dim % 2 == 1:
            emb = torch.cat([emb, torch.zeros(num_embeddings, 1, dtype=dtype, device=device)], dim=1)
        if padding_idx is not None:
            emb[padding_idx, :] = 0
        return emb.detach()

    @staticmethod
    def get_position_ids(tensor: Tensor, padding_idx: int | None = None):
        """
        Replace non-padding symbols with their position numbers.

        Position numbers begin at padding_idx+1. Padding symbols are ignored.
        """
        # The series of casts and type-conversions here are carefully
        # balanced to both work with ONNX export and XLA. In particular XLA
        # prefers ints, cumsum defaults to output longs, and ONNX doesn't know
        # how to handle the dtype kwarg in cumsum.
        if padding_idx is None:
            return torch.cumsum(tensor.new_ones(tensor.size(1), dtype=torch.long), dim=0) - 1
        mask = tensor.ne(padding_idx).long()
        return torch.cumsum(mask, dim=1, dtype=torch.long) * mask + padding_idx

    def forward(self, input_ids: Tensor) -> Tensor:
        if not self._initialized:
            self.update_weight(self.num_embeddings)
            self._initialized = True
        _, seq_length = input_ids.shape[:2]
        # expand embeddings if needed
        max_position = seq_length + self.bias + 1
        if self.padding_idx is not None:
            max_position += self.padding_idx
        if max_position > self.weight.size(0):
            self.update_weight(max_position)
        # Need to shift the position ids by the padding index
        position_ids = self.get_position_ids(input_ids, self.padding_idx) + self.bias
        return super().forward(position_ids)
```

Oh ok I see - No need to add it on the weights as well, module is enough.
Otherwise, preferred way is to add the proper initialization to `_init_weights` 🤗