Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -0,0 +1,9 @@
+{
+  "word_embedding_dimension": 1024,
+  "pooling_mode_cls_token": true,
+  "pooling_mode_mean_tokens": false,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false
+}

config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "./export_bge_m3",
+  "architectures": [
+    "BgeM3Model"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoConfig": "configuration_bge_m3.BgeM3Config",
+    "AutoModel": "modeling_bge_m3.BgeM3Model"
+  },
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "colbert_dim": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 8194,
+  "model_type": "bge-m3",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "sentence_pooling_method": "cls",
+  "torch_dtype": "float32",
+  "transformers_version": "4.37.2",
+  "type_vocab_size": 1,
+  "unused_tokens": [
+    0,
+    2,
+    1,
+    3
+  ],
+  "use_cache": true,
+  "vocab_size": 250002
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.3.1",
+    "transformers": "4.37.2",
+    "pytorch": "2.2.0a0+81ea7a4"
+  }
+}

configuration_bge_m3.py

@@ -0,0 +1,52 @@
+from transformers.configuration_utils import PretrainedConfig
+
+
+# Copied from transformers.models.xlm_roberta.configuration_xlm_roberta.XLMRobertaConfig with XLMRoberta->BgeM3
+class BgeM3Config(PretrainedConfig):
+    model_type = "bge-m3"
+
+    def __init__(
+        self,
+        vocab_size=30522,
+        hidden_size=768,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        intermediate_size=3072,
+        hidden_act="gelu",
+        hidden_dropout_prob=0.1,
+        attention_probs_dropout_prob=0.1,
+        max_position_embeddings=512,
+        type_vocab_size=2,
+        initializer_range=0.02,
+        layer_norm_eps=1e-12,
+        pad_token_id=1,
+        bos_token_id=0,
+        eos_token_id=2,
+        position_embedding_type="absolute",
+        use_cache=True,
+        classifier_dropout=None,
+        colbert_dim=None,
+        sentence_pooling_method='cls',
+        unused_tokens=None,
+        **kwargs,
+    ):
+        super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.hidden_act = hidden_act
+        self.intermediate_size = intermediate_size
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_probs_dropout_prob = attention_probs_dropout_prob
+        self.max_position_embeddings = max_position_embeddings
+        self.type_vocab_size = type_vocab_size
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.position_embedding_type = position_embedding_type
+        self.use_cache = use_cache
+        self.classifier_dropout = classifier_dropout
+        self.colbert_dim = colbert_dim
+        self.sentence_pooling_method = sentence_pooling_method
+        self.unused_tokens = unused_tokens

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cfba41811196728d7604ac67da00fd749beeba0e6557eb685620898a032f1eb
+size 2271071852

modeling_bge_m3.py

@@ -0,0 +1,201 @@
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple, Union
+
+import torch
+from torch import nn
+from transformers.modeling_outputs import BaseModelOutputWithPoolingAndCrossAttentions, ModelOutput
+from transformers.models.xlm_roberta import (
+    XLMRobertaModel,
+    XLMRobertaPreTrainedModel,
+)
+
+from .configuration_bge_m3 import BgeM3Config
+
+
+@dataclass
+class BgeM3ModelOutput(ModelOutput):
+    last_hidden_state: torch.FloatTensor = None
+    pooler_output: torch.FloatTensor = None
+    dense_output: torch.FloatTensor = None
+    colbert_output: Optional[List[torch.FloatTensor]] = None
+    sparse_output: Optional[Dict[int, float]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+    cross_attentions: Optional[Tuple[torch.FloatTensor]] = None
+
+
+class BgeM3Model(XLMRobertaPreTrainedModel):
+    config_class = BgeM3Config
+
+    def __init__(self, config: BgeM3Config):
+        super().__init__(config)
+        self.roberta = XLMRobertaModel(config, add_pooling_layer=False)
+        # TODO: Check the dtype of these linear layers
+        self.colbert_linear = nn.Linear(
+            in_features=config.hidden_size,
+            out_features=config.hidden_size if config.colbert_dim is None else config.colbert_dim,
+        )
+        self.sparse_linear = nn.Linear(in_features=config.hidden_size, out_features=1)
+        self.sentence_pooling_method = config.sentence_pooling_method
+
+        self.init_weights()
+
+    def dense_embedding(self, hidden_state, mask):
+        if self.sentence_pooling_method == "cls":
+            return hidden_state[:, 0]
+        elif self.sentence_pooling_method == "mean":
+            s = torch.sum(hidden_state * mask.unsqueeze(-1).float(), dim=1)
+            d = mask.sum(axis=1, keepdim=True).float()
+            return s / d
+
+    def sparse_embedding(self, hidden_state, input_ids, return_embedding: bool = False):
+        token_weights = torch.relu(self.sparse_linear(hidden_state))
+        if not return_embedding:
+            return token_weights
+
+        sparse_embedding = torch.zeros(
+            input_ids.size(0),
+            input_ids.size(1),
+            self.config.vocab_size,
+            dtype=token_weights.dtype,
+            device=token_weights.device,
+        )
+        sparse_embedding = torch.scatter(sparse_embedding, dim=-1, index=input_ids.unsqueeze(-1), src=token_weights)
+
+        unused_tokens = self.config.unused_tokens
+        sparse_embedding = torch.max(sparse_embedding, dim=1).values
+        sparse_embedding[:, unused_tokens] *= 0.0
+        return sparse_embedding
+
+    def colbert_embedding(self, last_hidden_state, mask):
+        colbert_vecs = self.colbert_linear(last_hidden_state[:, 1:])
+        colbert_vecs = colbert_vecs * mask[:, 1:][:, :, None].float()
+        return colbert_vecs
+
+    def _process_token_weights(self, token_weights, input_ids, mask):
+        token_weights = token_weights.squeeze(-1)
+        # conver to dict
+        all_result = []
+        unused_tokens = self.config.unused_tokens
+        unused_tokens = torch.tensor(unused_tokens, device=input_ids.device)
+
+        # 获取有效的 token 的索引
+        valid_indices = ~torch.isin(input_ids, unused_tokens)
+        # weight必须大于0
+        valid_indices = (valid_indices & (token_weights > 0)).bool()
+        # 结合 attention mask，获取有效的 token 的索引
+        valid_indices = (valid_indices & mask).bool()
+
+        for i, valid in enumerate(valid_indices):
+            result = defaultdict(int)
+
+            # 获取有效的 weights 和 ids
+            valid_weights = token_weights[i][valid]
+            valid_ids = input_ids[i][valid]
+
+            # 获取每个 id 的最大权重
+            unique_ids, inverse_indices = torch.unique(valid_ids, return_inverse=True)
+
+            # 使用一个循环来找到每个 unique id 的最大权重
+            for i in range(unique_ids.shape[0]):
+                id_mask = inverse_indices == i
+                result[str(unique_ids[i].item())] = valid_weights[id_mask].max().item()
+
+            all_result.append(result)
+        # token_weights = np.ceil(token_weights * 100)
+        # for w, idx, num in zip(token_weights, input_ids, tokens_num):
+        #     r = defaultdict(int)
+        #     token_weight = w[:num]
+        #     idx = idx[:num]
+
+        #     for t_w, t_idx in zip(token_weight, idx):
+        #         if t_idx.item() not in unused_tokens:
+        #             t_idx = str(t_idx.item())
+        #             if t_w > r[t_idx]:
+        #                 r[t_idx] = t_w.item()
+
+        #     result.append(r)
+
+        # if idx not in unused_tokens and w > 0:
+        #     idx = str(idx)
+        #     # w = int(w)
+        #     if w > result[idx]:
+        #         result[idx] = w
+        return all_result
+
+    def _process_colbert_vecs(self, colbert_vecs, tokens_num) -> List[torch.Tensor]:
+        # delte the vectors of padding tokens
+        vecs = []
+        for i in range(len(tokens_num)):
+            vecs.append(colbert_vecs[i, : tokens_num[i] - 1])
+        return vecs
+
+    # Copied from transformers.models.bert.modeling_bert.BertModel.forward
+    def forward(
+        self,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        token_type_ids: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+        head_mask: Optional[torch.Tensor] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.Tensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple[torch.Tensor], BgeM3ModelOutput]:
+        roberta_output: BaseModelOutputWithPoolingAndCrossAttentions = self.roberta(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+            head_mask=head_mask,
+            inputs_embeds=inputs_embeds,
+            encoder_hidden_states=encoder_hidden_states,
+            encoder_attention_mask=encoder_attention_mask,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=True,
+        )
+
+        last_hidden_state = roberta_output.last_hidden_state
+        dense_output = self.dense_embedding(last_hidden_state, attention_mask)
+
+        tokens_num = attention_mask.sum(dim=1)
+        colbert_output = self.colbert_embedding(last_hidden_state, attention_mask)
+        colbert_output = self._process_colbert_vecs(colbert_output, tokens_num)
+
+        sparse_output = self.sparse_embedding(last_hidden_state, input_ids)
+        sparse_output = self._process_token_weights(sparse_output, input_ids, attention_mask)
+
+        if not return_dict:
+            return (
+                last_hidden_state,
+                roberta_output.pooler_output,
+                dense_output,
+                colbert_output,
+                sparse_output,
+                roberta_output.hidden_states,
+                roberta_output.past_key_values,
+                roberta_output.attentions,
+                roberta_output.cross_attentions,
+            )
+
+        return BgeM3ModelOutput(
+            last_hidden_state=last_hidden_state,
+            dense_output=dense_output,
+            pooler_output=roberta_output.pooler_output,
+            colbert_output=colbert_output,
+            sparse_output=sparse_output,
+            hidden_states=roberta_output.hidden_states,
+            past_key_values=roberta_output.past_key_values,
+            attentions=roberta_output.attentions,
+            cross_attentions=roberta_output.cross_attentions,
+        )

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 8192,
+  "do_lower_case": false
+}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6710678b12670bc442b99edc952c4d996ae309a7020c1fa0096dd245c2faf790
+size 17082821

tokenizer_config.json

@@ -0,0 +1,55 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "250001": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 8192,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}