update model files

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "[CDEC]": 32102,
+  "[ENC]": 32100,
+  "[TDEC]": 32101
+}

config.json

@@ -0,0 +1,44 @@
+{
+  "_name_or_path": "Salesforce/codet5p-110m-embedding",
+  "architectures": [
+    "CodeT5p_Embedding"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_codet5p_embedding.CodeT5pEmbeddingConfig",
+    "AutoModel": "modeling_codet5p_embedding.CodeT5pEmbeddingModel"
+  },
+  "bos_token_id": 1,
+  "d_ff": 3072,
+  "d_kv": 64,
+  "d_model": 768,
+  "embed_dim": 256,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "relu",
+  "dropout_rate": 0.1,
+  "eos_token_id": 2,
+  "feed_forward_proj": "relu",
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": false,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "codet5p_embedding",
+  "n_positions": 512,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+
+  "torch_dtype": "float32",
+  "transformers_version": "4.21.3",
+  "use_cache": true,
+  "vocab_size": 32103
+}

configuration_codet5p_embedding.py

@@ -0,0 +1,72 @@
+# coding=utf-8
+# Copyright 2023 Salesforce authors, The EleutherAI, and HuggingFace Teams. All rights reserved.
+
+""" CodeT5+ embedding model configuration"""
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class CodeT5pEmbeddingConfig(PretrainedConfig):
+    model_type = "codet5p_embedding"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}
+
+    def __init__(
+            self,
+            vocab_size=32103,
+            d_model=768,
+            embed_dim=256,
+            d_kv=64,
+            d_ff=3072,
+            num_layers=12,
+            num_heads=12,
+            relative_attention_num_buckets=32,
+            relative_attention_max_distance=128,
+            dropout_rate=0.1,
+            layer_norm_epsilon=1e-6,
+            initializer_factor=1.0,
+            feed_forward_proj="relu",
+            is_encoder_decoder=False,
+            use_cache=True,
+            pad_token_id=0,
+            eos_token_id=2,
+            **kwargs
+    ):
+        self.vocab_size = vocab_size
+        self.d_model = d_model
+        self.embed_dim = embed_dim
+        self.d_kv = d_kv
+        self.d_ff = d_ff
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+        self.relative_attention_num_buckets = relative_attention_num_buckets
+        self.relative_attention_max_distance = relative_attention_max_distance
+        self.dropout_rate = dropout_rate
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_factor = initializer_factor
+        self.feed_forward_proj = feed_forward_proj
+        self.use_cache = use_cache
+
+        act_info = self.feed_forward_proj.split("-")
+        self.dense_act_fn = act_info[-1]
+        self.is_gated_act = act_info[0] == "gated"
+
+        if len(act_info) > 1 and act_info[0] != "gated" or len(act_info) > 2:
+            raise ValueError(
+                f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."
+                "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
+                "'gated-gelu' or 'relu'"
+            )
+
+        # for backwards compatibility
+        if feed_forward_proj == "gated-gelu":
+            self.dense_act_fn = "gelu_new"
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            eos_token_id=eos_token_id,
+            is_encoder_decoder=is_encoder_decoder,
+            **kwargs,
+        )

modeling_codet5p_embedding.py

@@ -0,0 +1,53 @@
+# coding=utf-8
+# Copyright 2023 Salesforce authors, The EleutherAI, and HuggingFace Teams. All rights reserved.
+""" PyTorch CodeT5+ mbedding models.
+The implementation is based on transformers.models.t5.modeling_t5 by adding a projection layer on T5EncoderModel
+"""
+
+from typing import Optional, Tuple, Union
+import torch
+from torch import nn
+import torch.nn.functional as F
+from transformers import T5EncoderModel
+from transformers.modeling_outputs import (
+    BaseModelOutput,
+)
+from .configuration_codet5p_embedding import CodeT5pEmbeddingConfig
+
+
+class CodeT5pEmbeddingModel(T5EncoderModel):
+    config_class = CodeT5pEmbeddingConfig
+
+    authorized_missing_keys = [
+        r"encoder.embed_tokens.weight",
+    ]
+
+    def __init__(self, config: CodeT5pEmbeddingConfig):
+        super().__init__(config)
+        self.proj = nn.Linear(config.d_model, config.embed_dim)
+
+    def forward(
+            self,
+            input_ids: Optional[torch.LongTensor] = None,
+            attention_mask: Optional[torch.FloatTensor] = None,
+            head_mask: Optional[torch.FloatTensor] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple[torch.FloatTensor], BaseModelOutput]:
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        encoder_outputs = self.encoder(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            head_mask=head_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        embedding = F.normalize(self.proj(encoder_outputs.last_hidden_state[:, 0, :]), dim=-1)
+
+        return embedding

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:097d1bd8c5df11eb82aa5b750d208eee17d570babf94c77158279dc992c6829b
+size 439257889

special_tokens_map.json

@@ -0,0 +1,56 @@
+{
+  "additional_special_tokens": [
+    "[ENC]",
+    "[TDEC]",
+    "[CDEC]"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,64 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}