update model files

README.md

@@ -1,3 +1,70 @@
 ---
 license: bsd-3-clause
 ---
+
+# CodeT5+ 220M Bimodal Models
+
+## Model description
+
+[CodeT5+](https://github.com/salesforce/CodeT5/tree/main/CodeT5+) is a new family of open code large language models
+with an encoder-decoder architecture that can flexibly operate in different modes (i.e. _encoder-only_, _decoder-only_,
+and _encoder-decoder_) to support a wide range of code understanding and generation tasks.
+It is introduced in the paper:
+
+[CodeT5+: Open Code Large Language Models for Code Understanding and Generation](https://arxiv.org/pdf/2305.07922.pdf)
+by [Yue Wang](https://yuewang-cuhk.github.io/)\*, [Hung Le](https://sites.google.com/view/henryle2018/home?pli=1)\*, [Akhilesh Deepak Gotmare](https://akhileshgotmare.github.io/), [Nghi D.Q. Bui](https://bdqnghi.github.io/), [Junnan Li](https://sites.google.com/site/junnanlics), [Steven C.H. Hoi](https://sites.google.com/view/stevenhoi/home) (*
+indicates equal contribution).
+
+Compared to the original CodeT5 family (base: `220M`, large: `770M`), CodeT5+ is pretrained with a diverse set of
+pretraining tasks including _span denoising_, _causal language modeling_, _contrastive learning_, and _text-code
+matching_ to learn rich representations from both unimodal code data and bimodal code-text data.
+Additionally, it employs a simple yet effective _compute-efficient pretraining_ method to initialize the model
+components with frozen off-the-shelf LLMs such as [CodeGen](https://github.com/salesforce/CodeGen) to efficiently scale
+up the model (i.e. `2B`, `6B`, `16B`), and adopts a "shallow encoder and deep decoder" architecture.
+Furthermore, it is instruction-tuned to align with natural language instructions (see our InstructCodeT5+ 16B)
+following [Code Alpaca](https://github.com/sahil280114/codealpaca).
+
+## How to use
+
+This model can be easily loaded using the `AutoModel` functionality and employs the [CodeT5](https://github.com/salesforce/CodeT5) tokenizer with three special tokens added (`[ENC]`, `[TDEC]`, `[CDEC]`).
+This checkpoint consists of a CodeT5+ 220M model and a projection layer and an itm_head layer for text-code matching.
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+checkpoint = "Salesforce/codet5p-220m-bimodal"
+device = "cuda"  # for GPU usage or "cpu" for CPU usage
+
+tokenizer = AutoTokenizer.from_pretrained(checkpoint, trust_remote_code=True)
+model = AutoModel.from_pretrained(checkpoint, trust_remote_code=True).to(device)
+```
+
+## Pretraining data
+
+This checkpoint is trained on the stricter permissive subset of the deduplicated version of
+the [github-code dataset](https://huggingface.co/datasets/codeparrot/github-code).
+The data is preprocessed by reserving only permissively licensed code ("mit" “apache-2”, “bsd-3-clause”, “bsd-2-clause”,
+“cc0-1.0”, “unlicense”, “isc”).
+Supported languages (9 in total) are as follows:
+`c`, `c++`, `c-sharp`,  `go`, `java`, `javascript`,  `php`, `python`, `ruby.`
+
+## Training procedure
+
+This checkpoint is first trained on the unimodal code data at the first-stage pretraining and then on bimodal text-code
+pair data using the proposed mixture of pretraining tasks.
+Please refer to the paper for more details.
+
+## Evaluation results
+
+Please refer to the paper and the official GitHub repo for more details.
+
+## BibTeX entry and citation info
+
+```bibtex
+@article{wang2023codet5plus,
+  title={CodeT5+: Open Code Large Language Models for Code Understanding and Generation},
+  author={Wang, Yue and Le, Hung and Gotmare, Akhilesh Deepak and Bui, Nghi D.Q. and Li, Junnan and Hoi, Steven C. H.},
+  journal={arXiv preprint},
+  year={2023}
+}
+```

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "[CDEC]": 32102,
+  "[ENC]": 32100,
+  "[TDEC]": 32101
+}

config.json

@@ -0,0 +1,45 @@
+{
+  "_name_or_path": "Salesforce/codet5p-220m-bimodal",
+  "architectures": [
+    "CodeT5pBimodalModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_codet5p_bimodal.CodeT5pBimodalConfig",
+    "AutoModel": "modeling_codet5p_bimodal.CodeT5pBimodalModel"
+  },
+  "bos_token_id": 1,
+  "d_ff": 3072,
+  "d_kv": 64,
+  "d_model": 768,
+  "embed_dim": 256,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "relu",
+  "dropout_rate": 0.1,
+  "eos_token_id": 2,
+  "feed_forward_proj": "relu",
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": false,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "codet5p_bimodal",
+  "n_positions": 512,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+
+  "torch_dtype": "float32",
+  "transformers_version": "4.30.2",
+  "use_cache": true,
+  "vocab_size": 32103
+}

configuration_codet5p_matching.py

@@ -0,0 +1,76 @@
+# coding=utf-8
+# Copyright 2023 Salesforce authors, The EleutherAI, and HuggingFace Teams. All rights reserved.
+
+""" CodeT5+ embedding model configuration"""
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+class CodeT5pMatchingConfig(PretrainedConfig):
+    model_type = "codet5p_matching"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    attribute_map = {"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}
+
+    def __init__(
+            self,
+            vocab_size=32103,
+            d_model=768,
+            embed_dim=256,
+            d_kv=64,
+            d_ff=3072,
+            num_layers=12,
+            num_decoder_layers=None,
+            num_heads=12,
+            relative_attention_num_buckets=32,
+            relative_attention_max_distance=128,
+            dropout_rate=0.1,
+            layer_norm_epsilon=1e-6,
+            initializer_factor=1.0,
+            feed_forward_proj="relu",
+            is_encoder_decoder=False,
+            use_cache=True,
+            pad_token_id=0,
+            eos_token_id=2,
+            **kwargs
+    ):
+        self.vocab_size = vocab_size
+        self.d_model = d_model
+        self.embed_dim = embed_dim
+        self.d_kv = d_kv
+        self.d_ff = d_ff
+        self.num_layers = num_layers
+        self.num_decoder_layers = (
+            num_decoder_layers if num_decoder_layers is not None else self.num_layers
+        )  # default = symmetry
+        self.num_heads = num_heads
+        self.relative_attention_num_buckets = relative_attention_num_buckets
+        self.relative_attention_max_distance = relative_attention_max_distance
+        self.dropout_rate = dropout_rate
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.initializer_factor = initializer_factor
+        self.feed_forward_proj = feed_forward_proj
+        self.use_cache = use_cache
+
+        act_info = self.feed_forward_proj.split("-")
+        self.dense_act_fn = act_info[-1]
+        self.is_gated_act = act_info[0] == "gated"
+
+        if len(act_info) > 1 and act_info[0] != "gated" or len(act_info) > 2:
+            raise ValueError(
+                f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."
+                "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
+                "'gated-gelu' or 'relu'"
+            )
+
+        # for backwards compatibility
+        if feed_forward_proj == "gated-gelu":
+            self.dense_act_fn = "gelu_new"
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            eos_token_id=eos_token_id,
+            is_encoder_decoder=is_encoder_decoder,
+            **kwargs,
+        )

modeling_codet5p_matching.py

@@ -0,0 +1,28 @@
+# coding=utf-8
+# Copyright 2023 Salesforce authors, The EleutherAI, and HuggingFace Teams. All rights reserved.
+""" PyTorch CodeT5+ matching models.
+The implementation is based on transformers.models.t5.modeling_t5 by adding a projection layer on T5EncoderModel
+"""
+
+from typing import Optional, Tuple, Union
+import torch
+from torch import nn
+import torch.nn.functional as F
+from transformers import T5ForConditionalGeneration
+from transformers.modeling_outputs import (
+    BaseModelOutput,
+)
+from configuration_codet5p_matching import CodeT5pMatchingConfig
+
+
+class CodeT5pMatchingModel(T5ForConditionalGeneration):
+    config_class = CodeT5pMatchingConfig
+
+    authorized_missing_keys = [
+        r"encoder.embed_tokens.weight",
+    ]
+
+    def __init__(self, config: CodeT5pMatchingConfig):
+        super().__init__(config)
+        self.proj = nn.Linear(config.d_model, config.embed_dim)
+        self.itm_head = nn.Linear(config.d_model, 2)

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42fb839e42789ccaa6e7ed10b6dd8b6906c09bf1e4281e20e5c9eedbea60de6c
+size 892417313

special_tokens_map.json

@@ -0,0 +1,56 @@
+{
+  "additional_special_tokens": [
+    "[ENC]",
+    "[TDEC]",
+    "[CDEC]"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,64 @@
+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}