Upload 7 files

config.json

@@ -0,0 +1,38 @@
+{
+  "_name_or_path": "fairseq-roberta-all-model/checkpoint_last.pt",
+  "architectures": [
+    "XLMRobertaForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.19.2",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 52001
+}

gitattributes.txt

@@ -0,0 +1,27 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e837cd62414db71e958b767cf9ef0f1ff98c48afc40300b5826ebb9a6b589934
+size 503996397

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6628eae3bc0eb53ddfdd1d988cfd6527738fde88e873193055f2fdce468edcc0
+size 1228505

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}}

tokenization_roberta_spm.py

@@ -0,0 +1,200 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors, The HuggingFace Inc. team and Gyeongmin Kim
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from transformers.models.xlm_roberta.tokenization_xlm_roberta import XLMRobertaTokenizer
+
+SPIECE_UNDERLINE = "▁"
+
+VOCAB_FILES_NAMES = {"spm_model": "spm.model", "custom_vocab_file": "dict.txt"}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "spm_model": {
+        "fairseq-roberta-spm-normal": "fairseq-roberta-all-model/spm.model",
+    },
+    "custom_vocab_file": {
+        "fairseq-roberta-spm-normal": "fairseq-roberta-all-model/dict.txt",
+    }
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "fairseq-roberta-spm-normal": 512,
+}
+
+
+class FairSeqRobertaSentencePieceTokenizer(XLMRobertaTokenizer):
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+
+    def __init__(
+            self,
+            spm_model,
+            custom_vocab_file,
+            bos_token="[CLS]",
+            eos_token="[SEP]",
+            sep_token="[SEP]",
+            cls_token="[CLS]",
+            unk_token="[UNK]",
+            pad_token="[PAD]",
+            mask_token="[MASK]",
+            **kwargs
+    ):
+        super().__init__(
+            vocab_file=spm_model,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            sep_token=sep_token,
+            cls_token=cls_token,
+            pad_token=pad_token,
+            mask_token=mask_token,
+            **kwargs,
+        )
+
+        # FairSeq dictioanry: <s>, <pad>, </s>, <unk>, token1, token2, ..., tokenN, <mask>
+        self.symbols = []
+        self.count = []
+        self.spm_id_to_fairseq_id = {}
+        self._add_symbol(self.sp_model.PieceToId(bos_token))
+        self._add_symbol(self.sp_model.PieceToId(pad_token))
+        self._add_symbol(self.sp_model.PieceToId(eos_token))
+        self._add_symbol(self.sp_model.PieceToId(unk_token))
+        self._add_from_file(custom_vocab_file)
+        self._add_symbol(self.sp_model.PieceToId(mask_token))
+
+        self.fairseq_tokens_to_ids = {}
+        self.fairseq_tokens_to_ids = self._build_fairseq_tokens_to_ids()
+        # self.spm_id_to_fairseq_id(bridge vocab)을 이용해서 real token -> fairseq id로 연결해주는 vocabulary
+        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}
+
+        # Collect some stats like OOV rate.
+        self._num_tokens_converted = 0
+        self._num_tokens_oov = 0
+
+    @property
+    def vocab_size(self):
+        return len(self.symbols)
+
+    @property
+    def pad_token_id(self):
+        return self.fairseq_tokens_to_ids.get(self.pad_token)
+
+    @property
+    def unk_token_id(self):
+        return self.fairseq_tokens_to_ids.get(self.unk_token)
+
+    def reset_stats(self):
+        self._num_tokens_converted = 0
+        self._num_tokens_oov = 0
+
+    def get_stats(self):
+        oov_rate = self._num_tokens_oov / self._num_tokens_converted
+        result = {
+            "total": self._num_tokens_converted,
+            "oov": self._num_tokens_oov,
+            "oov_rate": oov_rate
+        }
+        return result
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str) in an id using the vocab. """
+        self._num_tokens_converted += 1
+        if token in self.fairseq_tokens_to_ids:
+            return self.fairseq_tokens_to_ids[token]
+        else:
+            self._num_tokens_oov += 1
+            return self.unk_token_id
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.fairseq_ids_to_tokens:
+            return self.fairseq_ids_to_tokens[index]
+        else:
+            return self.unk_token
+
+    def _add_from_file(self, f):
+        """
+        Source: FairSeq Dictionary class.
+        Loads a pre-existing dictionary from a text file and adds its symbols
+        to this instance.
+        """
+        if isinstance(f, str):
+            try:
+                with open(f, "r", encoding="utf-8") as fd:
+                    self._add_from_file(fd)
+            except FileNotFoundError as fnfe:
+                raise fnfe
+            except UnicodeError:
+                raise Exception(
+                    "Incorrect encoding detected in {}, please "
+                    "rebuild the dataset".format(f)
+                )
+            return
+
+        lines = f.readlines()
+        indices_start_line = 0
+
+        for line in lines[indices_start_line:]:
+            try:
+                line, field = line.rstrip().rsplit(" ", 1)
+                if field == "#fairseq:overwrite":
+                    overwrite = True
+                    line, field = line.rsplit(" ", 1)
+                else:
+                    overwrite = False
+                count = int(field)
+                spm_id = line
+                if spm_id in self.spm_id_to_fairseq_id and not overwrite:
+                    raise RuntimeError(
+                        "Duplicate word found when loading Dictionary: '{}'. "
+                        "Duplicate words can overwrite earlier ones by adding the "
+                        "#fairseq:overwrite flag at the end of the corresponding row "
+                        "in the dictionary file. If using the Camembert model, please "
+                        "download an updated copy of the model file."
+                        .format(spm_id)
+                    )
+                self._add_symbol(spm_id, n=count, overwrite=overwrite)
+            except ValueError:
+                raise ValueError(
+                    "Incorrect dictionary format, expected '<token> <cnt> [flags]'"
+                )
+
+    def _add_symbol(self, spm_id, n=1, overwrite=False):
+        """
+        Source: FairSeq Dictionary class.
+        Adds a word to the dictionary
+        """
+        if spm_id in self.spm_id_to_fairseq_id and not overwrite:
+            idx = self.spm_id_to_fairseq_id[spm_id]
+            self.count[idx] = self.count[idx] + n
+            return idx
+        else:
+            idx = len(self.symbols)
+            self.spm_id_to_fairseq_id[spm_id] = idx
+            self.symbols.append(spm_id)
+            self.count.append(n)
+            return idx
+
+    def _build_fairseq_tokens_to_ids(self):
+        # self.spm_id_to_fairseq_id(bridge vocab)을 이용해서 real token -> fairseq id로 연결해주는 vocabulary 빌드
+        fairseq_tokens_to_ids = self.fairseq_tokens_to_ids
+        for spm_id, fairseq_id in self.spm_id_to_fairseq_id.items():
+            if isinstance(spm_id, str) and "madeup" in spm_id:
+                print("[PASS] spm_id: {} | fairseq_id: {}".format(spm_id, fairseq_id))
+                continue
+            token = self.sp_model.IdToPiece(int(spm_id))
+            # print("token: {} | spm_id: {} | fairseq_id: {}".format(token, spm_id, fairseq_id))
+            fairseq_tokens_to_ids[str(token)] = fairseq_id
+        return fairseq_tokens_to_ids

tokenizer_config.json

@@ -0,0 +1 @@
+{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "cls_token": "[CLS]", "pad_token": "[PAD]", "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sp_model_kwargs": {}, "special_tokens_map_file": null, "name_or_path": "fairseq-roberta-all-model", "tokenizer_class": "FairSeqRobertaSentencePieceTokenizer"}