EddieChen372
/

incoder-1B-finetuned-jest

Text Generation

Inference Endpoints

Model card Files Files and versions Community

EddieChen372 commited on Jun 26, 2022

Commit

ef59ff8

•

1 Parent(s): 39598bf

add tokenizer

Files changed (2) hide show

special_tokens_map.json +8 -1
tokenizer.json +59 -8

special_tokens_map.json CHANGED Viewed

@@ -1,3 +1,10 @@
 {
-  "bos_token": "<|endoftext|>"
 }

 {
+  "additional_special_tokens": [
+    "[react]",
+    "[end]",
+    "[jest]"
+  ],
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
 }

tokenizer.json CHANGED Viewed

@@ -2324,6 +2324,33 @@
       "rstrip": false,
       "normalized": false,
       "special": true
     }
   ],
   "normalizer": null,
@@ -2350,14 +2377,14 @@
     "type": "TemplateProcessing",
     "single": [
       {
-        "SpecialToken": {
-          "id": "<|endoftext|>",
           "type_id": 0
         }
       },
       {
-        "Sequence": {
-          "id": "A",
           "type_id": 0
         }
       }
@@ -2365,7 +2392,7 @@
     "pair": [
       {
         "SpecialToken": {
-          "id": "<|endoftext|>",
           "type_id": 0
         }
       },
@@ -2377,14 +2404,20 @@
       },
       {
         "SpecialToken": {
-          "id": "<|endoftext|>",
-          "type_id": 1
         }
       },
       {
         "Sequence": {
           "id": "B",
-          "type_id": 1
         }
       }
     ],
@@ -2397,6 +2430,24 @@
         "tokens": [
           "<|endoftext|>"
         ]
       }
     }
   },

       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 52000,
+      "content": "[react]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 52001,
+      "content": "[end]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 52002,
+      "content": "[jest]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
     "type": "TemplateProcessing",
     "single": [
       {
+        "Sequence": {
+          "id": "A",
           "type_id": 0
         }
       },
       {
+        "SpecialToken": {
+          "id": "<|endoftext|>",
           "type_id": 0
         }
       }
     "pair": [
       {
         "SpecialToken": {
+          "id": "[react]",
           "type_id": 0
         }
       },
       },
       {
         "SpecialToken": {
+          "id": "[jest]",
+          "type_id": 0
         }
       },
       {
         "Sequence": {
           "id": "B",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<|endoftext|>",
+          "type_id": 0
         }
       }
     ],
         "tokens": [
           "<|endoftext|>"
         ]
+      },
+      "[jest]": {
+        "id": "[jest]",
+        "ids": [
+          52002
+        ],
+        "tokens": [
+          "[jest]"
+        ]
+      },
+      "[react]": {
+        "id": "[react]",
+        "ids": [
+          52000
+        ],
+        "tokens": [
+          "[react]"
+        ]
       }
     }
   },