hyunwoongko commited on
Commit
7a6ce4c
1 Parent(s): b4eab72

Upload tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +5 -5
tokenizer.json CHANGED
@@ -5,7 +5,7 @@
5
  "added_tokens": [
6
  {
7
  "id": 0,
8
- "content": "<pad>",
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
@@ -14,7 +14,7 @@
14
  },
15
  {
16
  "id": 1,
17
- "content": "<eos>",
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
@@ -107,8 +107,8 @@
107
  "end_of_word_suffix": null,
108
  "fuse_unk": false,
109
  "vocab": {
110
- "<pad>": 0,
111
- "<eos>": 1,
112
  "<|endoftext|>": 2,
113
  "<|sep|>": 3,
114
  "!": 4,
@@ -59851,4 +59851,4 @@
59851
  "ìºIJ롤ëĿ¼ ìĿ´ëĤĺ"
59852
  ]
59853
  }
59854
- }
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
8
+ "content": "<|unused0|>",
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
 
14
  },
15
  {
16
  "id": 1,
17
+ "content": "<|unused1|>",
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
 
107
  "end_of_word_suffix": null,
108
  "fuse_unk": false,
109
  "vocab": {
110
+ "<|unused0|>": 0,
111
+ "<|unused1|>": 1,
112
  "<|endoftext|>": 2,
113
  "<|sep|>": 3,
114
  "!": 4,
 
59851
  "ìºIJ롤ëĿ¼ ìĿ´ëĤĺ"
59852
  ]
59853
  }
59854
+ }