StevenTang commited on
Commit
65f46f6
1 Parent(s): 324b8ed

Update vocab

Browse files
Files changed (3) hide show
  1. added_tokens.json +4 -0
  2. tokenizer.json +0 -18
  3. vocab.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "[SEP]": 50265,
3
+ "[X_SEP]": 50266
4
+ }
tokenizer.json CHANGED
@@ -47,24 +47,6 @@
47
  "rstrip": false,
48
  "normalized": false,
49
  "special": true
50
- },
51
- {
52
- "id": 50265,
53
- "content": "[SEP]",
54
- "single_word": false,
55
- "lstrip": true,
56
- "rstrip": false,
57
- "normalized": false,
58
- "special": false
59
- },
60
- {
61
- "id": 50266,
62
- "content": "[X_SEP]",
63
- "single_word": false,
64
- "lstrip": true,
65
- "rstrip": false,
66
- "normalized": false,
67
- "special": false
68
  }
69
  ],
70
  "normalizer": null,
 
47
  "rstrip": false,
48
  "normalized": false,
49
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  }
51
  ],
52
  "normalizer": null,
vocab.json CHANGED
The diff for this file is too large to render. See raw diff