Tianyi Tang commited on
Commit
a08abbe
1 Parent(s): ab74966

Update vocab

Browse files
Files changed (1) hide show
  1. tokenizer.json +0 -18
tokenizer.json CHANGED
@@ -47,24 +47,6 @@
47
  "rstrip": false,
48
  "normalized": false,
49
  "special": true
50
- },
51
- {
52
- "id": 50265,
53
- "content": "[SEP]",
54
- "single_word": false,
55
- "lstrip": true,
56
- "rstrip": false,
57
- "normalized": false,
58
- "special": false
59
- },
60
- {
61
- "id": 50266,
62
- "content": "[X_SEP]",
63
- "single_word": false,
64
- "lstrip": true,
65
- "rstrip": false,
66
- "normalized": false,
67
- "special": false
68
  }
69
  ],
70
  "normalizer": null,
 
47
  "rstrip": false,
48
  "normalized": false,
49
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  }
51
  ],
52
  "normalizer": null,