johngiorgi commited on
Commit
780539e
1 Parent(s): b975141

Upload tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -16
tokenizer_config.json CHANGED
@@ -1,20 +1,5 @@
1
  {
2
  "add_prefix_space": false,
3
- "additional_special_tokens": [
4
- "<background>",
5
- "</background>",
6
- "<ref>",
7
- "</ref>",
8
- "<sep>",
9
- "<pop>",
10
- "</pop>",
11
- "<int>",
12
- "</int>",
13
- "<out>",
14
- "</out>",
15
- "<evidence>",
16
- "</evidence>"
17
- ],
18
  "bos_token": {
19
  "__type": "AddedToken",
20
  "content": "<s>",
@@ -66,7 +51,7 @@
66
  "rstrip": false,
67
  "single_word": false
68
  },
69
- "special_tokens_map_file": "/Users/johngiorgi/.cache/huggingface/transformers/05da652a7fca41c1c18027c1201e473217bb373e370d1283e3de49d5880cbf0c.cb2244924ab24d706b02fd7fcedaea4531566537687a539ebb94db511fd122a0",
70
  "tokenizer_class": "LEDTokenizer",
71
  "trim_offsets": true,
72
  "unk_token": {
 
1
  {
2
  "add_prefix_space": false,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  "bos_token": {
4
  "__type": "AddedToken",
5
  "content": "<s>",
 
51
  "rstrip": false,
52
  "single_word": false
53
  },
54
+ "special_tokens_map_file": "/home/johnmg/.cache/huggingface/transformers/05da652a7fca41c1c18027c1201e473217bb373e370d1283e3de49d5880cbf0c.cb2244924ab24d706b02fd7fcedaea4531566537687a539ebb94db511fd122a0",
55
  "tokenizer_class": "LEDTokenizer",
56
  "trim_offsets": true,
57
  "unk_token": {