AdamOswald1 commited on
Commit
0df9344
1 Parent(s): ae37e52

Update tokenizer/special_tokens_map.json

Browse files
Files changed (1) hide show
  1. tokenizer/special_tokens_map.json +0 -23
tokenizer/special_tokens_map.json CHANGED
@@ -22,26 +22,3 @@
22
  "single_word": false
23
  }
24
  }
25
-
26
- {
27
- "bos_token": {
28
- "content": "<|startoftext|>",
29
- "single_word": false,
30
- "lstrip": false,
31
- "rstrip": false,
32
- "normalized": true
33
- },
34
- "eos_token": {
35
- "content": "<|endoftext|>",
36
- "single_word": false,
37
- "lstrip": false, "rstrip": false,
38
- "normalized": true
39
- },
40
- "unk_token": {"content": "<|endoftext|>",
41
- "single_word": false,
42
- "lstrip": false,
43
- "rstrip": false,
44
- "normalized": true
45
- },
46
- "pad_token": "<|endoftext|>"
47
- }
 
22
  "single_word": false
23
  }
24
  }