leukas commited on
Commit
629498b
1 Parent(s): ff16563

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -1
tokenizer_config.json CHANGED
@@ -126,7 +126,6 @@
126
  "<extra_id_123>",
127
  "<extra_id_124>"
128
  ],
129
- "clean_up_tokenization_spaces": true,
130
  "eos_token": {
131
  "__type": "AddedToken",
132
  "content": "</s>",
@@ -137,6 +136,7 @@
137
  },
138
  "extra_ids": 125,
139
  "model_max_length": 1000000000000000019884624838656,
 
140
  "pad_token": {
141
  "__type": "AddedToken",
142
  "content": "<pad>",
@@ -145,6 +145,7 @@
145
  "rstrip": false,
146
  "single_word": false
147
  },
 
148
  "tokenizer_class": "ByT5Tokenizer",
149
  "unk_token": {
150
  "__type": "AddedToken",
 
126
  "<extra_id_123>",
127
  "<extra_id_124>"
128
  ],
 
129
  "eos_token": {
130
  "__type": "AddedToken",
131
  "content": "</s>",
 
136
  },
137
  "extra_ids": 125,
138
  "model_max_length": 1000000000000000019884624838656,
139
+ "name_or_path": "google/byt5-large",
140
  "pad_token": {
141
  "__type": "AddedToken",
142
  "content": "<pad>",
 
145
  "rstrip": false,
146
  "single_word": false
147
  },
148
+ "special_tokens_map_file": "/scratch/p284491/.cache/huggingface/transformers/models--google--byt5-large/snapshots/ecfdcb58a4246be99fda0597bf398afabb7319c5/special_tokens_map.json",
149
  "tokenizer_class": "ByT5Tokenizer",
150
  "unk_token": {
151
  "__type": "AddedToken",