asahi417 commited on
Commit
85d7b18
1 Parent(s): c236d13

add tokenizer

Browse files
added_tokens.json CHANGED
@@ -1 +1,3 @@
1
- {"<hl>": 250027}
 
 
 
1
+ {
2
+ "<hl>": 250027
3
+ }
special_tokens_map.json CHANGED
@@ -1 +1,18 @@
1
- {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}, "additional_special_tokens": ["<hl>"]}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<hl>"
4
+ ],
5
+ "bos_token": "<s>",
6
+ "cls_token": "<s>",
7
+ "eos_token": "</s>",
8
+ "mask_token": {
9
+ "content": "<mask>",
10
+ "lstrip": true,
11
+ "normalized": false,
12
+ "rstrip": false,
13
+ "single_word": false
14
+ },
15
+ "pad_token": "<pad>",
16
+ "sep_token": "</s>",
17
+ "unk_token": "<unk>"
18
+ }
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3059e2f9b01b617fd5cdb5a0561151e431782bf9a31c660556982db8af7a4890
3
  size 17088356
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a11892dd2e34180882f7d01c5f35400945ce4e3883fd977d42e9ba9d37dbece2
3
  size 17088356
tokenizer_config.json CHANGED
@@ -1 +1,23 @@
1
- {"bos_token": "<s>", "eos_token": "</s>", "sep_token": "</s>", "cls_token": "<s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "src_lang": null, "tgt_lang": null, "additional_special_tokens": null, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "lmqg_output/mbart_large_cc25_dequad/model_ywshaf/epoch_5", "tokenizer_class": "MBartTokenizer"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": null,
3
+ "bos_token": "<s>",
4
+ "cls_token": "<s>",
5
+ "eos_token": "</s>",
6
+ "mask_token": {
7
+ "__type": "AddedToken",
8
+ "content": "<mask>",
9
+ "lstrip": true,
10
+ "normalized": true,
11
+ "rstrip": false,
12
+ "single_word": false
13
+ },
14
+ "model_max_length": 1024,
15
+ "name_or_path": "mbart-large-cc25-dequad",
16
+ "pad_token": "<pad>",
17
+ "sep_token": "</s>",
18
+ "special_tokens_map_file": null,
19
+ "src_lang": null,
20
+ "tgt_lang": null,
21
+ "tokenizer_class": "MBartTokenizer",
22
+ "unk_token": "<unk>"
23
+ }