hadifar commited on
Commit
7271d17
1 Parent(s): 24e61e2

Upload tokenizer

Browse files
Files changed (3) hide show
  1. merges.txt +1 -1
  2. tokenizer.json +7 -1
  3. tokenizer_config.json +1 -1
merges.txt CHANGED
@@ -1,4 +1,4 @@
1
- #version: 0.2 - Trained by `huggingface/tokenizers`
2
  Ġ t
3
  Ġ a
4
  h e
 
1
+ #version: 0.2
2
  Ġ t
3
  Ġ a
4
  h e
tokenizer.json CHANGED
@@ -1,6 +1,11 @@
1
  {
2
  "version": "1.0",
3
- "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
@@ -82,6 +87,7 @@
82
  "continuing_subword_prefix": "",
83
  "end_of_word_suffix": "",
84
  "fuse_unk": false,
 
85
  "vocab": {
86
  "<s>": 0,
87
  "<pad>": 1,
 
1
  {
2
  "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 512,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
  "padding": null,
10
  "added_tokens": [
11
  {
 
87
  "continuing_subword_prefix": "",
88
  "end_of_word_suffix": "",
89
  "fuse_unk": false,
90
+ "byte_fallback": false,
91
  "vocab": {
92
  "<s>": 0,
93
  "<pad>": 1,
tokenizer_config.json CHANGED
@@ -1,6 +1,7 @@
1
  {
2
  "add_prefix_space": false,
3
  "bos_token": "<s>",
 
4
  "cls_token": "<s>",
5
  "eos_token": "</s>",
6
  "errors": "replace",
@@ -8,7 +9,6 @@
8
  "model_max_length": 512,
9
  "pad_token": "<pad>",
10
  "sep_token": "</s>",
11
- "special_tokens_map_file": null,
12
  "tokenizer_class": "RobertaTokenizer",
13
  "trim_offsets": true,
14
  "unk_token": "<unk>"
 
1
  {
2
  "add_prefix_space": false,
3
  "bos_token": "<s>",
4
+ "clean_up_tokenization_spaces": true,
5
  "cls_token": "<s>",
6
  "eos_token": "</s>",
7
  "errors": "replace",
 
9
  "model_max_length": 512,
10
  "pad_token": "<pad>",
11
  "sep_token": "</s>",
 
12
  "tokenizer_class": "RobertaTokenizer",
13
  "trim_offsets": true,
14
  "unk_token": "<unk>"