change hebrew code from iw (deprecated) to he
Browse filesSigned-off-by: Alex Volkov <altryne@gmail.com>
- README.md +1 -1
- added_tokens.json +1 -1
- generation_config.json +1 -1
- special_tokens_map.json +1 -1
- tokenizer.json +1 -1
README.md
CHANGED
@@ -20,7 +20,7 @@ language:
|
|
20 |
- hi
|
21 |
- fi
|
22 |
- vi
|
23 |
-
-
|
24 |
- uk
|
25 |
- el
|
26 |
- ms
|
|
|
20 |
- hi
|
21 |
- fi
|
22 |
- vi
|
23 |
+
- he
|
24 |
- uk
|
25 |
- el
|
26 |
- ms
|
added_tokens.json
CHANGED
@@ -29,6 +29,7 @@
|
|
29 |
"<|gu|>": 50333,
|
30 |
"<|haw|>": 50352,
|
31 |
"<|ha|>": 50354,
|
|
|
32 |
"<|hi|>": 50276,
|
33 |
"<|hr|>": 50291,
|
34 |
"<|ht|>": 50339,
|
@@ -37,7 +38,6 @@
|
|
37 |
"<|id|>": 50275,
|
38 |
"<|is|>": 50311,
|
39 |
"<|it|>": 50274,
|
40 |
-
"<|iw|>": 50279,
|
41 |
"<|ja|>": 50266,
|
42 |
"<|jw|>": 50356,
|
43 |
"<|ka|>": 50329,
|
|
|
29 |
"<|gu|>": 50333,
|
30 |
"<|haw|>": 50352,
|
31 |
"<|ha|>": 50354,
|
32 |
+
"<|he|>": 50279,
|
33 |
"<|hi|>": 50276,
|
34 |
"<|hr|>": 50291,
|
35 |
"<|ht|>": 50339,
|
|
|
38 |
"<|id|>": 50275,
|
39 |
"<|is|>": 50311,
|
40 |
"<|it|>": 50274,
|
|
|
41 |
"<|ja|>": 50266,
|
42 |
"<|jw|>": 50356,
|
43 |
"<|ka|>": 50329,
|
generation_config.json
CHANGED
@@ -49,6 +49,7 @@
|
|
49 |
"<|gu|>": 50333,
|
50 |
"<|haw|>": 50352,
|
51 |
"<|ha|>": 50354,
|
|
|
52 |
"<|hi|>": 50276,
|
53 |
"<|hr|>": 50291,
|
54 |
"<|ht|>": 50339,
|
@@ -57,7 +58,6 @@
|
|
57 |
"<|id|>": 50275,
|
58 |
"<|is|>": 50311,
|
59 |
"<|it|>": 50274,
|
60 |
-
"<|iw|>": 50279,
|
61 |
"<|ja|>": 50266,
|
62 |
"<|jw|>": 50356,
|
63 |
"<|ka|>": 50329,
|
|
|
49 |
"<|gu|>": 50333,
|
50 |
"<|haw|>": 50352,
|
51 |
"<|ha|>": 50354,
|
52 |
+
"<|he|>": 50279,
|
53 |
"<|hi|>": 50276,
|
54 |
"<|hr|>": 50291,
|
55 |
"<|ht|>": 50339,
|
|
|
58 |
"<|id|>": 50275,
|
59 |
"<|is|>": 50311,
|
60 |
"<|it|>": 50274,
|
|
|
61 |
"<|ja|>": 50266,
|
62 |
"<|jw|>": 50356,
|
63 |
"<|ka|>": 50329,
|
special_tokens_map.json
CHANGED
@@ -22,7 +22,7 @@
|
|
22 |
"<|hi|>",
|
23 |
"<|fi|>",
|
24 |
"<|vi|>",
|
25 |
-
"<|
|
26 |
"<|uk|>",
|
27 |
"<|el|>",
|
28 |
"<|ms|>",
|
|
|
22 |
"<|hi|>",
|
23 |
"<|fi|>",
|
24 |
"<|vi|>",
|
25 |
+
"<|he|>",
|
26 |
"<|uk|>",
|
27 |
"<|el|>",
|
28 |
"<|ms|>",
|
tokenizer.json
CHANGED
@@ -203,7 +203,7 @@
|
|
203 |
},
|
204 |
{
|
205 |
"id": 50279,
|
206 |
-
"content": "<|
|
207 |
"single_word": false,
|
208 |
"lstrip": false,
|
209 |
"rstrip": false,
|
|
|
203 |
},
|
204 |
{
|
205 |
"id": 50279,
|
206 |
+
"content": "<|he|>",
|
207 |
"single_word": false,
|
208 |
"lstrip": false,
|
209 |
"rstrip": false,
|