mnazari commited on
Commit
ff937ff
1 Parent(s): be14937

Upload tokenizer

Browse files
Files changed (2) hide show
  1. added_tokens.json +2 -2
  2. vocab.json +16 -20
added_tokens.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "</s>": 38,
3
- "<s>": 37
4
  }
 
1
  {
2
+ "</s>": 34,
3
+ "<s>": 33
4
  }
vocab.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
- "[PAD]": 36,
3
- "[UNK]": 35,
4
  "a": 1,
5
  "b": 2,
6
  "c": 3,
@@ -17,23 +17,19 @@
17
  "n": 14,
18
  "o": 15,
19
  "p": 16,
20
- "q": 17,
21
- "r": 18,
22
- "s": 19,
23
- "t": 20,
24
- "u": 21,
25
- "v": 22,
26
- "w": 23,
27
- "x": 24,
28
- "y": 25,
29
- "z": 26,
30
  "|": 0,
31
- "ç": 27,
32
- "ë": 28,
33
- "ö": 29,
34
- "ü": 30,
35
- "ğ": 31,
36
- "ı": 32,
37
- "ş": 33,
38
- "̇": 34
39
  }
 
1
  {
2
+ "[PAD]": 32,
3
+ "[UNK]": 31,
4
  "a": 1,
5
  "b": 2,
6
  "c": 3,
 
17
  "n": 14,
18
  "o": 15,
19
  "p": 16,
20
+ "r": 17,
21
+ "s": 18,
22
+ "t": 19,
23
+ "u": 20,
24
+ "v": 21,
25
+ "y": 22,
26
+ "z": 23,
 
 
 
27
  "|": 0,
28
+ "ç": 24,
29
+ "ö": 25,
30
+ "ü": 26,
31
+ "ğ": 27,
32
+ "ı": 28,
33
+ "ş": 29,
34
+ "̇": 30
 
35
  }