psuwannapich
commited on
merge from main
Browse files- added_tokens.json +6 -0
- config.json +1 -1
- special_tokens_map.json +1 -1
- tokenizer_config.json +1 -1
added_tokens.json
CHANGED
@@ -1,4 +1,5 @@
|
|
1 |
<<<<<<< HEAD
|
|
|
2 |
{
|
3 |
"‼": 248733,
|
4 |
"⁉": 248850,
|
@@ -841,3 +842,8 @@ version https://git-lfs.github.com/spec/v1
|
|
841 |
oid sha256:efad36c9884d586b5921f4701e27739ea45b56d4248fb87355b7b217ea44b1ab
|
842 |
size 14981
|
843 |
>>>>>>> origin/main
|
|
|
|
|
|
|
|
|
|
|
|
1 |
<<<<<<< HEAD
|
2 |
+
<<<<<<< HEAD
|
3 |
{
|
4 |
"‼": 248733,
|
5 |
"⁉": 248850,
|
|
|
842 |
oid sha256:efad36c9884d586b5921f4701e27739ea45b56d4248fb87355b7b217ea44b1ab
|
843 |
size 14981
|
844 |
>>>>>>> origin/main
|
845 |
+
=======
|
846 |
+
version https://git-lfs.github.com/spec/v1
|
847 |
+
oid sha256:efad36c9884d586b5921f4701e27739ea45b56d4248fb87355b7b217ea44b1ab
|
848 |
+
size 14981
|
849 |
+
>>>>>>> origin/main
|
config.json
CHANGED
@@ -31,4 +31,4 @@
|
|
31 |
"type_vocab_size": 1,
|
32 |
"use_cache": true,
|
33 |
"vocab_size": 249262
|
34 |
-
}
|
|
|
31 |
"type_vocab_size": 1,
|
32 |
"use_cache": true,
|
33 |
"vocab_size": 249262
|
34 |
+
}
|
special_tokens_map.json
CHANGED
@@ -17,4 +17,4 @@
|
|
17 |
"pad_token": "<pad>",
|
18 |
"sep_token": "</s>",
|
19 |
"unk_token": "<unk>"
|
20 |
-
}
|
|
|
17 |
"pad_token": "<pad>",
|
18 |
"sep_token": "</s>",
|
19 |
"unk_token": "<unk>"
|
20 |
+
}
|
tokenizer_config.json
CHANGED
@@ -23,4 +23,4 @@
|
|
23 |
"sp_model_kwargs": {},
|
24 |
"tokenizer_class": "CamembertTokenizer",
|
25 |
"unk_token": "<unk>"
|
26 |
-
}
|
|
|
23 |
"sp_model_kwargs": {},
|
24 |
"tokenizer_class": "CamembertTokenizer",
|
25 |
"unk_token": "<unk>"
|
26 |
+
}
|