Hugging Face's logo

muzammil-eds
/
aralense-base-v3.0

Image-Text-to-Text
Transformers
PyTorch
vision-encoder-decoder
Inference Endpoints
Model card Files Community
4

Upload tokenizer

#4
by muzammil-eds - opened
base: refs/heads/main
โ†
from: refs/pr/4
Discussion Files changed
+128498
-0
Files changed (4) hide show
  1. special_tokens_map.json +7 -0
  2. tokenizer.json +0 -0
  3. tokenizer_config.json +48 -0
  4. vocab.txt +0 -0
special_tokens_map.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": "[CLS]",
3
+ "mask_token": "[MASK]",
4
+ "pad_token": "[PAD]",
5
+ "sep_token": "[SEP]",
6
+ "unk_token": "[UNK]"
7
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "clean_up_tokenization_spaces": true,
3
+ "cls_token": "[CLS]",
4
+ "do_basic_tokenize": true,
5
+ "do_lower_case": false,
6
+ "mask_token": "[MASK]",
7
+ "max_len": 512,
8
+ "model_max_length": 512,
9
+ "never_split": [
10
+ "+ูƒ",
11
+ "+ูƒู…ุง",
12
+ "ูƒ+",
13
+ "+ูˆุง",
14
+ "+ูŠู†",
15
+ "ูˆ+",
16
+ "+ูƒู†",
17
+ "+ุงู†",
18
+ "+ู‡ู…",
19
+ "+ุฉ",
20
+ "[ุจุฑูŠุฏ]",
21
+ "ู„ู„+",
22
+ "+ูŠ",
23
+ "+ุช",
24
+ "+ู†",
25
+ "ุณ+",
26
+ "ู„+",
27
+ "[ู…ุณุชุฎุฏู…]",
28
+ "+ูƒู…",
29
+ "+ุง",
30
+ "ุจ+",
31
+ "ู+",
32
+ "+ู†ุง",
33
+ "+ู‡ุง",
34
+ "+ูˆู†",
35
+ "+ู‡ู…ุง",
36
+ "ุงู„+",
37
+ "+ู‡",
38
+ "+ู‡ู†",
39
+ "+ุงุช",
40
+ "[ุฑุงุจุท]"
41
+ ],
42
+ "pad_token": "[PAD]",
43
+ "sep_token": "[SEP]",
44
+ "strip_accents": null,
45
+ "tokenize_chinese_chars": true,
46
+ "tokenizer_class": "BertTokenizer",
47
+ "unk_token": "[UNK]"
48
+ }
vocab.txt ADDED
The diff for this file is too large to render. See raw diff