Hugging Face's logo

EddieChen372
/
longT5-js2jest

Text2Text Generation
Transformers
PyTorch
t5
Inference Endpoints
text-generation-inference
Model card Files Community
EddieChen372 commited on
Commit
57c8433
•
1 Parent(s): 43aa45f

add tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +3 -100
  2. tokenizer.json +52 -1
special_tokens_map.json CHANGED
@@ -1,105 +1,8 @@
1
  {
2
  "additional_special_tokens": [
3
- "<extra_id_0>",
4
- "<extra_id_1>",
5
- "<extra_id_2>",
6
- "<extra_id_3>",
7
- "<extra_id_4>",
8
- "<extra_id_5>",
9
- "<extra_id_6>",
10
- "<extra_id_7>",
11
- "<extra_id_8>",
12
- "<extra_id_9>",
13
- "<extra_id_10>",
14
- "<extra_id_11>",
15
- "<extra_id_12>",
16
- "<extra_id_13>",
17
- "<extra_id_14>",
18
- "<extra_id_15>",
19
- "<extra_id_16>",
20
- "<extra_id_17>",
21
- "<extra_id_18>",
22
- "<extra_id_19>",
23
- "<extra_id_20>",
24
- "<extra_id_21>",
25
- "<extra_id_22>",
26
- "<extra_id_23>",
27
- "<extra_id_24>",
28
- "<extra_id_25>",
29
- "<extra_id_26>",
30
- "<extra_id_27>",
31
- "<extra_id_28>",
32
- "<extra_id_29>",
33
- "<extra_id_30>",
34
- "<extra_id_31>",
35
- "<extra_id_32>",
36
- "<extra_id_33>",
37
- "<extra_id_34>",
38
- "<extra_id_35>",
39
- "<extra_id_36>",
40
- "<extra_id_37>",
41
- "<extra_id_38>",
42
- "<extra_id_39>",
43
- "<extra_id_40>",
44
- "<extra_id_41>",
45
- "<extra_id_42>",
46
- "<extra_id_43>",
47
- "<extra_id_44>",
48
- "<extra_id_45>",
49
- "<extra_id_46>",
50
- "<extra_id_47>",
51
- "<extra_id_48>",
52
- "<extra_id_49>",
53
- "<extra_id_50>",
54
- "<extra_id_51>",
55
- "<extra_id_52>",
56
- "<extra_id_53>",
57
- "<extra_id_54>",
58
- "<extra_id_55>",
59
- "<extra_id_56>",
60
- "<extra_id_57>",
61
- "<extra_id_58>",
62
- "<extra_id_59>",
63
- "<extra_id_60>",
64
- "<extra_id_61>",
65
- "<extra_id_62>",
66
- "<extra_id_63>",
67
- "<extra_id_64>",
68
- "<extra_id_65>",
69
- "<extra_id_66>",
70
- "<extra_id_67>",
71
- "<extra_id_68>",
72
- "<extra_id_69>",
73
- "<extra_id_70>",
74
- "<extra_id_71>",
75
- "<extra_id_72>",
76
- "<extra_id_73>",
77
- "<extra_id_74>",
78
- "<extra_id_75>",
79
- "<extra_id_76>",
80
- "<extra_id_77>",
81
- "<extra_id_78>",
82
- "<extra_id_79>",
83
- "<extra_id_80>",
84
- "<extra_id_81>",
85
- "<extra_id_82>",
86
- "<extra_id_83>",
87
- "<extra_id_84>",
88
- "<extra_id_85>",
89
- "<extra_id_86>",
90
- "<extra_id_87>",
91
- "<extra_id_88>",
92
- "<extra_id_89>",
93
- "<extra_id_90>",
94
- "<extra_id_91>",
95
- "<extra_id_92>",
96
- "<extra_id_93>",
97
- "<extra_id_94>",
98
- "<extra_id_95>",
99
- "<extra_id_96>",
100
- "<extra_id_97>",
101
- "<extra_id_98>",
102
- "<extra_id_99>"
103
  ],
104
  "eos_token": "</s>",
105
  "pad_token": "<pad>",
 
1
  {
2
  "additional_special_tokens": [
3
+ "[react]",
4
+ "[end]",
5
+ "[jest]"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  ],
7
  "eos_token": "</s>",
8
  "pad_token": "<pad>",
tokenizer.json CHANGED
@@ -929,6 +929,33 @@
929
  "rstrip": false,
930
  "normalized": false,
931
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
932
  }
933
  ],
934
  "normalizer": {
@@ -965,6 +992,12 @@
965
  }
966
  ],
967
  "pair": [
 
 
 
 
 
 
968
  {
969
  "Sequence": {
970
  "id": "A",
@@ -973,7 +1006,7 @@
973
  },
974
  {
975
  "SpecialToken": {
976
- "id": "</s>",
977
  "type_id": 0
978
  }
979
  },
@@ -999,6 +1032,24 @@
999
  "tokens": [
1000
  "</s>"
1001
  ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1002
  }
1003
  }
1004
  },
 
929
  "rstrip": false,
930
  "normalized": false,
931
  "special": true
932
+ },
933
+ {
934
+ "id": 52000,
935
+ "content": "[react]",
936
+ "single_word": false,
937
+ "lstrip": false,
938
+ "rstrip": false,
939
+ "normalized": false,
940
+ "special": true
941
+ },
942
+ {
943
+ "id": 52001,
944
+ "content": "[end]",
945
+ "single_word": false,
946
+ "lstrip": false,
947
+ "rstrip": false,
948
+ "normalized": false,
949
+ "special": true
950
+ },
951
+ {
952
+ "id": 52002,
953
+ "content": "[jest]",
954
+ "single_word": false,
955
+ "lstrip": false,
956
+ "rstrip": false,
957
+ "normalized": false,
958
+ "special": true
959
  }
960
  ],
961
  "normalizer": {
 
992
  }
993
  ],
994
  "pair": [
995
+ {
996
+ "SpecialToken": {
997
+ "id": "[react]",
998
+ "type_id": 0
999
+ }
1000
+ },
1001
  {
1002
  "Sequence": {
1003
  "id": "A",
 
1006
  },
1007
  {
1008
  "SpecialToken": {
1009
+ "id": "[jest]",
1010
  "type_id": 0
1011
  }
1012
  },
 
1032
  "tokens": [
1033
  "</s>"
1034
  ]
1035
+ },
1036
+ "[jest]": {
1037
+ "id": "[jest]",
1038
+ "ids": [
1039
+ 52002
1040
+ ],
1041
+ "tokens": [
1042
+ "[jest]"
1043
+ ]
1044
+ },
1045
+ "[react]": {
1046
+ "id": "[react]",
1047
+ "ids": [
1048
+ 52000
1049
+ ],
1050
+ "tokens": [
1051
+ "[react]"
1052
+ ]
1053
  }
1054
  }
1055
  },