florentianayuwono commited on
Commit
8de1c05
1 Parent(s): 440b47d

Upload tokenizer

Browse files
added_tokens.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "~~~": 32100
3
+ }
special_tokens_map.json CHANGED
@@ -1,14 +1,125 @@
1
  {
2
  "additional_special_tokens": [
3
- {
4
- "content": "~~~",
5
- "lstrip": false,
6
- "normalized": false,
7
- "rstrip": false,
8
- "single_word": false
9
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  ],
11
- "eos_token": "</s>",
12
- "pad_token": "<pad>",
13
- "unk_token": "<unk>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  }
 
1
  {
2
  "additional_special_tokens": [
3
+ "<extra_id_0>",
4
+ "<extra_id_1>",
5
+ "<extra_id_2>",
6
+ "<extra_id_3>",
7
+ "<extra_id_4>",
8
+ "<extra_id_5>",
9
+ "<extra_id_6>",
10
+ "<extra_id_7>",
11
+ "<extra_id_8>",
12
+ "<extra_id_9>",
13
+ "<extra_id_10>",
14
+ "<extra_id_11>",
15
+ "<extra_id_12>",
16
+ "<extra_id_13>",
17
+ "<extra_id_14>",
18
+ "<extra_id_15>",
19
+ "<extra_id_16>",
20
+ "<extra_id_17>",
21
+ "<extra_id_18>",
22
+ "<extra_id_19>",
23
+ "<extra_id_20>",
24
+ "<extra_id_21>",
25
+ "<extra_id_22>",
26
+ "<extra_id_23>",
27
+ "<extra_id_24>",
28
+ "<extra_id_25>",
29
+ "<extra_id_26>",
30
+ "<extra_id_27>",
31
+ "<extra_id_28>",
32
+ "<extra_id_29>",
33
+ "<extra_id_30>",
34
+ "<extra_id_31>",
35
+ "<extra_id_32>",
36
+ "<extra_id_33>",
37
+ "<extra_id_34>",
38
+ "<extra_id_35>",
39
+ "<extra_id_36>",
40
+ "<extra_id_37>",
41
+ "<extra_id_38>",
42
+ "<extra_id_39>",
43
+ "<extra_id_40>",
44
+ "<extra_id_41>",
45
+ "<extra_id_42>",
46
+ "<extra_id_43>",
47
+ "<extra_id_44>",
48
+ "<extra_id_45>",
49
+ "<extra_id_46>",
50
+ "<extra_id_47>",
51
+ "<extra_id_48>",
52
+ "<extra_id_49>",
53
+ "<extra_id_50>",
54
+ "<extra_id_51>",
55
+ "<extra_id_52>",
56
+ "<extra_id_53>",
57
+ "<extra_id_54>",
58
+ "<extra_id_55>",
59
+ "<extra_id_56>",
60
+ "<extra_id_57>",
61
+ "<extra_id_58>",
62
+ "<extra_id_59>",
63
+ "<extra_id_60>",
64
+ "<extra_id_61>",
65
+ "<extra_id_62>",
66
+ "<extra_id_63>",
67
+ "<extra_id_64>",
68
+ "<extra_id_65>",
69
+ "<extra_id_66>",
70
+ "<extra_id_67>",
71
+ "<extra_id_68>",
72
+ "<extra_id_69>",
73
+ "<extra_id_70>",
74
+ "<extra_id_71>",
75
+ "<extra_id_72>",
76
+ "<extra_id_73>",
77
+ "<extra_id_74>",
78
+ "<extra_id_75>",
79
+ "<extra_id_76>",
80
+ "<extra_id_77>",
81
+ "<extra_id_78>",
82
+ "<extra_id_79>",
83
+ "<extra_id_80>",
84
+ "<extra_id_81>",
85
+ "<extra_id_82>",
86
+ "<extra_id_83>",
87
+ "<extra_id_84>",
88
+ "<extra_id_85>",
89
+ "<extra_id_86>",
90
+ "<extra_id_87>",
91
+ "<extra_id_88>",
92
+ "<extra_id_89>",
93
+ "<extra_id_90>",
94
+ "<extra_id_91>",
95
+ "<extra_id_92>",
96
+ "<extra_id_93>",
97
+ "<extra_id_94>",
98
+ "<extra_id_95>",
99
+ "<extra_id_96>",
100
+ "<extra_id_97>",
101
+ "<extra_id_98>",
102
+ "<extra_id_99>"
103
  ],
104
+ "eos_token": {
105
+ "content": "</s>",
106
+ "lstrip": false,
107
+ "normalized": false,
108
+ "rstrip": false,
109
+ "single_word": false
110
+ },
111
+ "pad_token": {
112
+ "content": "<pad>",
113
+ "lstrip": false,
114
+ "normalized": false,
115
+ "rstrip": false,
116
+ "single_word": false
117
+ },
118
+ "unk_token": {
119
+ "content": "<unk>",
120
+ "lstrip": false,
121
+ "normalized": false,
122
+ "rstrip": false,
123
+ "single_word": false
124
+ }
125
  }
tokenizer.json CHANGED
@@ -936,8 +936,8 @@
936
  "single_word": false,
937
  "lstrip": false,
938
  "rstrip": false,
939
- "normalized": false,
940
- "special": true
941
  }
942
  ],
943
  "normalizer": {
 
936
  "single_word": false,
937
  "lstrip": false,
938
  "rstrip": false,
939
+ "normalized": true,
940
+ "special": false
941
  }
942
  ],
943
  "normalizer": {
tokenizer_config.json CHANGED
@@ -827,19 +827,118 @@
827
  "32100": {
828
  "content": "~~~",
829
  "lstrip": false,
830
- "normalized": false,
831
  "rstrip": false,
832
  "single_word": false,
833
- "special": true
834
  }
835
  },
836
  "additional_special_tokens": [
837
- "~~~"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
838
  ],
839
  "clean_up_tokenization_spaces": true,
840
  "eos_token": "</s>",
841
  "extra_ids": 100,
842
- "model_max_length": 1000000000000000019884624838656,
843
  "pad_token": "<pad>",
844
  "tokenizer_class": "T5Tokenizer",
845
  "unk_token": "<unk>"
 
827
  "32100": {
828
  "content": "~~~",
829
  "lstrip": false,
830
+ "normalized": true,
831
  "rstrip": false,
832
  "single_word": false,
833
+ "special": false
834
  }
835
  },
836
  "additional_special_tokens": [
837
+ "<extra_id_0>",
838
+ "<extra_id_1>",
839
+ "<extra_id_2>",
840
+ "<extra_id_3>",
841
+ "<extra_id_4>",
842
+ "<extra_id_5>",
843
+ "<extra_id_6>",
844
+ "<extra_id_7>",
845
+ "<extra_id_8>",
846
+ "<extra_id_9>",
847
+ "<extra_id_10>",
848
+ "<extra_id_11>",
849
+ "<extra_id_12>",
850
+ "<extra_id_13>",
851
+ "<extra_id_14>",
852
+ "<extra_id_15>",
853
+ "<extra_id_16>",
854
+ "<extra_id_17>",
855
+ "<extra_id_18>",
856
+ "<extra_id_19>",
857
+ "<extra_id_20>",
858
+ "<extra_id_21>",
859
+ "<extra_id_22>",
860
+ "<extra_id_23>",
861
+ "<extra_id_24>",
862
+ "<extra_id_25>",
863
+ "<extra_id_26>",
864
+ "<extra_id_27>",
865
+ "<extra_id_28>",
866
+ "<extra_id_29>",
867
+ "<extra_id_30>",
868
+ "<extra_id_31>",
869
+ "<extra_id_32>",
870
+ "<extra_id_33>",
871
+ "<extra_id_34>",
872
+ "<extra_id_35>",
873
+ "<extra_id_36>",
874
+ "<extra_id_37>",
875
+ "<extra_id_38>",
876
+ "<extra_id_39>",
877
+ "<extra_id_40>",
878
+ "<extra_id_41>",
879
+ "<extra_id_42>",
880
+ "<extra_id_43>",
881
+ "<extra_id_44>",
882
+ "<extra_id_45>",
883
+ "<extra_id_46>",
884
+ "<extra_id_47>",
885
+ "<extra_id_48>",
886
+ "<extra_id_49>",
887
+ "<extra_id_50>",
888
+ "<extra_id_51>",
889
+ "<extra_id_52>",
890
+ "<extra_id_53>",
891
+ "<extra_id_54>",
892
+ "<extra_id_55>",
893
+ "<extra_id_56>",
894
+ "<extra_id_57>",
895
+ "<extra_id_58>",
896
+ "<extra_id_59>",
897
+ "<extra_id_60>",
898
+ "<extra_id_61>",
899
+ "<extra_id_62>",
900
+ "<extra_id_63>",
901
+ "<extra_id_64>",
902
+ "<extra_id_65>",
903
+ "<extra_id_66>",
904
+ "<extra_id_67>",
905
+ "<extra_id_68>",
906
+ "<extra_id_69>",
907
+ "<extra_id_70>",
908
+ "<extra_id_71>",
909
+ "<extra_id_72>",
910
+ "<extra_id_73>",
911
+ "<extra_id_74>",
912
+ "<extra_id_75>",
913
+ "<extra_id_76>",
914
+ "<extra_id_77>",
915
+ "<extra_id_78>",
916
+ "<extra_id_79>",
917
+ "<extra_id_80>",
918
+ "<extra_id_81>",
919
+ "<extra_id_82>",
920
+ "<extra_id_83>",
921
+ "<extra_id_84>",
922
+ "<extra_id_85>",
923
+ "<extra_id_86>",
924
+ "<extra_id_87>",
925
+ "<extra_id_88>",
926
+ "<extra_id_89>",
927
+ "<extra_id_90>",
928
+ "<extra_id_91>",
929
+ "<extra_id_92>",
930
+ "<extra_id_93>",
931
+ "<extra_id_94>",
932
+ "<extra_id_95>",
933
+ "<extra_id_96>",
934
+ "<extra_id_97>",
935
+ "<extra_id_98>",
936
+ "<extra_id_99>"
937
  ],
938
  "clean_up_tokenization_spaces": true,
939
  "eos_token": "</s>",
940
  "extra_ids": 100,
941
+ "model_max_length": 512,
942
  "pad_token": "<pad>",
943
  "tokenizer_class": "T5Tokenizer",
944
  "unk_token": "<unk>"