yvs007
/

bert-base-uncased-2022-habana

Model card Files Files and versions Community

yvs007 commited on Feb 7, 2023

Commit

afc904a

•

1 Parent(s): d191cb7

Upload tokenizer

Browse files

Files changed (2) hide show

tokenizer.json +43 -86
vocab.txt +19 -62

tokenizer.json CHANGED Viewed

@@ -150,93 +150,50 @@
       "[CLS]": 2,
       "[SEP]": 3,
       "[MASK]": 4,
-      "/": 5,
-      "a": 6,
-      "c": 7,
-      "d": 8,
-      "e": 9,
-      "g": 10,
-      "h": 11,
-      "i": 12,
-      "k": 13,
-      "l": 14,
-      "m": 15,
-      "n": 16,
-      "o": 17,
-      "p": 18,
-      "r": 19,
-      "s": 20,
-      "t": 21,
-      "u": 22,
-      "w": 23,
-      "y": 24,
       "##a": 25,
-      "##s": 26,
-      "##t": 27,
-      "##g": 28,
-      "##e": 29,
-      "##r": 30,
-      "##h": 31,
-      "##n": 32,
-      "##i": 33,
-      "##c": 34,
-      "##y": 35,
-      "##o": 36,
-      "##l": 37,
-      "##u": 38,
-      "##m": 39,
-      "##k": 40,
-      "pl": 41,
-      "##as": 42,
-      "##ar": 43,
-      "##re": 44,
-      "##it": 45,
-      "##ot": 46,
-      "##le": 47,
-      "plot": 48,
-      "cas": 49,
-      "di": 50,
-      "et": 51,
-      "ge": 52,
-      "or": 53,
-      "pa": 54,
-      "re": 55,
-      "tit": 56,
-      "wi": 57,
-      "ye": 58,
-      "##su": 59,
-      "##to": 60,
-      "##ge": 61,
-      "##gi": 62,
-      "##hn": 63,
-      "##nre": 64,
-      "##ic": 65,
-      "##igi": 66,
-      "##cto": 67,
-      "##mm": 68,
-      "##ki": 69,
-      "##ase": 70,
-      "##ary": 71,
-      "##recto": 72,
-      "##ity": 73,
-      "##lease": 74,
-      "plotsu": 75,
-      "cast": 76,
-      "directo": 77,
-      "ethn": 78,
-      "genre": 79,
-      "origi": 80,
-      "page": 81,
-      "release": 82,
-      "title": 83,
-      "wiki": 84,
-      "year": 85,
-      "##icity": 86,
-      "##mmary": 87,
-      "plotsummary": 88,
-      "director": 89,
-      "ethnicity": 90,
-      "origin": 91
     }
   }
 }

       "[CLS]": 2,
       "[SEP]": 3,
       "[MASK]": 4,
+      "a": 5,
+      "e": 6,
+      "g": 7,
+      "i": 8,
+      "l": 9,
+      "m": 10,
+      "n": 11,
+      "o": 12,
+      "p": 13,
+      "r": 14,
+      "s": 15,
+      "t": 16,
+      "u": 17,
+      "y": 18,
+      "##l": 19,
+      "##o": 20,
+      "##t": 21,
+      "##s": 22,
+      "##u": 23,
+      "##m": 24,
       "##a": 25,
+      "##r": 26,
+      "##y": 27,
+      "##e": 28,
+      "##n": 29,
+      "##i": 30,
+      "ge": 31,
+      "pl": 32,
+      "ti": 33,
+      "##le": 34,
+      "##ot": 35,
+      "##tle": 36,
+      "##su": 37,
+      "##mm": 38,
+      "##ar": 39,
+      "##re": 40,
+      "##nre": 41,
+      "genre": 42,
+      "plot": 43,
+      "title": 44,
+      "##summ": 45,
+      "##ary": 46,
+      "plotsumm": 47,
+      "plotsummary": 48
     }
   }
 }

vocab.txt CHANGED Viewed

@@ -3,15 +3,10 @@
 [CLS]
 [SEP]
 [MASK]
-/
 a
-c
-d
 e
 g
-h
 i
-k
 l
 m
 n
@@ -21,72 +16,34 @@ r
 s
 t
 u
-w
 y
-##a
-##s
 ##t
-##g
-##e
 ##r
-##h
 ##n
 ##i
-##c
-##y
-##o
-##l
-##u
-##m
-##k
 pl
-##as
-##ar
-##re
-##it
-##ot
 ##le
-plot
-cas
-di
-et
-ge
-or
-pa
-re
-tit
-wi
-ye
 ##su
-##to
-##ge
-##gi
-##hn
-##nre
-##ic
-##igi
-##cto
 ##mm
-##ki
-##ase
-##ary
-##recto
-##ity
-##lease
-plotsu
-cast
-directo
-ethn
 genre
-origi
-page
-release
 title
-wiki
-year
-##icity
-##mmary
 plotsummary
-director
-ethnicity
-origin

 [CLS]
 [SEP]
 [MASK]
 a
 e
 g
 i
 l
 m
 n
 s
 t
 u
 y
+##l
+##o
 ##t
+##s
+##u
+##m
+##a
 ##r
+##y
+##e
 ##n
 ##i
+ge
 pl
+ti
 ##le
+##ot
+##tle
 ##su
 ##mm
+##ar
+##re
+##nre
 genre
+plot
 title
+##summ
+##ary
+plotsumm
 plotsummary