Lakoc
/

common_voice_czech_unigram200

Model card Files Files and versions Community

Lakoc commited on Oct 14, 2023

Commit

b1b453c

1 Parent(s): 7fd4de6

Upload tokenizer

Browse files

Files changed (3) hide show

special_tokens_map.json +7 -0
tokenizer.json +971 -0
tokenizer_config.json +9 -0

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,7 @@

+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer.json ADDED Viewed

	@@ -0,0 +1,971 @@

+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "<mask>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Replace",
+        "pattern": {
+          "String": "``"
+        },
+        "content": "\""
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "String": "''"
+        },
+        "content": "\""
+      },
+      {
+        "type": "Lowercase"
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "add_prefix_space": true
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "</s>",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "</s>",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 1
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "</s>",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "</s>": {
+        "id": "</s>",
+        "ids": [
+          1
+        ],
+        "tokens": [
+          "</s>"
+        ]
+      },
+      "<s>": {
+        "id": "<s>",
+        "ids": [
+          0
+        ],
+        "tokens": [
+          "<s>"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "add_prefix_space": true
+  },
+  "model": {
+    "type": "Unigram",
+    "unk_id": 2,
+    "vocab": [
+      [
+        "<s>",
+        0.0
+      ],
+      [
+        "</s>",
+        0.0
+      ],
+      [
+        "<unk>",
+        0.0
+      ],
+      [
+        "<pad>",
+        0.0
+      ],
+      [
+        "<mask>",
+        0.0
+      ],
+      [
+        "▁",
+        -2.933881842641904
+      ],
+      [
+        ".",
+        -3.5477780942834354
+      ],
+      [
+        "t",
+        -3.589490346893994
+      ],
+      [
+        "a",
+        -3.644247019210544
+      ],
+      [
+        "m",
+        -3.6586101480214985
+      ],
+      [
+        "o",
+        -3.70095367793402
+      ],
+      [
+        "l",
+        -3.7651406443688806
+      ],
+      [
+        "d",
+        -3.8417028391607904
+      ],
+      [
+        "s",
+        -3.885144751638135
+      ],
+      [
+        "r",
+        -3.901295045785892
+      ],
+      [
+        "v",
+        -3.935602964994691
+      ],
+      [
+        "e",
+        -3.9940703496517287
+      ],
+      [
+        "á",
+        -4.033966857289286
+      ],
+      [
+        "u",
+        -4.05884556484637
+      ],
+      [
+        "i",
+        -4.1166233920812925
+      ],
+      [
+        "p",
+        -4.176589139748492
+      ],
+      [
+        "ní",
+        -4.279857833700591
+      ],
+      [
+        "k",
+        -4.289690282403042
+      ],
+      [
+        "í",
+        -4.293029126936448
+      ],
+      [
+        "ch",
+        -4.324255581534921
+      ],
+      [
+        "▁s",
+        -4.325405861628145
+      ],
+      [
+        "n",
+        -4.37776141455995
+      ],
+      [
+        "▁v",
+        -4.380004094093257
+      ],
+      [
+        "st",
+        -4.453846829326743
+      ],
+      [
+        "y",
+        -4.488741049922906
+      ],
+      [
+        "b",
+        -4.501608535676114
+      ],
+      [
+        "z",
+        -4.555001955831841
+      ],
+      [
+        "ě",
+        -4.595063469625593
+      ],
+      [
+        "▁a",
+        -4.6643161816163765
+      ],
+      [
+        "▁po",
+        -4.716152885007917
+      ],
+      [
+        "le",
+        -4.729191738166342
+      ],
+      [
+        "▁je",
+        -4.771673455593422
+      ],
+      [
+        "é",
+        -4.824781856505561
+      ],
+      [
+        "▁z",
+        -4.8481232618122245
+      ],
+      [
+        "▁na",
+        -4.9181854570660395
+      ],
+      [
+        "ra",
+        -4.9291603874766015
+      ],
+      [
+        "la",
+        -4.9390965232116475
+      ],
+      [
+        "▁se",
+        -4.940769771372215
+      ],
+      [
+        "ho",
+        -4.948526161364832
+      ],
+      [
+        "ů",
+        -4.960219219866243
+      ],
+      [
+        "č",
+        -4.97484752703166
+      ],
+      [
+        "h",
+        -4.975810635483684
+      ],
+      [
+        "ž",
+        -4.994197816023373
+      ],
+      [
+        "ně",
+        -5.0492874780193056
+      ],
+      [
+        "ce",
+        -5.055632153128059
+      ],
+      [
+        "▁t",
+        -5.067221321042702
+      ],
+      [
+        "c",
+        -5.103308475546731
+      ],
+      [
+        "j",
+        -5.113139030749984
+      ],
+      [
+        "▁k",
+        -5.121312549520173
+      ],
+      [
+        "li",
+        -5.158555052585243
+      ],
+      [
+        "to",
+        -5.187829843778575
+      ],
+      [
+        "te",
+        -5.199580018751819
+      ],
+      [
+        "ni",
+        -5.237632267398226
+      ],
+      [
+        "va",
+        -5.271035361443101
+      ],
+      [
+        "▁o",
+        -5.286563564192571
+      ],
+      [
+        "ou",
+        -5.299940532930255
+      ],
+      [
+        "▁do",
+        -5.315008911633093
+      ],
+      [
+        "▁b",
+        -5.321459316608349
+      ],
+      [
+        "lo",
+        -5.337670312261877
+      ],
+      [
+        "na",
+        -5.3707587757007245
+      ],
+      [
+        "▁pro",
+        -5.400835750750204
+      ],
+      [
+        "ro",
+        -5.407363420434973
+      ],
+      [
+        "š",
+        -5.410775069982266
+      ],
+      [
+        "▁d",
+        -5.416578704818147
+      ],
+      [
+        "ý",
+        -5.425608165733997
+      ],
+      [
+        "en",
+        -5.431546361498073
+      ],
+      [
+        "ře",
+        -5.446446776902486
+      ],
+      [
+        "g",
+        -5.472971854258994
+      ],
+      [
+        "▁ne",
+        -5.48942936029952
+      ],
+      [
+        "▁za",
+        -5.5360003618990365
+      ],
+      [
+        "em",
+        -5.540943553270047
+      ],
+      [
+        "ko",
+        -5.551449196510767
+      ],
+      [
+        "né",
+        -5.622361112385544
+      ],
+      [
+        "▁h",
+        -5.6239437580030085
+      ],
+      [
+        "de",
+        -5.643325386718073
+      ],
+      [
+        "ti",
+        -5.6447253918913765
+      ],
+      [
+        "▁byl",
+        -5.648337857155493
+      ],
+      [
+        "ka",
+        -5.654853359802825
+      ],
+      [
+        "ku",
+        -5.664006533642916
+      ],
+      [
+        "▁ve",
+        -5.725900255027681
+      ],
+      [
+        "ci",
+        -5.7649409188479614
+      ],
+      [
+        "ná",
+        -5.769164693906317
+      ],
+      [
+        "mi",
+        -5.774644808059395
+      ],
+      [
+        "▁ob",
+        -5.778962817957801
+      ],
+      [
+        "▁vy",
+        -5.789650303559062
+      ],
+      [
+        "jí",
+        -5.794495124613842
+      ],
+      [
+        "vě",
+        -5.798185586095333
+      ],
+      [
+        "me",
+        -5.80913158514987
+      ],
+      [
+        "ze",
+        -5.814166428372028
+      ],
+      [
+        "▁p",
+        -5.824208440583925
+      ],
+      [
+        "▁u",
+        -5.878757555875056
+      ],
+      [
+        ",",
+        -5.903152170013643
+      ],
+      [
+        "no",
+        -5.9057725403152395
+      ],
+      [
+        "ji",
+        -5.920428277572507
+      ],
+      [
+        "▁to",
+        -5.926340496642947
+      ],
+      [
+        "tu",
+        -5.927828549586592
+      ],
+      [
+        "ky",
+        -5.9371858287916615
+      ],
+      [
+        "or",
+        -5.946365129713133
+      ],
+      [
+        "ný",
+        -5.99112481218983
+      ],
+      [
+        "že",
+        -5.993760496634453
+      ],
+      [
+        "ské",
+        -6.004036828021916
+      ],
+      [
+        "ř",
+        -6.043049654958058
+      ],
+      [
+        "in",
+        -6.044865678749619
+      ],
+      [
+        "y.",
+        -6.064267907044521
+      ],
+      [
+        "ší",
+        -6.066469419035859
+      ],
+      [
+        "▁f",
+        -6.086007502509733
+      ],
+      [
+        "vá",
+        -6.0929776823686215
+      ],
+      [
+        "cí",
+        -6.096087103918272
+      ],
+      [
+        "di",
+        -6.1258326693684
+      ],
+      [
+        "ri",
+        -6.130335674191909
+      ],
+      [
+        "je",
+        -6.137264395372371
+      ],
+      [
+        "ne",
+        -6.1531957187534765
+      ],
+      [
+        "▁vý",
+        -6.156522965057197
+      ],
+      [
+        "▁ná",
+        -6.183729574743225
+      ],
+      [
+        "ří",
+        -6.188204941940626
+      ],
+      [
+        "▁pa",
+        -6.19758569914038
+      ],
+      [
+        "ny",
+        -6.204870892400333
+      ],
+      [
+        "ži",
+        -6.211094353405346
+      ],
+      [
+        "oval",
+        -6.239595170380871
+      ],
+      [
+        "f",
+        -6.286645633774842
+      ],
+      [
+        "še",
+        -6.287121444594022
+      ],
+      [
+        "▁ú",
+        -6.295293312777933
+      ],
+      [
+        "by",
+        -6.301525122964281
+      ],
+      [
+        "ru",
+        -6.30809270263456
+      ],
+      [
+        "▁od",
+        -6.319961842625641
+      ],
+      [
+        "▁zá",
+        -6.328547643992921
+      ],
+      [
+        "vi",
+        -6.346802728367797
+      ],
+      [
+        "▁ma",
+        -6.431912156242294
+      ],
+      [
+        "lu",
+        -6.432754126816632
+      ],
+      [
+        "uje",
+        -6.443190781270953
+      ],
+      [
+        "ové",
+        -6.467014285714815
+      ],
+      [
+        "nost",
+        -6.549975301817593
+      ],
+      [
+        "▁pří",
+        -6.5569172549306725
+      ],
+      [
+        "nu",
+        -6.570250699560692
+      ],
+      [
+        "▁roz",
+        -6.589732746829957
+      ],
+      [
+        "▁mo",
+        -6.5921275544955575
+      ],
+      [
+        "▁při",
+        -6.592289291694437
+      ],
+      [
+        "▁sta",
+        -6.604274711569781
+      ],
+      [
+        "▁jako",
+        -6.640631120618143
+      ],
+      [
+        "▁re",
+        -6.641361350548964
+      ],
+      [
+        "▁pře",
+        -6.645933416903996
+      ],
+      [
+        "ři",
+        -6.665728435048961
+      ],
+      [
+        "▁před",
+        -6.688395456755879
+      ],
+      [
+        "ován",
+        -6.6986357693587975
+      ],
+      [
+        "ský",
+        -6.7345275093187515
+      ],
+      [
+        "či",
+        -6.7556332887967985
+      ],
+      [
+        "stav",
+        -6.7565629986059985
+      ],
+      [
+        "▁také",
+        -6.784564208271911
+      ],
+      [
+        "ické",
+        -6.822473413154058
+      ],
+      [
+        "▁jsou",
+        -6.839600800409185
+      ],
+      [
+        "▁kon",
+        -6.856932521302764
+      ],
+      [
+        "\"",
+        -6.940649196032627
+      ],
+      [
+        "děl",
+        -7.012240797906627
+      ],
+      [
+        "▁tak",
+        -7.056427922232195
+      ],
+      [
+        "▁ja",
+        -7.059445413822217
+      ],
+      [
+        "cký",
+        -7.083248888291322
+      ],
+      [
+        "x",
+        -7.094945147217752
+      ],
+      [
+        "ň",
+        -7.16444676938674
+      ],
+      [
+        "▁však",
+        -7.188759209502638
+      ],
+      [
+        "ště",
+        -7.197130013703209
+      ],
+      [
+        "▁měst",
+        -7.2631846765783745
+      ],
+      [
+        "▁druh",
+        -7.282570677527317
+      ],
+      [
+        "▁měl",
+        -7.384552941270997
+      ],
+      [
+        "?",
+        -7.407692759728381
+      ],
+      [
+        "chází",
+        -7.435960685282815
+      ],
+      [
+        "▁mezi",
+        -7.507108909210451
+      ],
+      [
+        "w",
+        -7.535083601752331
+      ],
+      [
+        "▁několik",
+        -7.590293335462103
+      ],
+      [
+        "▁musí",
+        -7.677520603750372
+      ],
+      [
+        "▁později",
+        -7.718353421312045
+      ],
+      [
+        "▁evropsk",
+        -7.830696406547949
+      ],
+      [
+        "▁společn",
+        -7.874088576602077
+      ],
+      [
+        "ť",
+        -7.901054161965922
+      ],
+      [
+        "▁současn",
+        -7.9524684253043665
+      ],
+      [
+        "ď",
+        -8.011743748876743
+      ],
+      [
+        "ó",
+        -8.058627228696277
+      ],
+      [
+        "!",
+        -8.390840756999921
+      ],
+      [
+        "▁působil",
+        -8.398303443996245
+      ],
+      [
+        "▁univerzit",
+        -8.428723893618406
+      ],
+      [
+        "ú",
+        -8.69197148730532
+      ],
+      [
+        "“",
+        -8.798163432450156
+      ],
+      [
+        "-",
+        -9.81112772875104
+      ],
+      [
+        "–",
+        -10.093664746434053
+      ],
+      [
+        ";",
+        -10.766412245976962
+      ],
+      [
+        "ö",
+        -10.940654670219764
+      ],
+      [
+        "ü",
+        -11.276765781343222
+      ],
+      [
+        "/",
+        -12.36962292424032
+      ],
+      [
+        "è",
+        -12.369622924240325
+      ],
+      [
+        "ä",
+        -12.86962292424032
+      ],
+      [
+        ":",
+        -12.869622924240325
+      ],
+      [
+        "q",
+        -13.869522924240323
+      ],
+      [
+        "„",
+        -13.869622924240325
+      ],
+      [
+        "—",
+        -13.869622924240325
+      ],
+      [
+        "…",
+        -13.869622924240325
+      ],
+      [
+        "ï",
+        -13.869622924240325
+      ]
+    ]
+  }
+}

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}