Upload all models and assets for cs (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-slavic_west
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,14 +33,14 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.130
+    value: 4.591
   - name: best_isotropy
     type: isotropy
-    value: 0.7709
+    value: 0.7988
   - name: vocabulary_size
     type: vocab
-    value: 1000000
-generated: 2025-12-29
+    value: 0
+generated: 2026-01-08
 ---
 
 # Czech - Wikilangs Models
@@ -44,12 +54,13 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 ### Models & Assets
 
 - Tokenizers (8k, 16k, 32k, 64k)
-- N-gram models (2, 3, 4-gram)
-- Markov chains (context of 1, 2, 3 and 4)
+- N-gram models (2, 3, 4, 5-gram)
+- Markov chains (context of 1, 2, 3, 4 and 5)
 - Subword N-gram and Markov chains
-- Embeddings in various sizes and dimensions
+- Embeddings in various sizes and dimensions (aligned and unaligned)
 - Language Vocabulary
 - Language Statistics
+
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
 ### Analysis and Evaluation
@@ -59,7 +70,8 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Summary & Recommendations](#6-summary--recommendations)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
+- [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
 
@@ -68,57 +80,57 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 ![Tokenizer Compression](visualizations/tokenizer_compression.png)
 
+![Tokenizer Fertility](visualizations/tokenizer_fertility.png)
+
+![Tokenizer OOV](visualizations/tokenizer_oov.png)
+
+![Total Tokens](visualizations/tokenizer_total_tokens.png)
+
 ### Results
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.197x | 3.17 | 0.1005% | 3,306,013 |
-| **16k** | 3.544x | 3.51 | 0.1114% | 2,981,907 |
-| **32k** | 3.862x | 3.82 | 0.1214% | 2,736,410 |
-| **64k** | 4.130x 🏆 | 4.09 | 0.1298% | 2,558,883 |
+| **8k** | 3.417x | 3.42 | 0.0769% | 2,893,388 |
+| **16k** | 3.845x | 3.85 | 0.0865% | 2,570,989 |
+| **32k** | 4.245x | 4.25 | 0.0955% | 2,328,840 |
+| **64k** | 4.591x 🏆 | 4.59 | 0.1033% | 2,153,192 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `TJ Sigma MŽ Olomouc je historický název těchto klubů:
-SK Sigma Olomouc – fotbalo...`
+**Sample 1:** `<tr> Související články Seznam kulturních památek v okrese Znojmo Externí odkazy...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁tj ▁si g ma ▁m ž ▁olomouc ▁je ▁historický ▁název ... (+31 more)` | 41 |
-| 16k | `▁tj ▁sig ma ▁m ž ▁olomouc ▁je ▁historický ▁název ▁těchto ... (+25 more)` | 35 |
-| 32k | `▁tj ▁sigma ▁m ž ▁olomouc ▁je ▁historický ▁název ▁těchto ▁klubů ... (+21 more)` | 31 |
-| 64k | `▁tj ▁sigma ▁m ž ▁olomouc ▁je ▁historický ▁název ▁těchto ▁klubů ... (+20 more)` | 30 |
+| 8k | `▁< tr > ▁související ▁články ▁seznam ▁kultur ních ▁pam átek ... (+17 more)` | 27 |
+| 16k | `▁< tr > ▁související ▁články ▁seznam ▁kulturních ▁památek ▁v ▁okrese ... (+13 more)` | 23 |
+| 32k | `▁< tr > ▁související ▁články ▁seznam ▁kulturních ▁památek ▁v ▁okrese ... (+11 more)` | 21 |
+| 64k | `▁< tr > ▁související ▁články ▁seznam ▁kulturních ▁památek ▁v ▁okrese ... (+11 more)` | 21 |
 
-**Sample 2:** `PŘESMĚRUJ Chrpa čekánek
-
-Kategorie:Přesměrování z vědeckého jména`
+**Sample 2:** `Mirovice <tr> Sochovice <tr> Související články Seznam kulturních památek v okre...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁přes měru j ▁ch r pa ▁če ká nek ▁kategorie ... (+9 more)` | 19 |
-| 16k | `▁přes měru j ▁ch r pa ▁če ká nek ▁kategorie ... (+8 more)` | 18 |
-| 32k | `▁přes měru j ▁chr pa ▁čeká nek ▁kategorie : přes ... (+5 more)` | 15 |
-| 64k | `▁přes měru j ▁chr pa ▁čeká nek ▁kategorie : přes ... (+5 more)` | 15 |
-
-**Sample 3:** `Pello jsou dvě obce se stejným názvem:
+| 8k | `▁mi rovice ▁< tr > ▁so ch ovice ▁< tr ... (+17 more)` | 27 |
+| 16k | `▁mi rovice ▁< tr > ▁so chovice ▁< tr > ... (+14 more)` | 24 |
+| 32k | `▁mi rovice ▁< tr > ▁so chovice ▁< tr > ... (+14 more)` | 24 |
+| 64k | `▁mi rovice ▁< tr > ▁so chovice ▁< tr > ... (+14 more)` | 24 |
 
- Pello (Finsko) – obec ve Finsku
- Pello ...`
+**Sample 3:** `Sabra může být: sabra – hebrejské slovo Sabra (tank) Sabra – sídlo v Libanonu, d...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁pel lo ▁jsou ▁dvě ▁obce ▁se ▁stej ným ▁názvem : ... (+23 more)` | 33 |
-| 16k | `▁pel lo ▁jsou ▁dvě ▁obce ▁se ▁stejným ▁názvem : ▁pel ... (+19 more)` | 29 |
-| 32k | `▁pel lo ▁jsou ▁dvě ▁obce ▁se ▁stejným ▁názvem : ▁pel ... (+19 more)` | 29 |
-| 64k | `▁pel lo ▁jsou ▁dvě ▁obce ▁se ▁stejným ▁názvem : ▁pel ... (+17 more)` | 27 |
+| 8k | `▁sa bra ▁může ▁být : ▁sa bra ▁– ▁hebrej ské ... (+22 more)` | 32 |
+| 16k | `▁sa bra ▁může ▁být : ▁sa bra ▁– ▁hebrej ské ... (+21 more)` | 31 |
+| 32k | `▁sa bra ▁může ▁být : ▁sa bra ▁– ▁hebrejské ▁slovo ... (+17 more)` | 27 |
+| 64k | `▁sa bra ▁může ▁být : ▁sa bra ▁– ▁hebrejské ▁slovo ... (+15 more)` | 25 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.130x compression
-- **Lowest UNK Rate:** 8k with 0.1005% unknown tokens
+- **Best Compression:** 64k achieves 4.591x compression
+- **Lowest UNK Rate:** 8k with 0.0769% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -127,57 +139,111 @@ Kategorie:Přesměrování z vědeckého jména`
 
 ![N-gram Perplexity](visualizations/ngram_perplexity.png)
 
+![N-gram Unique](visualizations/ngram_unique.png)
+
 ![N-gram Coverage](visualizations/ngram_coverage.png)
 
 ### Results
 
-| N-gram | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
-|--------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | 295,975 🏆 | 18.18 | 5,487,215 | 10.4% | 21.2% |
-| **2-gram** | 540 🏆 | 9.08 | 35,551 | 50.3% | 96.3% |
-| **3-gram** | 1,394,157 | 20.41 | 12,698,712 | 5.9% | 11.9% |
-| **3-gram** | 5,799 | 12.50 | 338,538 | 15.9% | 51.3% |
-| **4-gram** | 3,715,211 | 21.83 | 23,000,459 | 5.1% | 8.9% |
-| **4-gram** | 39,713 | 15.28 | 2,482,399 | 7.1% | 24.3% |
+| N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
+|--------|---------|------------|---------|----------------|------------------|-------------------|
+| **2-gram** | Word | 644,039 | 19.30 | 4,952,358 | 4.8% | 11.9% |
+| **2-gram** | Subword | 449 🏆 | 8.81 | 30,223 | 53.9% | 98.0% |
+| **3-gram** | Word | 2,339,059 | 21.16 | 8,925,525 | 2.6% | 6.4% |
+| **3-gram** | Subword | 4,755 | 12.22 | 255,109 | 16.7% | 54.3% |
+| **4-gram** | Word | 5,475,376 | 22.38 | 14,408,434 | 1.3% | 3.9% |
+| **4-gram** | Subword | 32,796 | 15.00 | 1,646,964 | 6.8% | 24.8% |
+| **5-gram** | Word | 4,645,198 | 22.15 | 10,221,820 | 1.0% | 3.6% |
+| **5-gram** | Subword | 160,592 | 17.29 | 6,437,902 | 3.7% | 13.8% |
 
 ### Top 5 N-grams by Size
 
-**2-grams:**
+**2-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `v roce` | 1,319,715 |
+| 2 | `externí odkazy` | 445,741 |
+| 3 | `odkazy reference` | 238,320 |
+| 4 | `reference externí` | 226,335 |
+| 5 | `v letech` | 212,278 |
+
+**3-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `reference externí odkazy` | 226,294 |
+| 2 | `odkazy reference externí` | 124,877 |
+| 3 | `v roce v` | 123,855 |
+| 4 | `v roce se` | 91,582 |
+| 5 | `v roce byl` | 64,824 |
+
+**4-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `kategorie :` | 2,810,345 |
-| 2 | `| |` | 1,600,369 |
-| 3 | `v roce` | 1,324,349 |
-| 4 | `. v` | 1,082,474 |
-| 5 | `) ,` | 872,870 |
+| 1 | `odkazy reference externí odkazy` | 124,850 |
+| 2 | `odkazy reference související články` | 42,127 |
+| 3 | `v roce v roce` | 34,075 |
+| 4 | `reference externí odkazy v` | 29,798 |
+| 5 | `externí odkazy oficiální stránky` | 20,103 |
 
-**3-grams:**
+**5-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `| align =` | 701,748 |
-| 2 | `| | align` | 686,964 |
-| 3 | `align = right` | 683,296 |
-| 4 | `= right |` | 683,269 |
-| 5 | `kategorie : narození` | 390,185 |
+| 1 | `odkazy reference externí odkazy v` | 16,236 |
+| 2 | `odkazy reference literatura externí odkazy` | 12,685 |
+| 3 | `reference externí odkazy oficiální stránky` | 11,834 |
+| 4 | `historie první písemná zmínka o` | 11,754 |
+| 5 | `reference externí odkazy v okrese` | 11,425 |
 
-**4-grams:**
+**2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `| | align =` | 686,964 |
-| 2 | `| align = right` | 683,285 |
-| 3 | `align = right |` | 683,268 |
-| 4 | `kategorie : narození v` | 246,211 |
-| 5 | `externí odkazy kategorie :` | 191,110 |
+| 1 | `a _` | 24,781,439 |
+| 2 | `_ p` | 22,589,509 |
+| 3 | `e _` | 22,268,109 |
+| 4 | `_ s` | 22,095,879 |
+| 5 | `_ v` | 19,926,387 |
+
+**3-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `n í _` | 7,673,842 |
+| 2 | `_ p o` | 7,582,650 |
+| 3 | `_ v _` | 7,272,309 |
+| 4 | `n a _` | 6,690,107 |
+| 5 | `_ a _` | 6,501,417 |
+
+**4-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ n a _` | 3,511,209 |
+| 2 | `_ s e _` | 3,364,693 |
+| 3 | `_ p r o` | 3,186,267 |
+| 4 | `_ b y l` | 2,542,448 |
+| 5 | `ý c h _` | 2,252,305 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `_ k t e r` | 1,412,346 |
+| 2 | `_ r o c e` | 1,383,042 |
+| 3 | `_ v _ r o` | 1,382,611 |
+| 4 | `r o c e _` | 1,354,432 |
+| 5 | `v _ r o c` | 1,321,210 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram with 540
+- **Best Perplexity:** 2-gram (subword) with 449
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~24% of corpus
+- **Coverage:** Top-1000 patterns cover ~14% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -185,55 +251,86 @@ Kategorie:Přesměrování z vědeckého jména`
 
 ![Markov Entropy](visualizations/markov_entropy.png)
 
+![Markov Contexts](visualizations/markov_contexts.png)
+
 ![Markov Branching](visualizations/markov_branching.png)
 
 ### Results
 
-| Context | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
-|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | 0.7467 | 1.678 | 10.64 | 5,180,358 | 25.3% |
-| **1** | 1.5920 | 3.015 | 12.43 | 10,644 | 0.0% |
-| **2** | 0.4222 | 1.340 | 2.71 | 55,087,986 | 57.8% |
-| **2** | 0.7625 | 1.696 | 5.46 | 132,235 | 23.8% |
-| **3** | 0.1844 | 1.136 | 1.46 | 149,279,731 | 81.6% |
-| **3** | 0.9138 | 1.884 | 5.69 | 722,190 | 8.6% |
-| **4** | 0.0848 🏆 | 1.061 | 1.17 | 217,279,300 | 91.5% |
-| **4** | 0.8187 🏆 | 1.764 | 4.11 | 4,107,972 | 18.1% |
+| Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
+|---------|---------|-------------|------------|------------------|-----------------|----------------|
+| **1** | Word | 1.0698 | 2.099 | 16.20 | 3,817,910 | 0.0% |
+| **1** | Subword | 1.2123 | 2.317 | 8.62 | 14,369 | 0.0% |
+| **2** | Word | 0.3832 | 1.304 | 2.35 | 61,779,051 | 61.7% |
+| **2** | Subword | 0.6716 | 1.593 | 4.71 | 123,767 | 32.8% |
+| **3** | Word | 0.1433 | 1.104 | 1.31 | 144,949,424 | 85.7% |
+| **3** | Subword | 0.7660 | 1.701 | 4.77 | 583,275 | 23.4% |
+| **4** | Word | 0.0564 🏆 | 1.040 | 1.10 | 189,649,924 | 94.4% |
+| **4** | Subword | 0.7409 | 1.671 | 4.00 | 2,782,368 | 25.9% |
 
-### Generated Text Samples
+### Generated Text Samples (Word-based)
 
-Below are text samples generated from each Markov chain model:
+Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `. a jeho šířením civilizace původní zahrady . předpokládá se tak , který pracoval v nemocnici`
-2. `, ale ona . curtiss c v modelové animace ) . sk . y mhz )`
-3. `v maďarské metropole vojvodiny . pupeny se používalo pro osvětovou činností policie ontario : oceněn...`
+1. `v podobě vystavěn byl opětovně pohřbena ve dveřích některých případech může vytvořit jediné dopravní...`
+2. `a příslušník staré město zbiroh živa je americký teoretický kvantový stav potrvá v létě odešel na`
+3. `na fakt že neměl v červenci i z původních 113 120 metrů vysokém tlaku na východě`
 
 **Context Size 2:**
 
-1. `kategorie : muži kategorie : francouzští fotbalisté kategorie : úmrtí v roce 1884 kategorie : čeští ...`
-2. `| | align = right | 2 | 13 , resurrection ( 1993 - 2000 elke delugan`
-3. `v roce 2016 uskutečněno referendum ve spojeném království . dne 22 . března 1993li pcheng ili pcheng`
+1. `v roce lidé 6 prosince praha byl michal kraus čssd čssd 48 rychnov nad kněžnou kaple stojí`
+2. `externí odkazy jihovýchodní evropy jihozápadní asie kavkazu číny sibiře východní asie hustě chlupatá...`
+3. `odkazy reference externí odkazy sdružení na praze 4 rozhovor vznikl v roce kde bojoval proti ostrogó...`
 
 **Context Size 3:**
 
-1. `| align = right | 7 , 74 | | align = right | 4 , 366 |`
-2. `| | align = right | 3 , 506 | | mba | | 8 | | 14`
-3. `align = right | 9 , 09 | | align = right | 0 , 072 | |`
+1. `reference externí odkazy v ternopilské oblasti na řece strypa v historickém regionu horní lužice mim...`
+2. `odkazy reference externí odkazy speleologická společnost vševěd romantismu hudební skladatelé klavír...`
+3. `v roce v angličtině se pro celou skupinu alfred crompton catherine musinsky jose bonaparte bhart anj...`
 
 **Context Size 4:**
 
-1. `| | align = right | 2 , 615 | | align = right | 2 , 697 |`
-2. `| align = right | 2 , 610 | | mba | | 6 . května 2000 | |`
-3. `align = right | 0 , 199 | | mba | | 29 . července 2000 | | socorro`
+1. `odkazy reference externí odkazy strategie série`
+2. `odkazy reference související články fotografie v norsku externí odkazy na seznamu světového dědictví...`
+3. `v roce v roce v praze pilotní školu druhá světová válka po roce vojenské služby v polské armádě prot...`
+
+
+### Generated Text Samples (Subword-based)
+
+Below are text samples generated from each subword-based Markov chain model:
+
+**Context Size 1:**
+
+1. `_hraloponodovo._`
+2. `os_zu_va_vu_dulo`
+3. `ekodici_micl_v_s`
+
+**Context Size 2:**
+
+1. `a_stříjna_se_rozh`
+2. `_příčku_uraven_pe`
+3. `e_na_vítlická_hov`
+
+**Context Size 3:**
+
+1. `ní_nejčastoru_o_sp`
+2. `_polik_v_com_trans`
+3. `_v_195_zúčasná_náz`
+
+**Context Size 4:**
+
+1. `_na_v_nicméně_chlaz`
+2. `_se_proje_asistenci`
+3. `_pro_pozdně,_lze_sa`
 
 
 ### Key Findings
 
-- **Best Predictability:** Context-4 with 91.5% predictability
+- **Best Predictability:** Context-4 (word) with 94.4% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (4,107,972 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (2,782,368 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -249,64 +346,64 @@ Below are text samples generated from each Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 1,000,000 |
-| Total Tokens | 260,225,789 |
-| Mean Frequency | 260.23 |
-| Median Frequency | 14 |
-| Frequency Std Dev | 13208.48 |
+| Vocabulary Size | 1,830,714 |
+| Total Tokens | 237,612,209 |
+| Mean Frequency | 129.79 |
+| Median Frequency | 5 |
+| Frequency Std Dev | 9362.17 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | v | 7,430,819 |
-| 2 | a | 6,664,349 |
-| 3 | na | 3,549,580 |
-| 4 | se | 3,405,018 |
-| 5 | kategorie | 2,831,795 |
-| 6 | je | 2,114,934 |
-| 7 | s | 1,854,182 |
-| 8 | z | 1,755,717 |
-| 9 | do | 1,449,955 |
-| 10 | roce | 1,387,784 |
+| 1 | v | 7,396,110 |
+| 2 | a | 6,633,731 |
+| 3 | na | 3,536,561 |
+| 4 | se | 3,396,490 |
+| 5 | je | 2,110,163 |
+| 6 | s | 1,781,636 |
+| 7 | z | 1,747,028 |
+| 8 | do | 1,440,810 |
+| 9 | roce | 1,383,007 |
+| 10 | ve | 1,284,897 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | čechovovu | 4 |
-| 2 | vychloubavý | 4 |
-| 3 | euroatlantický | 4 |
-| 4 | postkomunistickým | 4 |
-| 5 | nadnesli | 4 |
-| 6 | sulyoka | 4 |
-| 7 | srebrenickém | 4 |
-| 8 | odepřený | 4 |
-| 9 | kosovskoalbánských | 4 |
-| 10 | ipap | 4 |
+| 1 | mihty | 2 |
+| 2 | socionaut | 2 |
+| 3 | mafjar | 2 |
+| 4 | vlta | 2 |
+| 5 | havlátková | 2 |
+| 6 | makbúsu | 2 |
+| 7 | propfanů | 2 |
+| 8 | propfanu | 2 |
+| 9 | ochmeloff | 2 |
+| 10 | luncași | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 0.9367 |
-| R² (Goodness of Fit) | 0.997046 |
+| Zipf Coefficient | 0.9138 |
+| R² (Goodness of Fit) | 0.997539 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 27.3% |
-| Top 1,000 | 47.0% |
-| Top 5,000 | 64.5% |
-| Top 10,000 | 72.1% |
+| Top 100 | 27.1% |
+| Top 1,000 | 45.7% |
+| Top 5,000 | 63.0% |
+| Top 10,000 | 70.6% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9970 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 27.3% of corpus
-- **Long Tail:** 990,000 words needed for remaining 27.9% coverage
+- **Zipf Compliance:** R²=0.9975 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 27.1% of corpus
+- **Long Tail:** 1,820,714 words needed for remaining 29.4% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -319,24 +416,124 @@ Below are text samples generated from each Markov chain model:
 
 ![t-SNE Sentences](visualizations/tsne_sentences.png)
 
-### Model Comparison
 
-| Model | Vocab Size | Dimension | Avg Norm | Std Norm | Isotropy |
-|-------|------------|-----------|----------|----------|----------|
-| **mono_32d** | 1,659,956 | 32 | 3.229 | 0.994 | 0.7709 🏆 |
-| **mono_64d** | 1,659,956 | 64 | 3.639 | 0.952 | 0.7533 |
-| **mono_128d** | 1,659,956 | 128 | 4.126 | 0.969 | 0.7045 |
-| **embeddings_enhanced** | 0 | 0 | 0.000 | 0.000 | 0.0000 |
+### 5.1 Cross-Lingual Alignment
+
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
+
+
+### 5.2 Model Comparison
+
+| Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
+|-------|-----------|----------|------------------|---------------|----------------|
+| **mono_32d** | 32 | 0.7988 | 0.3622 | N/A | N/A |
+| **mono_64d** | 64 | 0.7835 | 0.2893 | N/A | N/A |
+| **mono_128d** | 128 | 0.7363 | 0.2299 | N/A | N/A |
+| **aligned_32d** | 32 | 0.7988 🏆 | 0.3646 | 0.3500 | 0.7360 |
+| **aligned_64d** | 64 | 0.7835 | 0.2898 | 0.5900 | 0.8980 |
+| **aligned_128d** | 128 | 0.7363 | 0.2271 | 0.7320 | 0.9520 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.7709 (more uniform distribution)
-- **Dimension Trade-off:** Higher dimensions capture more semantics but reduce isotropy
-- **Vocabulary Coverage:** All models cover 1,659,956 words
-- **Recommendation:** 100d for balanced semantic capture and efficiency
+- **Best Isotropy:** aligned_32d with 0.7988 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2938. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 73.2% R@1 in cross-lingual retrieval.
+- **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
-## 6. Summary & Recommendations
+## 6.  Morphological Analysis (Experimental)
+
+This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
+
+### 6.1 Productivity & Complexity
+
+| Metric | Value | Interpretation | Recommendation |
+|--------|-------|----------------|----------------|
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **-0.741** | Low formulaic content | - |
+
+### 6.2 Affix Inventory (Productive Units)
+
+These are the most productive prefixes and suffixes identified by sampling the vocabulary for global substitutability patterns. A unit is considered an affix if stripping it leaves a valid stem that appears in other contexts.
+
+#### Productive Prefixes
+| Prefix | Examples |
+|--------|----------|
+| `-ne` | nezamítl, neomorf, nenapájeným |
+| `-po` | poštulky, ponoršťování, powerkiting |
+
+#### Productive Suffixes
+| Suffix | Examples |
+|--------|----------|
+| `-em` | charmsem, treitschkem, holtem |
+| `-ch` | orbitalech, lekebusch, sklízených |
+| `-ho` | vladivostockého, sertoliho, cenokarpního |
+| `-ou` | hobgarskou, výfukovou, robotou |
+
+### 6.3 Bound Stems (Lexical Roots)
+
+Bound stems are high-frequency subword units that are semantically cohesive but rarely appear as standalone words. These often correspond to the 'core' of a word that requires inflection or derivation to be valid.
+
+| Stem | Cohesion | Substitutability | Examples |
+|------|----------|------------------|----------|
+| `ovýc` | 2.16x | 487 contexts | ových, xových, nových |
+| `skéh` | 2.15x | 392 contexts | ského, lského, urského |
+| `skýc` | 1.97x | 237 contexts | ských, skýcov, tských |
+| `ický` | 1.57x | 496 contexts | tický, bický, úpický |
+| `nské` | 1.53x | 491 contexts | anské, inské, ínské |
+| `ován` | 1.44x | 594 contexts | ování, kován, zování |
+| `ické` | 1.46x | 499 contexts | tické, lické, mické |
+| `ledn` | 1.59x | 250 contexts | lednu, ledna, ledný |
+| `itel` | 1.36x | 634 contexts | nitel, litel, pitel |
+| `cház` | 1.52x | 287 contexts | chází, schází, ochází |
+| `dkaz` | 2.66x | 23 contexts | odkaz, odkaze, odkazy |
+| `xter` | 1.81x | 76 contexts | exter, xterm, extern |
+
+### 6.4 Affix Compatibility (Co-occurrence)
+
+This table shows which prefixes and suffixes most frequently co-occur on the same stems, revealing the 'stacking' rules of the language's morphology.
+
+| Prefix | Suffix | Frequency | Examples |
+|--------|--------|-----------|----------|
+| `-ne` | `-ch` | 14 words | nepropouštějících, netermínovaných |
+| `-ne` | `-ho` | 10 words | nejpokročilejšího, nezpochybnitelného |
+| `-ne` | `-ou` | 9 words | nestejnou, nerozšiřitelnou |
+| `-po` | `-ho` | 9 words | podmínkového, polštářovitého |
+| `-po` | `-ch` | 7 words | pohodlnějších, polohovkách |
+| `-po` | `-ou` | 6 words | ponitranskou, pomátnou |
+| `-po` | `-em` | 3 words | pollackem, povříslem |
+
+### 6.5 Recursive Morpheme Segmentation
+
+Using **Recursive Hierarchical Substitutability**, we decompose complex words into their constituent morphemes. This approach handles nested affixes (e.g., `prefix-prefix-root-suffix`).
+
+| Word | Suggested Split | Confidence | Stem |
+|------|-----------------|------------|------|
+| nedoloženou | **`ne-doložen-ou`** | 6.0 | `doložen` |
+| nepochybovala | **`ne-po-chybovala`** | 6.0 | `chybovala` |
+| nepostaral | **`ne-po-staral`** | 6.0 | `staral` |
+| nacionálem | **`nacionál-em`** | 4.5 | `nacionál` |
+| chimentiho | **`chimenti-ho`** | 4.5 | `chimenti` |
+| prostonárodního | **`prostonárodní-ho`** | 4.5 | `prostonárodní` |
+| klokotských | **`klokotský-ch`** | 4.5 | `klokotský` |
+| bibliografického | **`bibliografické-ho`** | 4.5 | `bibliografické` |
+| nesvědčily | **`ne-svědčily`** | 4.5 | `svědčily` |
+| nenavázali | **`ne-navázali`** | 4.5 | `navázali` |
+| ibragimovem | **`ibragimov-em`** | 4.5 | `ibragimov` |
+| zeměplošských | **`zeměplošský-ch`** | 4.5 | `zeměplošský` |
+| hliníkových | **`hliníkový-ch`** | 4.5 | `hliníkový` |
+| etylenglykolem | **`etylenglykol-em`** | 4.5 | `etylenglykol` |
+| mnohosamicového | **`mnohosamicové-ho`** | 4.5 | `mnohosamicové` |
+
+### 6.6 Linguistic Interpretation
+
+> **Automated Insight:**
+The language Czech shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
+
+---
+## 7. Summary & Recommendations
 
 ![Performance Dashboard](visualizations/performance_dashboard.png)
 
@@ -344,11 +541,12 @@ Below are text samples generated from each Markov chain model:
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **32k BPE** | Best compression (4.13x) with low UNK rate |
-| N-gram | **5-gram** | Lowest perplexity (540) |
-| Markov | **Context-4** | Highest predictability (91.5%) |
+| Tokenizer | **64k BPE** | Best compression (4.59x) |
+| N-gram | **2-gram** | Lowest perplexity (449) |
+| Markov | **Context-4** | Highest predictability (94.4%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
+
 ---
 ## Appendix: Metrics Glossary & Interpretation Guide
 
@@ -538,7 +736,8 @@ If you use these models in your research, please cite:
   author = {Kamali, Omar},
   title = {Wikilangs: Open NLP Models for Wikipedia Languages},
   year = {2025},
-  publisher = {HuggingFace},
+  doi = {10.5281/zenodo.18073153},
+  publisher = {Zenodo},
   url = {https://huggingface.co/wikilangs}
   institution = {Omneity Labs}
 }
@@ -554,7 +753,8 @@ MIT License - Free for academic and commercial use.
 - 🤗 Models: [huggingface.co/wikilangs](https://huggingface.co/wikilangs)
 - 📊 Data: [wikipedia-monthly](https://huggingface.co/datasets/omarkamali/wikipedia-monthly)
 - 👤 Author: [Omar Kamali](https://huggingface.co/omarkamali)
+- 🤝 Sponsor: [Featherless AI](https://featherless.ai)
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2025-12-29 05:36:16*
+*Report Date: 2026-01-08 17:02:58*

models/embeddings/aligned/cs_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee73a43ec364b9064a5ce15c015db41e81048adbd31d37088f27c0d6393be945
+size 2592173419

models/embeddings/aligned/cs_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "cs", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/cs_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fd4a4b58d02359e21ce26e20e4016044953c9fff3f7dc1b954aff16af53fbf2
+size 65664

models/embeddings/aligned/cs_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "cs",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 248433,
+  "vocab_size": 1503404
+}

models/embeddings/aligned/cs_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d1c30712ea75937478c58b7fb8c1c3caf83309663a594ca004504a00193bb46
+size 669559147

models/embeddings/aligned/cs_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "cs", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/cs_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efd67a9b2efccb4b9ebbd0d7fdbe5b7b688d27853ec7334edef1eb0fa5a25a6f
+size 4224

models/embeddings/aligned/cs_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "cs",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 248433,
+  "vocab_size": 1503404
+}

models/embeddings/aligned/cs_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d7e7c706963168c9f66fbb72372f5f81606711130ef7466f09e14ed88e7a4d0
+size 1310430571

models/embeddings/aligned/cs_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "cs", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/cs_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcb8041a156dac565236b7bc938205bae3bdf77a3b16b4d40c14ffeb65c7cadb
+size 16512

models/embeddings/aligned/cs_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "cs",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 248433,
+  "vocab_size": 1503404
+}

models/embeddings/monolingual/cs_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:034be2a68b65bcb979f8fd0ad025b0db9cab145669e15df9a7f1486b7f1a70d7
-size 2755994697
+oid sha256:ee73a43ec364b9064a5ce15c015db41e81048adbd31d37088f27c0d6393be945
+size 2592173419

models/embeddings/monolingual/cs_128d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 128,
   "version": "monolingual",
   "training_params": {
-    "dim": 128,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 128
   },
-  "vocab_size": 1659956
+  "vocab_size": 1503404
 }

models/embeddings/monolingual/cs_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4319e47ddfbdaf7e51194eb7d0f217f4c9be97174c76fb134b4ee3ffeb2e0897
-size 713148489
+oid sha256:4d1c30712ea75937478c58b7fb8c1c3caf83309663a594ca004504a00193bb46
+size 669559147

models/embeddings/monolingual/cs_32d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 32,
   "version": "monolingual",
   "training_params": {
-    "dim": 32,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 32
   },
-  "vocab_size": 1659956
+  "vocab_size": 1503404
 }

models/embeddings/monolingual/cs_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8217b6554747d00396fbef9bba76405a29ea8883630e21ebcb1e06530642d7d1
-size 1394097225
+oid sha256:8d7e7c706963168c9f66fbb72372f5f81606711130ef7466f09e14ed88e7a4d0
+size 1310430571

models/embeddings/monolingual/cs_64d_metadata.json

@@ -3,11 +3,13 @@
   "dimension": 64,
   "version": "monolingual",
   "training_params": {
-    "dim": 64,
+    "algorithm": "skipgram",
     "min_count": 5,
     "window": 5,
     "negative": 5,
-    "epochs": 5
+    "epochs": 5,
+    "encoding_method": "rope",
+    "dim": 64
   },
-  "vocab_size": 1659956
+  "vocab_size": 1503404
 }

models/subword_markov/cs_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1189c98a361867f4246e4275f1b3cc259688644d0e3fd5ef3e0f131b7fa96c1d
-size 812432
+oid sha256:f728385f32db4d56eb98f894af4f69b3ed5eff19c0b9d1c11e941cba1f3f8c54
+size 839807

models/subword_markov/cs_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "cs",
-  "unique_contexts": 10644,
-  "total_transitions": 1795628327
+  "unique_contexts": 14369,
+  "total_transitions": 1591006734
 }

models/subword_markov/cs_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d80c00661e7ca277f295ef570e079bc0f40addf0e5da5aee5a2c8484c8ff03b1
-size 5964730
+oid sha256:9c686d9a25d6802505309d41e46898ae44a9610103bbf6b43b177215526f053b
+size 4923338

models/subword_markov/cs_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "cs",
-  "unique_contexts": 132235,
-  "total_transitions": 1795047002
+  "unique_contexts": 123767,
+  "total_transitions": 1590425862
 }

models/subword_markov/cs_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6bb7c3a0114ea8d93ca96ea2f76cc79ae31651b3708426fdbdca9ef5019f3703
-size 30364313
+oid sha256:2efc4336d93106340240bfa8c0fcdb41850c4e711995576d18cac4a39b2921fc
+size 21935499

models/subword_markov/cs_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "cs",
-  "unique_contexts": 722190,
-  "total_transitions": 1794465677
+  "unique_contexts": 583275,
+  "total_transitions": 1589844990
 }

models/subword_markov/cs_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:54f5b723aa026548afb4a796c1f266538d54dad4a3b88971d09bae038aa52c12
-size 130573936
+oid sha256:16d70dfb8facb64efbcd9b152d807fa295c04a1b0c433d6daa1b99f45fcf62c6
+size 89927521

models/subword_markov/cs_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "cs",
-  "unique_contexts": 4107972,
-  "total_transitions": 1793884352
+  "unique_contexts": 2782368,
+  "total_transitions": 1589264118
 }

models/subword_ngram/cs_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:699c97e596d231e7e2c92b0a1a01c3fee395be13f0eaebd6cc7a5de244db366b
-size 505831
+oid sha256:f3bca592f8a143683b686f0e37a72e554f552c21ef033b2ff317d7964000b9bc
+size 433014

models/subword_ngram/cs_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "cs",
-  "unique_ngrams": 35551,
-  "total_ngrams": 1795628327
+  "unique_ngrams": 30223,
+  "total_ngrams": 1591006734
 }

models/subword_ngram/cs_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d7a587c6dec2716a1308f0986b57b783fb4e190cfa5d13628ba80dfd2ade9cf
-size 4237349
+oid sha256:39fc7ee4da840ca0792f53822766d4e880f321021b7b2192a655d5d175332d23
+size 3254803

models/subword_ngram/cs_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "cs",
-  "unique_ngrams": 338538,
-  "total_ngrams": 1795047002
+  "unique_ngrams": 255109,
+  "total_ngrams": 1590425862
 }

models/subword_ngram/cs_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4c9df563dea7bf57f71855ad9bbb6313209791d033cad384a57f751da02a0f2
-size 29932999
+oid sha256:b789c05230d10bdbd3be1e9661bec890f8aa337f333ec24cdfd811b63240337a
+size 20392535

models/subword_ngram/cs_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "cs",
-  "unique_ngrams": 2482399,
-  "total_ngrams": 1794465677
+  "unique_ngrams": 1646964,
+  "total_ngrams": 1589844990
 }

models/subword_ngram/cs_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c457c38c3eb730bc5b170bc78b1ffb26e1125256cf7ff9765eaec66b334f858a
+size 80495807

models/subword_ngram/cs_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "cs",
+  "unique_ngrams": 6437902,
+  "total_ngrams": 1589264118
+}

models/tokenizer/cs_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6f21f9f740610ed50f3312df4811940dcbc3c9f89f0b0898ad2cb28d0733451
-size 514778
+oid sha256:a79a59e7e6f05b35164ccd20e22fd357803c388d68fffcb0739fea4e66115909
+size 514491

models/tokenizer/cs_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:073977c2b03615dfcc7f5b7f9a464f331d5c93321d46434604c98a0295b16fa8
-size 804922
+oid sha256:92d519e9300c95d83a96fa4f4de0a13584fb4b643d9dd8c8d09f4fad15dff835
+size 803835

models/tokenizer/cs_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8da8677349daac2fa7f0977407831d7669c7cde4e4213febebdb1aeec967857f
-size 1402061
+oid sha256:3e547414ac25c5799f3cbcf8c9d78e1473f21398b1b2e6e9e6d4668900be78de
+size 1397324

models/tokenizer/cs_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f1bbe8a7e6a224eed7d53e339bcf4df48d3fd66f77f1bf2609e42d9ab6b8f5b
-size 374540
+oid sha256:a2101aaa123afbd98d94b7170eb4cf00b0c5ef0dfc983c7359e00730b098aed0
+size 374590

models/vocabulary/cs_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8a85a74123473b20ac5f44858db1993e00f646437af9ce9d0e0b0194c590114
-size 15653412
+oid sha256:0f4ac7c5f2fe22fcc2625fb967be7f5b9111f19cfce6c05ef270a509337034c8
+size 28462430

models/vocabulary/cs_vocabulary_metadata.json

@@ -1,16 +1,17 @@
 {
   "language": "cs",
-  "vocabulary_size": 1000000,
+  "vocabulary_size": 1830714,
+  "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.019459510539549023,
+    "type_token_ratio": 0.01593860093466459,
     "coverage": {
-      "top_100": 0.2667156178234961,
-      "top_1000": 0.459097000188151,
-      "top_5000": 0.6309909217632615,
-      "top_10000": 0.7050773559377174
+      "top_100": 0.26912381903885885,
+      "top_1000": 0.4535615447036626,
+      "top_5000": 0.6242837292543639,
+      "top_10000": 0.7003857714922751
     },
-    "hapax_count": 3049908,
-    "hapax_ratio": 0.5888045382175596,
-    "total_documents": 581325
+    "hapax_count": 1988181,
+    "hapax_ratio": 0.520616827642551,
+    "total_documents": 580872
   }
 }

models/vocabulary/cs_vocabulary_top.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b61261bafc47e9fd112290a889706214bc8ebfd90036ac510310fd8a7f0f99bf
+size 15708837

models/vocabulary/cs_vocabulary_top_metadata.json

@@ -0,0 +1,20 @@
+{
+  "language": "cs",
+  "vocabulary_size": 1000000,
+  "variant": "top",
+  "statistics": {
+    "type_token_ratio": 0.01593860093466459,
+    "coverage": {
+      "top_100": 0.26912381903885885,
+      "top_1000": 0.4535615447036626,
+      "top_5000": 0.6242837292543639,
+      "top_10000": 0.7003857714922751
+    },
+    "hapax_count": 1988181,
+    "hapax_ratio": 0.520616827642551,
+    "total_documents": 580872,
+    "top_vocab_size": 1000000,
+    "coverage_ratio": 0.983133787887407,
+    "tokens_excluded": 830714
+  }
+}

models/word_markov/cs_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8eae25b71953a72a3b4ad0cdb7cc9159ab32d4feb13cd01007eba2a1452ef77d
-size 529880714
+oid sha256:daba16a55ffa7e1547d39e480bc3ed52259c7bc93bc3947e6db1312e0e5cdc35
+size 571097267

models/word_markov/cs_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "cs",
-  "unique_contexts": 5180358,
-  "total_transitions": 336349903
+  "unique_contexts": 3817910,
+  "total_transitions": 239019518
 }