Upload all models and assets for co (latest)

.gitattributes

@@ -39,3 +39,4 @@ visualizations/position_encoding_comparison.png filter=lfs diff=lfs merge=lfs -t
 visualizations/tsne_sentences.png filter=lfs diff=lfs merge=lfs -text
 visualizations/tsne_words.png filter=lfs diff=lfs merge=lfs -text
 visualizations/zipf_law.png filter=lfs diff=lfs merge=lfs -text
+visualizations/embedding_tsne_multilingual.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 language: co
-language_name: CO
+language_name: Corsican
 language_family: romance_galloitalic
 tags:
   - wikilangs
@@ -10,11 +10,21 @@ tags:
   - n-gram
   - markov
   - wikipedia
+  - feature-extraction
+  - sentence-similarity
+  - tokenization
+  - n-grams
+  - markov-chain
+  - text-mining
+  - fasttext
+  - babelvec
+  - vocabulous
+  - vocabulary
   - monolingual
   - family-romance_galloitalic
 license: mit
 library_name: wikilangs
-pipeline_tag: feature-extraction
+pipeline_tag: text-generation
 datasets:
   - omarkamali/wikipedia-monthly
 dataset_info:
@@ -23,20 +33,20 @@ dataset_info:
 metrics:
   - name: best_compression_ratio
     type: compression
-    value: 4.197
+    value: 4.216
   - name: best_isotropy
     type: isotropy
-    value: 0.8272
+    value: 0.8262
   - name: vocabulary_size
     type: vocab
     value: 0
 generated: 2026-01-03
 ---
 
-# CO - Wikilangs Models
+# Corsican - Wikilangs Models
 ## Comprehensive Research Report & Full Ablation Study
 
-This repository contains NLP models trained and evaluated by Wikilangs, specifically on **CO** Wikipedia data.
+This repository contains NLP models trained and evaluated by Wikilangs, specifically on **Corsican** Wikipedia data.
 We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and word embeddings.
 
 ## 📋 Repository Contents
@@ -60,7 +70,7 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 - [3. Markov Chain Evaluation](#3-markov-chain-evaluation)
 - [4. Vocabulary Analysis](#4-vocabulary-analysis)
 - [5. Word Embeddings Evaluation](#5-word-embeddings-evaluation)
-- [6. Morphological Analysis (Experimental)](#6-morphological-analysis)
+- [6. Morphological Analysis (Experimental)](#6--morphological-analysis-experimental)
 - [7. Summary & Recommendations](#7-summary--recommendations)
 - [Metrics Glossary](#appendix-metrics-glossary--interpretation-guide)
 - [Visualizations Index](#visualizations-index)
@@ -80,47 +90,47 @@ We analyze tokenizers, n-gram models, Markov chains, vocabulary statistics, and
 
 | Vocab Size | Compression | Avg Token Len | UNK Rate | Total Tokens |
 |------------|-------------|---------------|----------|--------------|
-| **8k** | 3.418x | 3.42 | 0.0321% | 368,161 |
-| **16k** | 3.691x | 3.69 | 0.0346% | 340,883 |
-| **32k** | 3.970x | 3.97 | 0.0372% | 316,946 |
-| **64k** | 4.197x 🏆 | 4.20 | 0.0394% | 299,775 |
+| **8k** | 3.429x | 3.43 | 0.0264% | 363,461 |
+| **16k** | 3.706x | 3.71 | 0.0285% | 336,335 |
+| **32k** | 3.986x | 3.99 | 0.0307% | 312,675 |
+| **64k** | 4.216x 🏆 | 4.22 | 0.0325% | 295,625 |
 
 ### Tokenization Examples
 
 Below are sample sentences tokenized with each vocabulary size:
 
-**Sample 1:** `Agliana hè una cumuna toscana di a pruvincia di Pistoia. Teni abitanti cumuna di...`
+**Sample 1:** `Ophrys splendida hè una pianta chì face partita di a famiglia di l'orchidaceae. ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁a gli ana ▁hè ▁una ▁cumuna ▁toscana ▁di ▁a ▁pruvincia ... (+10 more)` | 20 |
-| 16k | `▁a gli ana ▁hè ▁una ▁cumuna ▁toscana ▁di ▁a ▁pruvincia ... (+8 more)` | 18 |
-| 32k | `▁agli ana ▁hè ▁una ▁cumuna ▁toscana ▁di ▁a ▁pruvincia ▁di ... (+7 more)` | 17 |
-| 64k | `▁agliana ▁hè ▁una ▁cumuna ▁toscana ▁di ▁a ▁pruvincia ▁di ▁pistoia ... (+6 more)` | 16 |
+| 8k | `▁ophrys ▁sp len di da ▁hè ▁una ▁pianta ▁chì ▁face ... (+13 more)` | 23 |
+| 16k | `▁ophrys ▁splen di da ▁hè ▁una ▁pianta ▁chì ▁face ▁partita ... (+12 more)` | 22 |
+| 32k | `▁ophrys ▁splendi da ▁hè ▁una ▁pianta ▁chì ▁face ▁partita ▁di ... (+11 more)` | 21 |
+| 64k | `▁ophrys ▁splendida ▁hè ▁una ▁pianta ▁chì ▁face ▁partita ▁di ▁a ... (+10 more)` | 20 |
 
-**Sample 2:** `Monteriggioni hè una cumuna toscana di a pruvincia di Siena.Teni 7.877 abitanti....`
+**Sample 2:** `U Mucale hè una cumuna di u dipartimentu di a Corsica suprana. Geografia Storia ...`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁monte ri ggi oni ▁hè ▁una ▁cumuna ▁toscana ▁di ▁a ... (+15 more)` | 25 |
-| 16k | `▁monte ri ggi oni ▁hè ▁una ▁cumuna ▁toscana ▁di ▁a ... (+15 more)` | 25 |
-| 32k | `▁monte ri ggi oni ▁hè ▁una ▁cumuna ▁toscana ▁di ▁a ... (+15 more)` | 25 |
-| 64k | `▁monteriggioni ▁hè ▁una ▁cumuna ▁toscana ▁di ▁a ▁pruvincia ▁di ▁siena ... (+12 more)` | 22 |
+| 8k | `▁u ▁mu cale ▁hè ▁una ▁cumuna ▁di ▁u ▁dipartimentu ▁di ... (+14 more)` | 24 |
+| 16k | `▁u ▁mu cale ▁hè ▁una ▁cumuna ▁di ▁u ▁dipartimentu ▁di ... (+14 more)` | 24 |
+| 32k | `▁u ▁mucale ▁hè ▁una ▁cumuna ▁di ▁u ▁dipartimentu ▁di ▁a ... (+13 more)` | 23 |
+| 64k | `▁u ▁mucale ▁hè ▁una ▁cumuna ▁di ▁u ▁dipartimentu ▁di ▁a ... (+13 more)` | 23 |
 
-**Sample 3:** `Sean Justin Penn hè un attore americanu. Biugrafia Da vede dinò The Thin Red Lin...`
+**Sample 3:** `L'Emilia è Romagna hè una regione taliana. taliana`
 
 | Vocab | Tokens | Count |
 |-------|--------|-------|
-| 8k | `▁s ean ▁j us tin ▁pen n ▁hè ▁un ▁attore ... (+22 more)` | 32 |
-| 16k | `▁sean ▁jus tin ▁pen n ▁hè ▁un ▁attore ▁americanu . ... (+16 more)` | 26 |
-| 32k | `▁sean ▁jus tin ▁penn ▁hè ▁un ▁attore ▁americanu . ▁biugrafia ... (+13 more)` | 23 |
-| 64k | `▁sean ▁justin ▁penn ▁hè ▁un ▁attore ▁americanu . ▁biugrafia ▁da ... (+12 more)` | 22 |
+| 8k | `▁l ' e mi lia ▁è ▁roma gna ▁hè ▁una ... (+4 more)` | 14 |
+| 16k | `▁l ' emi lia ▁è ▁roma gna ▁hè ▁una ▁regione ... (+3 more)` | 13 |
+| 32k | `▁l ' emi lia ▁è ▁romagna ▁hè ▁una ▁regione ▁taliana ... (+2 more)` | 12 |
+| 64k | `▁l ' emilia ▁è ▁romagna ▁hè ▁una ▁regione ▁taliana . ... (+1 more)` | 11 |
 
 
 ### Key Findings
 
-- **Best Compression:** 64k achieves 4.197x compression
-- **Lowest UNK Rate:** 8k with 0.0321% unknown tokens
+- **Best Compression:** 64k achieves 4.216x compression
+- **Lowest UNK Rate:** 8k with 0.0264% unknown tokens
 - **Trade-off:** Larger vocabularies improve compression but increase model size
 - **Recommendation:** 32k vocabulary provides optimal balance for production use
 
@@ -137,12 +147,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | N-gram | Variant | Perplexity | Entropy | Unique N-grams | Top-100 Coverage | Top-1000 Coverage |
 |--------|---------|------------|---------|----------------|------------------|-------------------|
-| **2-gram** | Word | 9,228 | 13.17 | 49,319 | 22.0% | 44.8% |
-| **2-gram** | Subword | 221 🏆 | 7.79 | 3,181 | 71.2% | 99.6% |
-| **3-gram** | Word | 24,246 | 14.57 | 83,012 | 11.1% | 30.7% |
-| **3-gram** | Subword | 1,706 | 10.74 | 22,404 | 28.4% | 77.6% |
-| **4-gram** | Word | 41,538 | 15.34 | 136,699 | 9.4% | 25.8% |
-| **4-gram** | Subword | 9,044 | 13.14 | 107,042 | 13.9% | 42.6% |
+| **2-gram** | Word | 9,217 | 13.17 | 49,361 | 22.0% | 44.8% |
+| **2-gram** | Subword | 220 🏆 | 7.78 | 3,170 | 71.3% | 99.6% |
+| **3-gram** | Word | 24,245 | 14.57 | 83,032 | 11.2% | 30.7% |
+| **3-gram** | Subword | 1,698 | 10.73 | 22,203 | 28.4% | 77.7% |
+| **4-gram** | Word | 41,699 | 15.35 | 137,212 | 9.3% | 25.7% |
+| **4-gram** | Subword | 9,000 | 13.14 | 106,299 | 13.9% | 42.6% |
+| **5-gram** | Word | 36,326 | 15.15 | 111,629 | 9.3% | 26.7% |
+| **5-gram** | Subword | 31,819 | 14.96 | 280,787 | 8.5% | 26.7% |
 
 ### Top 5 N-grams by Size
 
@@ -150,21 +162,21 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `di u` | 18,783 |
-| 2 | `di a` | 18,523 |
-| 3 | `di l` | 13,279 |
-| 4 | `di i` | 10,605 |
-| 5 | `à u` | 9,199 |
+| 1 | `di u` | 18,692 |
+| 2 | `di a` | 18,500 |
+| 3 | `di l` | 13,231 |
+| 4 | `di i` | 10,603 |
+| 5 | `à u` | 9,233 |
 
 **3-grams (Word):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
 | 1 | `a famiglia di` | 4,349 |
-| 2 | `hè una spezia` | 3,355 |
-| 3 | `di a famiglia` | 2,698 |
+| 2 | `hè una spezia` | 3,359 |
+| 3 | `di a famiglia` | 2,699 |
 | 4 | `hè una pianta` | 2,612 |
-| 5 | `una spezia di` | 2,287 |
+| 5 | `una spezia di` | 2,290 |
 
 **4-grams (Word):**
 
@@ -172,46 +184,66 @@ Below are sample sentences tokenized with each vocabulary size:
 |------|--------|-------|
 | 1 | `di a famiglia di` | 2,629 |
 | 2 | `a famiglia di i` | 2,171 |
-| 3 | `hè una spezia di` | 2,061 |
+| 3 | `hè una spezia di` | 2,064 |
 | 4 | `annantu à wikimedia commons` | 1,945 |
-| 5 | `à wikimedia commons di` | 1,923 |
+| 5 | `à wikimedia commons di` | 1,924 |
+
+**5-grams (Word):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `annantu à wikimedia commons di` | 1,924 |
+| 2 | `à wikimedia commons di corsica` | 1,923 |
+| 3 | `appartinendu à a famiglia di` | 1,506 |
+| 4 | `flora corsica 2 ed edisud` | 1,421 |
+| 5 | `d gamisans j flora corsica` | 1,419 |
 
 **2-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `i _` | 432,981 |
-| 2 | `a _` | 404,157 |
-| 3 | `u _` | 316,081 |
-| 4 | `_ d` | 246,351 |
-| 5 | `d i` | 217,005 |
+| 1 | `i _` | 432,205 |
+| 2 | `a _` | 403,888 |
+| 3 | `u _` | 315,849 |
+| 4 | `_ d` | 246,098 |
+| 5 | `d i` | 216,563 |
 
 **3-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ d i` | 173,124 |
-| 2 | `d i _` | 152,141 |
-| 3 | `_ i n` | 82,653 |
-| 4 | `_ u _` | 81,426 |
-| 5 | `_ a _` | 72,871 |
+| 1 | `_ d i` | 172,754 |
+| 2 | `d i _` | 151,658 |
+| 3 | `_ i n` | 82,722 |
+| 4 | `_ u _` | 81,534 |
+| 5 | `_ a _` | 73,027 |
 
 **4-grams (Subword):**
 
 | Rank | N-gram | Count |
 |------|--------|-------|
-| 1 | `_ d i _` | 143,493 |
-| 2 | `_ i n _` | 57,416 |
-| 3 | `a _ d i` | 45,268 |
-| 4 | `_ h è _` | 44,732 |
-| 5 | `i _ d i` | 35,176 |
+| 1 | `_ d i _` | 143,050 |
+| 2 | `_ i n _` | 57,478 |
+| 3 | `a _ d i` | 45,041 |
+| 4 | `_ h è _` | 45,025 |
+| 5 | `i _ d i` | 35,043 |
+
+**5-grams (Subword):**
+
+| Rank | N-gram | Count |
+|------|--------|-------|
+| 1 | `a _ d i _` | 37,617 |
+| 2 | `i _ d i _` | 29,786 |
+| 3 | `u _ d i _` | 28,746 |
+| 4 | `e _ d i _` | 24,400 |
+| 5 | `i o n e _` | 21,123 |
 
 
 ### Key Findings
 
-- **Best Perplexity:** 2-gram (subword) with 221
+- **Best Perplexity:** 2-gram (subword) with 220
 - **Entropy Trend:** Decreases with larger n-grams (more predictable)
-- **Coverage:** Top-1000 patterns cover ~43% of corpus
+- **Coverage:** Top-1000 patterns cover ~27% of corpus
 - **Recommendation:** 4-gram or 5-gram for best predictive performance
 
 ---
@@ -227,14 +259,14 @@ Below are sample sentences tokenized with each vocabulary size:
 
 | Context | Variant | Avg Entropy | Perplexity | Branching Factor | Unique Contexts | Predictability |
 |---------|---------|-------------|------------|------------------|-----------------|----------------|
-| **1** | Word | 0.8949 | 1.860 | 5.60 | 123,267 | 10.5% |
-| **1** | Subword | 1.0516 | 2.073 | 8.41 | 976 | 0.0% |
-| **2** | Word | 0.3102 | 1.240 | 1.80 | 688,381 | 69.0% |
-| **2** | Subword | 0.9618 | 1.948 | 5.61 | 8,204 | 3.8% |
-| **3** | Word | 0.1337 | 1.097 | 1.25 | 1,235,287 | 86.6% |
-| **3** | Subword | 0.7919 | 1.731 | 3.99 | 46,007 | 20.8% |
-| **4** | Word | 0.0622 🏆 | 1.044 | 1.10 | 1,541,605 | 93.8% |
-| **4** | Subword | 0.6473 | 1.566 | 2.90 | 183,668 | 35.3% |
+| **1** | Word | 0.8927 | 1.857 | 5.58 | 123,322 | 10.7% |
+| **1** | Subword | 0.8627 | 1.818 | 6.97 | 1,238 | 13.7% |
+| **2** | Word | 0.3106 | 1.240 | 1.80 | 686,898 | 68.9% |
+| **2** | Subword | 0.9133 | 1.883 | 5.37 | 8,617 | 8.7% |
+| **3** | Word | 0.1339 | 1.097 | 1.25 | 1,233,325 | 86.6% |
+| **3** | Subword | 0.7817 | 1.719 | 3.96 | 46,221 | 21.8% |
+| **4** | Word | 0.0623 🏆 | 1.044 | 1.10 | 1,539,570 | 93.8% |
+| **4** | Subword | 0.6452 | 1.564 | 2.90 | 182,986 | 35.5% |
 
 ### Generated Text Samples (Word-based)
 
@@ -242,27 +274,27 @@ Below are text samples generated from each word-based Markov chain model:
 
 **Context Size 1:**
 
-1. `di l onore di culombu facciandine e rete di pianta di u guvernu à spessu in`
-2. `u so foglie basale lanceulate è difficili suprattuttu vocalici senza fà la tira glǝ munnǝ è`
-3. `a gioia quandu ridatti i casci sò forsi statu indipindente di l india di sporti ecunumia`
+1. `di tuda hè una spezia hè un missale rumanu mandatu pè a prutezzione di l isula`
+2. `u calendariu gregorianu evenimenti nascite morte celebrazione feste i primi cristiani è l euru e zon...`
+3. `a bellula chì faci cantà senza scoddhi e pratuline i bagni di 25 aprile di nettaru`
 
 **Context Size 2:**
 
-1. `di u muvimentu di a pruvincia agira aidone assoro calascibetta caltanissetta cl gangi pa leonforte n...`
-2. `di a corsica nordu africa uccidintali in sudafrica è in europa meridiunale è cintrale burhinus oedic...`
-3. `di l annu avenimenti in corsica jeanmonod d gamisans j flora corsica 2 ed edisud noti altri`
+1. `di u mare à trasporti maritimi portivechju hà ancu statu cunnisciuta sottu u nomu simonu a casata`
+2. `di a spagna un statu di spiritu turmintosa da veda dinò camisgia pilonu a camisgetta di corsica`
+3. `di l europa occidentale di cipru di u bacinu mediterraniu induv ella hè ghjunta in alisgiani u`
 
 **Context Size 3:**
 
-1. `a famiglia di e papaveraceae si distingue da i so piccioli sò ritti ramificati è cuparti à pela`
-2. `hè una spezia di pianta chì face parte di a famiglia di i labrinae ss articulu pruveni in`
-3. `di a famiglia di i poaceae discrizzioni poa bulbosa hè prisenti in l alpi i pirenei è i`
+1. `a famiglia di l orobanchaceae si distingui da i so grandi fiori gialli è arancini à forma di`
+2. `hè una spezia largamente sparta in a so aria di ripartizioni eppuri certi pupulazioni poni essa mina...`
+3. `di a famiglia di i brassicaceae si caratterizeghja da u so portu cispugliosu è cumpattu aghjunghjend...`
 
 **Context Size 4:**
 
-1. `di a famiglia di e polygonaceae si distingue da e so fiurarelli rusulatu pallidu à purpureu ragruppa...`
-2. `a famiglia di i dryopteridaceae ss articulu pruveni in parti o in tutalità da l articulu currispunde...`
-3. `hè una spezia di pianta arbacea vivaci appartinendu à a famiglia di i fabaceae discrizzioni ornithop...`
+1. `di a famiglia di l arecaceae ed hè largamenti apprizzatu par a so biddezza è u so simbulu astrunomic...`
+2. `a famiglia di i sapindaceae discrizzioni l acer negundo hè un arburi scascianti chì pò aghjunghja un...`
+3. `hè una spezia di pianta chì faci parti di a famiglia di l hirundinidae descrizzione a rundinella cas...`
 
 
 ### Generated Text Samples (Subword-based)
@@ -271,34 +303,34 @@ Below are text samples generated from each subword-based Markov chain model:
 
 **Context Size 1:**
 
-1. `_pinaga._i_cavac`
-2. `isa_se_ssa_villa`
-3. `ara_dinum'a_dici`
+1. `_diri_25_à_di_d'`
+2. `iori_hà_siceisu_`
+3. `adia_puvezota_fi`
 
 **Context Size 2:**
 
-1. `i_fece_fiazionduc`
-2. `a_hè_a_chì_nalegu`
-3. `u_50px_le_d'isi_f`
+1. `i_re_culupula_à_s`
+2. `a_ufoltrupatichar`
+3. `u_à_ligna_culanea`
 
 **Context Size 3:**
 
-1. `_di_"forte_pianu_p`
-2. `di_incamplica_è_fa`
-3. `_in_atlantunimentr`
+1. `_di_abbrunu,_cator`
+2. `di_arbaceae._nore_`
+3. `_induv'eddu;_annan`
 
 **Context Size 4:**
 
-1. `_di_frutti_in_corsa`
-2. `_in_u_harrisparterà`
-3. `a_di_lunghjadori_ri`
+1. `_di_yprestitudi_à_s`
+2. `_in_amba_di_l'incen`
+3. `a_di_l'aurolli_di_b`
 
 
 ### Key Findings
 
 - **Best Predictability:** Context-4 (word) with 93.8% predictability
 - **Branching Factor:** Decreases with context size (more deterministic)
-- **Memory Trade-off:** Larger contexts require more storage (183,668 contexts)
+- **Memory Trade-off:** Larger contexts require more storage (182,986 contexts)
 - **Recommendation:** Context-3 or Context-4 for text generation
 
 ---
@@ -314,64 +346,64 @@ Below are text samples generated from each subword-based Markov chain model:
 
 | Metric | Value |
 |--------|-------|
-| Vocabulary Size | 58,612 |
-| Total Tokens | 2,193,141 |
+| Vocabulary Size | 58,569 |
+| Total Tokens | 2,191,854 |
 | Mean Frequency | 37.42 |
 | Median Frequency | 4 |
-| Frequency Std Dev | 979.13 |
+| Frequency Std Dev | 979.31 |
 
 ### Most Common Words
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | di | 143,885 |
-| 2 | u | 84,171 |
-| 3 | a | 75,994 |
-| 4 | è | 66,959 |
-| 5 | in | 58,823 |
-| 6 | à | 58,335 |
-| 7 | l | 48,252 |
-| 8 | hè | 45,746 |
-| 9 | i | 45,068 |
-| 10 | da | 24,631 |
+| 1 | di | 143,436 |
+| 2 | u | 84,175 |
+| 3 | a | 76,019 |
+| 4 | è | 67,153 |
+| 5 | in | 58,881 |
+| 6 | à | 58,439 |
+| 7 | l | 48,309 |
+| 8 | hè | 46,050 |
+| 9 | i | 45,085 |
+| 10 | da | 24,609 |
 
 ### Least Common Words (from vocabulary)
 
 | Rank | Word | Frequency |
 |------|------|-----------|
-| 1 | zampigiallu | 2 |
-| 2 | lepeletier | 2 |
-| 3 | nigrithorax | 2 |
-| 4 | crabro | 2 |
-| 5 | entomologhi | 2 |
-| 6 | priculusità | 2 |
-| 7 | apiarie | 2 |
-| 8 | cottura | 2 |
-| 9 | risuttati | 2 |
-| 10 | tippicu | 2 |
+| 1 | hannovra | 2 |
+| 2 | multifau | 2 |
+| 3 | vendanges | 2 |
+| 4 | voceratrice | 2 |
+| 5 | paysage | 2 |
+| 6 | coin | 2 |
+| 7 | paysan | 2 |
+| 8 | spezialità | 2 |
+| 9 | alerta | 2 |
+| 10 | ꦈꦠꦩ | 2 |
 
 ### Zipf's Law Analysis
 
 | Metric | Value |
 |--------|-------|
-| Zipf Coefficient | 1.0564 |
-| R² (Goodness of Fit) | 0.996983 |
+| Zipf Coefficient | 1.0566 |
+| R² (Goodness of Fit) | 0.997058 |
 | Adherence Quality | **excellent** |
 
 ### Coverage Analysis
 
 | Top N Words | Coverage |
 |-------------|----------|
-| Top 100 | 48.8% |
+| Top 100 | 48.9% |
 | Top 1,000 | 69.5% |
 | Top 5,000 | 84.0% |
 | Top 10,000 | 89.4% |
 
 ### Key Findings
 
-- **Zipf Compliance:** R²=0.9970 indicates excellent adherence to Zipf's law
-- **High Frequency Dominance:** Top 100 words cover 48.8% of corpus
-- **Long Tail:** 48,612 words needed for remaining 10.6% coverage
+- **Zipf Compliance:** R²=0.9971 indicates excellent adherence to Zipf's law
+- **High Frequency Dominance:** Top 100 words cover 48.9% of corpus
+- **Long Tail:** 48,569 words needed for remaining 10.6% coverage
 
 ---
 ## 5. Word Embeddings Evaluation
@@ -387,37 +419,40 @@ Below are text samples generated from each subword-based Markov chain model:
 
 ### 5.1 Cross-Lingual Alignment
 
-> *Note: Multilingual alignment visualization not available for this language.*
+![Alignment Quality](visualizations/embedding_alignment_quality.png)
+
+![Multilingual t-SNE](visualizations/embedding_tsne_multilingual.png)
 
 
 ### 5.2 Model Comparison
 
 | Model | Dimension | Isotropy | Semantic Density | Alignment R@1 | Alignment R@10 |
 |-------|-----------|----------|------------------|---------------|----------------|
-| **mono_32d** | 32 | 0.8272 🏆 | 0.3408 | N/A | N/A |
-| **mono_64d** | 64 | 0.8188 | 0.2697 | N/A | N/A |
-| **mono_128d** | 128 | 0.7473 | 0.2166 | N/A | N/A |
+| **mono_32d** | 32 | 0.8262 🏆 | 0.3363 | N/A | N/A |
+| **mono_64d** | 64 | 0.8192 | 0.2582 | N/A | N/A |
+| **mono_128d** | 128 | 0.7654 | 0.2010 | N/A | N/A |
+| **aligned_32d** | 32 | 0.8262 | 0.3340 | 0.0540 | 0.2540 |
+| **aligned_64d** | 64 | 0.8192 | 0.2633 | 0.0880 | 0.3460 |
+| **aligned_128d** | 128 | 0.7654 | 0.1975 | 0.1560 | 0.4960 |
 
 ### Key Findings
 
-- **Best Isotropy:** mono_32d with 0.8272 (more uniform distribution)
-- **Semantic Density:** Average pairwise similarity of 0.2757. Lower values indicate better semantic separation.
-- **Alignment Quality:** No aligned models evaluated in this run.
+- **Best Isotropy:** mono_32d with 0.8262 (more uniform distribution)
+- **Semantic Density:** Average pairwise similarity of 0.2651. Lower values indicate better semantic separation.
+- **Alignment Quality:** Aligned models achieve up to 15.6% R@1 in cross-lingual retrieval.
 - **Recommendation:** 128d aligned for best cross-lingual performance
 
 ---
 ## 6.  Morphological Analysis (Experimental)
 
-> ⚠️ **Warning:** This language shows low morphological productivity. The statistical signals used for this analysis may be noisy or less reliable than for morphologically rich languages.
-
 This section presents an automated morphological analysis derived from the statistical divergence between word-level and subword-level models. By analyzing where subword predictability spikes and where word-level coverage fails, we can infer linguistic structures without supervised data.
 
 ### 6.1 Productivity & Complexity
 
 | Metric | Value | Interpretation | Recommendation |
 |--------|-------|----------------|----------------|
-| Productivity Index | **0.000** | Low morphological productivity | ⚠️ Likely unreliable |
-| Idiomaticity Gap | **-1.000** | Low formulaic content | - |
+| Productivity Index | **5.000** | High morphological productivity | Reliable analysis |
+| Idiomaticity Gap | **0.002** | Low formulaic content | - |
 
 ### 6.2 Affix Inventory (Productive Units)
 
@@ -426,26 +461,24 @@ These are the most productive prefixes and suffixes identified by sampling the v
 #### Productive Prefixes
 | Prefix | Examples |
 |--------|----------|
-| `-cu` | cullisori, cunisciutu, cumbinatoria |
-| `-ca` | cartagene, caerulescens, casagliò |
-| `-ri` | rizomatosu, ritiranu, ricustruita |
-| `-in` | innù, ingannatu, ingaghjatu |
-| `-pr` | produtta, predita, produzzione |
-| `-ma` | macidonia, matrimonii, maestro |
-| `-di` | difendidori, differenziale, dicriscenti |
-| `-pa` | pavillon, paola, parentella |
+| `-cu` | cunfutà, cuddazioni, cuntera |
+| `-ca` | castres, caprimulgus, calciu |
+| `-ri` | rivede, rispettà, riurganizò |
+| `-in` | ingegneri, incausà, indì |
+| `-pr` | pridatori, privileghju, preferisci |
+| `-di` | dinastìa, disintegra, dicennovi |
 
 #### Productive Suffixes
 | Suffix | Examples |
 |--------|----------|
-| `-a` | occhjatana, mdina, illeghjittima |
-| `-i` | petrignani, quindici, cullisori |
-| `-u` | locudoresu, glaucu, fuculaghju |
-| `-e` | phryganae, christine, volume |
-| `-tu` | cunisciutu, bassistu, ingannatu |
-| `-ni` | petrignani, parsicuzioni, bizantini |
-| `-ti` | viditi, dalmati, stupefacenti |
-| `-ta` | avvilanata, szocialista, produtta |
+| `-i` | addevi, ingegneri, midianti |
+| `-u` | spagnolu, belgiu, vòtu |
+| `-a` | dinastìa, leucoraja, seduta |
+| `-e` | rivede, uccidentale, marginale |
+| `-tu` | vòtu, validatu, prisirvatu |
+| `-ti` | midianti, rapprisintati, sminticati |
+| `-ni` | cuddazioni, vogliini, cardini |
+| `-ta` | seduta, atalanta, rota |
 
 ### 6.3 Bound Stems (Lexical Roots)
 
@@ -453,18 +486,18 @@ Bound stems are high-frequency subword units that are semantically cohesive but
 
 | Stem | Cohesion | Substitutability | Examples |
 |------|----------|------------------|----------|
-| `endu` | 2.27x | 73 contexts | fendu, vendu, dendu |
-| `enti` | 1.76x | 119 contexts | lenti, penti, menti |
-| `azio` | 1.86x | 55 contexts | tazio, lazio, orazio |
-| `aghj` | 1.50x | 141 contexts | aghje, aghju, aghja |
-| `ment` | 1.57x | 87 contexts | mentr, menti, menta |
-| `glia` | 1.64x | 69 contexts | aglia, figlia, voglia |
-| `zion` | 1.67x | 63 contexts | lezion, azione, nuzione |
-| `igli` | 1.44x | 112 contexts | figli, migli, cigli |
-| `tura` | 1.59x | 62 contexts | altura, matura, turaci |
-| `cors` | 1.85x | 33 contexts | corsa, corso, corsi |
-| `sica` | 1.55x | 37 contexts | fisica, sicani, musica |
-| `ific` | 1.48x | 43 contexts | edificà, pacific, unificà |
+| `endu` | 2.14x | 73 contexts | fendu, vendu, dendu |
+| `enti` | 1.81x | 118 contexts | nenti, denti, lenti |
+| `igli` | 1.63x | 112 contexts | gigli, migli, cigli |
+| `aghj` | 1.46x | 142 contexts | aghji, aghju, aghja |
+| `glia` | 1.66x | 70 contexts | aglia, paglia, figlia |
+| `azio` | 1.75x | 56 contexts | tazio, lazio, orazio |
+| `zion` | 1.65x | 64 contexts | azione, nozione, lezioni |
+| `ment` | 1.48x | 87 contexts | mente, menti, menta |
+| `cors` | 1.80x | 33 contexts | corso, corsa, corse |
+| `ific` | 1.57x | 45 contexts | pacific, unificò, unificà |
+| `tura` | 1.38x | 62 contexts | datura, altura, natura |
+| `sica` | 1.56x | 37 contexts | mùsica, fìsica, sicani |
 
 ### 6.4 Affix Compatibility (Co-occurrence)
 
@@ -472,16 +505,16 @@ This table shows which prefixes and suffixes most frequently co-occur on the sam
 
 | Prefix | Suffix | Frequency | Examples |
 |--------|--------|-----------|----------|
-| `-cu` | `-u` | 75 words | cumandamentu, cutratu |
-| `-cu` | `-i` | 74 words | cuttoli, custituenti |
-| `-ri` | `-i` | 69 words | riunghji, riferimenti |
-| `-pr` | `-i` | 66 words | prufundamenti, primuri |
-| `-in` | `-i` | 66 words | infruttuosi, indoauropei |
-| `-cu` | `-a` | 64 words | cumpattezza, cunghjunghja |
-| `-ca` | `-u` | 63 words | cancelieru, cattru |
-| `-cu` | `-e` | 58 words | cuuperazione, cuscione |
-| `-ca` | `-a` | 54 words | capua, cathartica |
-| `-ri` | `-a` | 51 words | rivolta, ridotta |
+| `-cu` | `-i` | 84 words | curteghji, cubiti |
+| `-cu` | `-u` | 82 words | cuntestatu, cunvertitu |
+| `-ri` | `-u` | 67 words | righjistru, riguardu |
+| `-cu` | `-a` | 64 words | cultelleria, cunsacra |
+| `-cu` | `-e` | 62 words | cundannate, cunstruzione |
+| `-in` | `-u` | 61 words | ingombru, inchietu |
+| `-ca` | `-a` | 59 words | calandra, cantata |
+| `-in` | `-i` | 58 words | insufficienti, intarsizioni |
+| `-ca` | `-u` | 58 words | caratteru, capistranu |
+| `-pr` | `-i` | 56 words | preparazioni, prisintati |
 
 ### 6.5 Recursive Morpheme Segmentation
 
@@ -489,26 +522,26 @@ Using **Recursive Hierarchical Substitutability**, we decompose complex words in
 
 | Word | Suggested Split | Confidence | Stem |
 |------|-----------------|------------|------|
-| incasciata | **`in-ca-scia-ta`** | 7.5 | `scia` |
-| mariteddu | **`ma-ri-teddu`** | 6.0 | `teddu` |
-| olivetani | **`olive-ta-ni`** | 6.0 | `olive` |
-| infattonu | **`in-fatto-nu`** | 6.0 | `fatto` |
-| indebulitu | **`in-debuli-tu`** | 6.0 | `debuli` |
-| cunvertuti | **`cu-nver-tu-ti`** | 4.5 | `nver` |
-| sustenenu | **`sustene-nu`** | 4.5 | `sustene` |
-| cunsultatu | **`cu-nsul-ta-tu`** | 4.5 | `nsul` |
-| dilimitatu | **`di-limi-ta-tu`** | 4.5 | `limi` |
-| reichardia | **`reichard-ia`** | 4.5 | `reichard` |
-| affissati | **`affissa-ti`** | 4.5 | `affissa` |
-| riabilità | **`ri-abilità`** | 4.5 | `abilità` |
-| siracusani | **`siracusa-ni`** | 4.5 | `siracusa` |
-| ripresenta | **`ri-pr-esen-ta`** | 4.5 | `esen` |
-| chjappani | **`chjappa-ni`** | 4.5 | `chjappa` |
+| indibulitu | **`in-di-buli-tu`** | 7.5 | `buli` |
+| dirighjitu | **`di-ri-ghji-tu`** | 7.5 | `ghji` |
+| dimustrati | **`di-mustra-ti`** | 6.0 | `mustra` |
+| ricustruisce | **`ri-cu-struisce`** | 6.0 | `struisce` |
+| ricustruite | **`ri-cu-struite`** | 6.0 | `struite` |
+| saturnianu | **`saturn-ia-nu`** | 6.0 | `saturn` |
+| rivoltani | **`ri-volta-ni`** | 6.0 | `volta` |
+| divenendu | **`di-venendu`** | 4.5 | `venendu` |
+| indicheghjanu | **`in-di-cheghja-nu`** | 4.5 | `cheghja` |
+| accupavanu | **`accupava-nu`** | 4.5 | `accupava` |
+| granulita | **`granuli-ta`** | 4.5 | `granuli` |
+| principionu | **`pr-in-cipio-nu`** | 4.5 | `cipio` |
+| attaccani | **`attacca-ni`** | 4.5 | `attacca` |
+| supranatu | **`suprana-tu`** | 4.5 | `suprana` |
+| asciuvatu | **`asciuva-tu`** | 4.5 | `asciuva` |
 
 ### 6.6 Linguistic Interpretation
 
 > **Automated Insight:**
-The language CO appears to be more isolating or has a highly fixed vocabulary. Word-level models perform nearly as well as subword models, indicating fewer productive morphological processes.
+The language Corsican shows high morphological productivity. The subword models are significantly more efficient than word models, suggesting a rich system of affixation or compounding.
 
 ---
 ## 7. Summary & Recommendations
@@ -519,8 +552,8 @@ The language CO appears to be more isolating or has a highly fixed vocabulary. W
 
 | Component | Recommended | Rationale |
 |-----------|-------------|-----------|
-| Tokenizer | **64k BPE** | Best compression (4.20x) |
-| N-gram | **2-gram** | Lowest perplexity (221) |
+| Tokenizer | **64k BPE** | Best compression (4.22x) |
+| N-gram | **2-gram** | Lowest perplexity (220) |
 | Markov | **Context-4** | Highest predictability (93.8%) |
 | Embeddings | **100d** | Balanced semantic capture and isotropy |
 
@@ -735,4 +768,4 @@ MIT License - Free for academic and commercial use.
 ---
 *Generated by Wikilangs Models Pipeline*
 
-*Report Date: 2026-01-03 10:28:53*
+*Report Date: 2026-01-03 20:37:45*

models/embeddings/aligned/co_128d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b1188259a56d403865faa99fa1350b00926f10763c1237aa3e872cced7601be
+size 1056840398

models/embeddings/aligned/co_128d.meta.json

@@ -0,0 +1 @@
+{"lang": "co", "dim": 128, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/co_128d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea211f0fda570a4fac50984b5cec2e1159b1e3f2e86395f518eede63ccabc5da
+size 65664

models/embeddings/aligned/co_128d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "co",
+  "dimension": 128,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 7516,
+  "vocab_size": 31527
+}

models/embeddings/aligned/co_32d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95e90e81f99abdbfd12e0afed22a1557827ab877d1a3a61a584eb491afd1abeb
+size 264627662

models/embeddings/aligned/co_32d.meta.json

@@ -0,0 +1 @@
+{"lang": "co", "dim": 32, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/co_32d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a26f04f573d7a17eb4b3498fa4df506e36707149738f9ed7355a9a36313b5cb9
+size 4224

models/embeddings/aligned/co_32d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "co",
+  "dimension": 32,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 7516,
+  "vocab_size": 31527
+}

models/embeddings/aligned/co_64d.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8aa3c06a1b412d20ef8dcbd2588899da958ae3efc40cc8b841a5e601b9f6e95
+size 528698574

models/embeddings/aligned/co_64d.meta.json

@@ -0,0 +1 @@
+{"lang": "co", "dim": 64, "max_seq_len": 512, "is_aligned": true}

models/embeddings/aligned/co_64d.projection.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1118cba50c88a05a3169c9b628b3c139e380b09637cfed076858f8d2a531505
+size 16512

models/embeddings/aligned/co_64d_metadata.json

@@ -0,0 +1,8 @@
+{
+  "language": "co",
+  "dimension": 64,
+  "version": "aligned",
+  "hub_language": "en",
+  "seed_vocab_size": 7516,
+  "vocab_size": 31527
+}

models/embeddings/monolingual/co_128d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96cf2c345e68270f8dcdbe99946da8e1091f69fe819c6b9a2158c5e48ed9e52c
-size 1056877904
+oid sha256:7b1188259a56d403865faa99fa1350b00926f10763c1237aa3e872cced7601be
+size 1056840398

models/embeddings/monolingual/co_128d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 128
   },
-  "vocab_size": 31563
+  "vocab_size": 31527
 }

models/embeddings/monolingual/co_32d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1067ad298ad58d7164db3f4db9081298ba2b534ee2da4cd72c1b7dd84f187b81
-size 264637520
+oid sha256:95e90e81f99abdbfd12e0afed22a1557827ab877d1a3a61a584eb491afd1abeb
+size 264627662

models/embeddings/monolingual/co_32d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 32
   },
-  "vocab_size": 31563
+  "vocab_size": 31527
 }

models/embeddings/monolingual/co_64d.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c664f3164ab6a4eac104c4ea7ff6212ebaa4e5e194ba8f7a21e25ea1c0d4871a
-size 528717648
+oid sha256:e8aa3c06a1b412d20ef8dcbd2588899da958ae3efc40cc8b841a5e601b9f6e95
+size 528698574

models/embeddings/monolingual/co_64d_metadata.json

@@ -11,5 +11,5 @@
     "encoding_method": "rope",
     "dim": 64
   },
-  "vocab_size": 31563
+  "vocab_size": 31527
 }

models/subword_markov/co_markov_ctx1_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8b5e0c398d4f5b2969e1db1c671105a3ac5242a1d374d84d5b0069106642b3d
-size 65281
+oid sha256:7ccba47788fdac20dbfe0dc14ede2e235d186ad963d505287d92784b261dc76f
+size 71545

models/subword_markov/co_markov_ctx1_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "subword",
   "language": "co",
-  "unique_contexts": 976,
-  "total_transitions": 13219556
+  "unique_contexts": 1238,
+  "total_transitions": 13211308
 }

models/subword_markov/co_markov_ctx2_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10d07b4e8c3757cc6b86c68087fcfea9af405d88517a6a0445f37de943be6b99
-size 363878
+oid sha256:755dec4f665ee36033a886b40d0259d4fa7d3ef9998d73c8e21b5586257d5116
+size 382643

models/subword_markov/co_markov_ctx2_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "subword",
   "language": "co",
-  "unique_contexts": 8204,
-  "total_transitions": 13211059
+  "unique_contexts": 8617,
+  "total_transitions": 13202755
 }

models/subword_markov/co_markov_ctx3_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab0ac62add6f840e66c319753e1b42d3784d61773053b38c73b6b277df4cca3a
-size 1441137
+oid sha256:627f6479b485a5eb7d1a5dba74948cad853724fa12e413e40fa722f8a3038f39
+size 1491620

models/subword_markov/co_markov_ctx3_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 3,
   "variant": "subword",
   "language": "co",
-  "unique_contexts": 46007,
-  "total_transitions": 13202562
+  "unique_contexts": 46221,
+  "total_transitions": 13194202
 }

models/subword_markov/co_markov_ctx4_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07c83362fcdbe265c5308c85c508b78deedca4f3c3c1ec1a96a44764934510ff
-size 4134190
+oid sha256:7e5d1a220a544ac400dd2df144119c1304ce71a19475f2ddcbd8ee7e0030575b
+size 4190273

models/subword_markov/co_markov_ctx4_subword_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 4,
   "variant": "subword",
   "language": "co",
-  "unique_contexts": 183668,
-  "total_transitions": 13194065
+  "unique_contexts": 182986,
+  "total_transitions": 13185649
 }

models/subword_ngram/co_2gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:899bc5e6584570580099e942da78aa713e4d3807f6ebea5a4ffd604efeea23f2
-size 42648
+oid sha256:ae24b051ed3af54f3565060e3552cd67a8d903e34ab258a4828b67ff9996b475
+size 42374

models/subword_ngram/co_2gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 2,
   "variant": "subword",
   "language": "co",
-  "unique_ngrams": 3181,
-  "total_ngrams": 13219556
+  "unique_ngrams": 3170,
+  "total_ngrams": 13211308
 }

models/subword_ngram/co_3gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7ed57d99baac180f6c586ad2f06d308b877fede514712ded41ed5e8989084810
-size 285900
+oid sha256:76c849c163a5eb7e124f5b49aa5b539517b8c3d1e08b885e4b6de423a7f6d6da
+size 283370

models/subword_ngram/co_3gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 3,
   "variant": "subword",
   "language": "co",
-  "unique_ngrams": 22404,
-  "total_ngrams": 13211059
+  "unique_ngrams": 22203,
+  "total_ngrams": 13202755
 }

models/subword_ngram/co_4gram_subword.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:094c924c5f844afd2646ce2d24dd1cfc9060e5723e944365817cf9c12106c248
-size 1233900
+oid sha256:2a3030923f89095c97a82d10b87ecd761efc7224def9c96d443f295f4f8d004a
+size 1229873

models/subword_ngram/co_4gram_subword_metadata.json

@@ -2,6 +2,6 @@
   "n": 4,
   "variant": "subword",
   "language": "co",
-  "unique_ngrams": 107042,
-  "total_ngrams": 13202562
+  "unique_ngrams": 106299,
+  "total_ngrams": 13194202
 }

models/subword_ngram/co_5gram_subword.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6439a330150f404c738100e1929f3dda0f560c148608e1c019d12a7520a2e4d0
+size 3273783

models/subword_ngram/co_5gram_subword_metadata.json

@@ -0,0 +1,7 @@
+{
+  "n": 5,
+  "variant": "subword",
+  "language": "co",
+  "unique_ngrams": 280787,
+  "total_ngrams": 13185649
+}

models/tokenizer/co_tokenizer_16k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b34f2ba9c5b244ddd93aba34ada7fef4711b8397b21d33fd688d1c317cc7cb41
-size 516047
+oid sha256:24b3fad0009298815c654db3ae8bb403f55f17281379bf6ee659a34f6c654a85
+size 515937

models/tokenizer/co_tokenizer_32k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6ae7a171bd1aba13305f06c6f0f2b827daf529b4e36eed333e8c0c56302860c
-size 803640
+oid sha256:ed439f585e2b42bfac5587d734a4dfc78dd34d7f9a2c031908d5b05136065565
+size 803263

models/tokenizer/co_tokenizer_64k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17214768ca1dd2e6a0c00363e0983130aeccf918e9ce567d882f1537de050491
-size 1383670
+oid sha256:e6c942ef7778bdf767b712f27ad74e9b074e60f2430bd878560984dffdd15fa3
+size 1384129

models/tokenizer/co_tokenizer_8k.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73dc30b9584b2f1bd0dec231fe033922d1286e98bf5022421f69b9b07cc653e2
-size 376505
+oid sha256:112936f030ea6a09d6e9e276c543e39a95b187e86bc60ea07bb2164cbddd0cbe
+size 376417

models/vocabulary/co_vocabulary.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf8b269f53cdfcf4bafb12ed650edcd581ad571e89880ac7222ad9b23086308c
-size 1013491
+oid sha256:508286f99a8790038e977d4e50f4780f9c94657fe53c2de7efb577d672952eee
+size 1021999

models/vocabulary/co_vocabulary_metadata.json

@@ -1,17 +1,17 @@
 {
   "language": "co",
-  "vocabulary_size": 58612,
+  "vocabulary_size": 58569,
   "variant": "full",
   "statistics": {
-    "type_token_ratio": 0.05464431503278916,
+    "type_token_ratio": 0.0546978960200255,
     "coverage": {
-      "top_100": 0.4740403851170396,
-      "top_1000": 0.6748029483890198,
-      "top_5000": 0.8154347093397393,
-      "top_10000": 0.8683801088705445
+      "top_100": 0.4749617033193706,
+      "top_1000": 0.6754865351219446,
+      "top_5000": 0.8156472903409058,
+      "top_10000": 0.8684880687616513
     },
-    "hapax_count": 64770,
-    "hapax_ratio": 0.5249550177497528,
-    "total_documents": 8497
+    "hapax_count": 64869,
+    "hapax_ratio": 0.5255188839741408,
+    "total_documents": 8553
   }
 }

models/word_markov/co_markov_ctx1_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb1cae5d79a7a73adf2ff0919afeebd9f38f7943e4b16f5e68a2a24aa799f516
-size 5678726
+oid sha256:5150ee1468d1b94738c58ba68b8c04fd75f386fed28e6125e3e6599c4b3f12b1
+size 5644828

models/word_markov/co_markov_ctx1_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 1,
   "variant": "word",
   "language": "co",
-  "unique_contexts": 123267,
-  "total_transitions": 2249414
+  "unique_contexts": 123322,
+  "total_transitions": 2248170
 }

models/word_markov/co_markov_ctx2_word.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:793e28fa70d170a16ec64307b53718d88a3f178234de84df35691f3c269d0ab2
-size 14184219
+oid sha256:9eaea00576501f682e63b2bbe8e53bc759e0df3e510b85995cbbd17505b9aa10
+size 14156294

models/word_markov/co_markov_ctx2_word_metadata.json

@@ -2,6 +2,6 @@
   "context_size": 2,
   "variant": "word",
   "language": "co",
-  "unique_contexts": 688381,
-  "total_transitions": 2240917
+  "unique_contexts": 686898,
+  "total_transitions": 2239617
 }