Model: khopilot/km-tokenizer-khmer
Author: khopilot
Pipeline: feature-extraction
Library: sentencepiece
Tags: sentencepiece, tokenizer, khmer, subword, text-generation, nlp, cambodia, southeast-asia, feature-extraction, km, dataset:khmer-corpus-648mb, license:apache-2.0, model-index, co2_eq_emissions, region:us
Card metadata: language: km
license: apache-2.0
tags: sentencepiece, tokenizer, khmer, subword, text-generation, nlp, cambodia, southeast-asia
library_name: sentencepiece
pipeline_tag: feature-extraction
widget: text: ព្រះរាជាណាចក្រកម្ពុជា
example_title: Kingdom of Cambodia, text: ការសិក្សាភាសាខ្មែរ
example_title: Khmer Language Education, text: អគ្គលេខាធិការគណៈកម្មាធិការជាតិអូឡាំពិកកម្ពុជា
example_title: NOCC Secretary General, text: លោក វ៉ាត់ ចំរើន
example_title: Mr. Vath Chamroeun, text: ការអំពាវនាវពលរដ្ឋកម្ពុជា
example_title: Appeal to Cambodian Citizens
datasets: khmer-corpus-648mb
metrics: accuracy, compression, efficiency
model-index: name: km-tokenizer-8k-production
results: task: type: text-tokenization
name: Text Tokenization
dataset: name: khmer-news-corpus
type: text
split: test
config: default
metrics: type: tokens_per_character
value: 0.144
name: Tokens Per Character (Overall)
verified: no, type: tokens_per_character_compounds
value: 0.087
name: Tokens Per Character (Compounds)
verified: no, type: tokens_per_character_real_text
value: 0.229
name: Tokens Per Character (Real News)
verified: no, type: compression_ratio
value: 6.94
name: Compression Ratio
verified: no, type: vocabulary_size
value: 8000
name: Vocabulary Size
verified: no, type: model_size_kb
value: 159.9
name: Model Size (KB)
verified: no, type: processing_speed_tokens_per_second
value: 425000
name: Processing Speed (Tokens/sec)
verified: no, task: type: linguistic-accuracy
name: Linguistic Accuracy Evaluation
dataset: name: khmer-linguistic-test-suite
type: structured
split: test
config: comprehensive
metrics: type: sanskrit_pali_accuracy
value: 100
name: Sanskrit/Pali Terms Accuracy (%)
verified: no, type: compound_words_accuracy
value: 100
name: Compound Words Accuracy (%)
verified: no, type: proper_names_accuracy
value: 100
name: Proper Names Accuracy (%)
verified: no, type: common_words_accuracy
value: 100
name: Common Words Accuracy (%)
verified: no, type: particles_accuracy
value: 100
name: Particles Accuracy (%)
verified: no, type: numbers_accuracy
value: 95
name: Numbers Accuracy (%)
verified: no, task: type: efficiency-benchmark
name: Efficiency vs Baseline
dataset: name: khmer-benchmark-texts
type: text
split: test
config: diverse
metrics: type: token_reduction_vs_char_level
value: 85.6
name: Token Reduction vs Character-level (%)
verified: no, type: token_reduction_vs_previous_model
value: 54.2
name: Token Reduction vs V6.5 (%)
verified: no, type: memory_footprint_mb
value: 0.16
name: Memory Footprint (MB)
verified: no, type: phd_evaluation_score
value: 76.1
name: PhD Evaluation Score (/100)
verified: no
co2_eq_emissions: emissions: 0.042
source: CodeCarbon
training_type: single-model
geographical_location: Cambodia
hardware_used: CPU-only
renewable_energy: yes
Likes: 3 | All-time downloads: 68
Created: 2025-08-21 05:46:56