p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] crates/cli/Cargo.toml (plaintext)
[package]
name = "crowd-pilot-serialize"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "CLI tool for serializing crowd-pilot's IDE interaction data"

[[bin]]
name = "crowd-pilot-serialize"
path = "src/main.rs"

[dependencies]
crowd-pilot-serializer-core = { path = "../core" }
clap = { version = "4.5", features = ["derive"] }
tokenizers = { version = "0.21", features = ["http"] }
serde_json = { workspace = true }
serde = { workspace = true }
minijinja = "2"
ureq = "2"
[content] crates/cli/src/main.rs (rust)
    // Report chat template status
    if tokenizer.supports_chat_template() {
        println!("  Chat template loaded - using accurate conversation token counting");
    } else {
        println!("  Warning: No chat template found - using approximate token counting");
    }

    // Determine system prompt early so it can be included in token counting
    let default_prompt = default_system_prompt(args.viewport_radius);
    let system_prompt = args.system_prompt.clone().unwrap_or(default_prompt.clone());
[content] crates/cli/src/main.rs (rust)
        // Use offsets to slice original string precisely
        let offsets = encoding.get_offsets();
        if let Some((_, end)) = offsets.get(max_tokens.saturating_sub(1)) {
            text[..*end].to_string()
        } else {
            // Fallback to decode method
            let truncated_ids: Vec<u32> = ids[..max_tokens].to_vec();
            self.inner
                .decode(&truncated_ids, true)
                .expect("Failed to decode truncated tokens")
        }
    }

    fn count_conversation_tokens(&self, messages: &[ChatMessage]) -> usize {
        if let Some(rendered) = self.apply_chat_template(messages) {
            // Tokenize with add_special_tokens=false since template already includes them
            self.inner
                .encode(rendered.as_str(), false)
                .expect("Failed to encode rendered chat template")
                .get_ids()
                .len()
        } else {
            // Fallback: sum individual message tokens
            messages.iter().map(|m| self.count_tokens(&m.content)).sum()
        }
    }

    fn supports_chat_template(&self) -> bool {
        self.chat_template.is_some()
[content] crates/cli/src/main.rs (rust)
/// Configuration loaded from tokenizer_config.json
#[derive(Debug, Deserialize, Default)]
struct TokenizerConfig {
    chat_template: Option<String>,
    bos_token: Option<StringOrObject>,
    eos_token: Option<StringOrObject>,
    unk_token: Option<StringOrObject>,
    pad_token: Option<StringOrObject>,
}

/// Token can be either a string or an object with a "content" field
#[derive(Debug, Deserialize, Clone)]
#[serde(untagged)]
enum StringOrObject {
    String(String),
    Object { content: String },
}

impl StringOrObject {
    fn as_str(&self) -> &str {
        match self {
            StringOrObject::String(s) => s,
            StringOrObject::Object { content } => content,
        }
    }
}
[content] crates/cli/src/main.rs (rust)
/// Add Python-compatible string methods to a minijinja Environment.
/// These are added as filters that work with method-call syntax via template preprocessing.
fn add_python_string_methods(env: &mut Environment<'_>) {
    // startswith filter
    env.add_filter("startswith", |s: &str, prefix: &str| -> bool {
        s.starts_with(prefix)
    });
    
    // endswith filter
    env.add_filter("endswith", |s: &str, suffix: &str| -> bool {
        s.ends_with(suffix)
    });
    
    // strip filter (with optional chars argument)
    env.add_filter("strip", |s: &str, chars: Option<&str>| -> String {
        match chars {
            Some(c) => s.trim_matches(|ch: char| c.contains(ch)).to_string(),
            None => s.trim().to_string(),
        }
    });
    
    // lstrip filter
    env.add_filter("lstrip", |s: &str, chars: Option<&str>| -> String {
        match chars {
            Some(c) => s.trim_start_matches(|ch: char| c.contains(ch)).to_string(),
            None => s.trim_start().to_string(),
        }
    });
    
    // rstrip filter
    env.add_filter("rstrip", |s: &str, chars: Option<&str>| -> String {
        match chars {
            Some(c) => s.trim_end_matches(|ch: char| c.contains(ch)).to_string(),
            None => s.trim_end().to_string(),
        }
    });
    
    // raise_exception function that some templates use
    env.add_function("raise_exception", |msg: String| -> Result<String, minijinja::Error> {
        Err(minijinja::Error::new(
            minijinja::ErrorKind::InvalidOperation,
            msg,
        ))
    });
}

/// Preprocess a Jinja2 template to convert Python method syntax to minijinja filter syntax.
/// E.g., `s.startswith('x')` -> `s is startswith('x')`
///       `s.strip('
')` -> `s | strip('
')`
fn preprocess_chat_template(template: &str) -> String {
    use regex::Regex;
    
    let mut result = template.to_string();
    
    // Convert .startswith('...') and .endswith('...') to test syntax: `is startswith('...')`
    // These return booleans so we use `is` test syntax
    let startswith_re = Regex::new(r"\.startswith\(([^)]+)\)").unwrap();
    result = startswith_re.replace_all(&result, " is startswith($1)").to_string();
    
    let endswith_re = Regex::new(r"\.endswith\(([^)]+)\)").unwrap();
    result = endswith_re.replace_all(&result, " is endswith($1)").to_string();
    
    // Convert .strip(), .lstrip(), .rstrip() to filter syntax: `| strip(...)`
    // These return strings so we use filter syntax
    let strip_re = Regex::new(r"\.strip\(([^)]*)\)").unwrap();
    result = strip_re.replace_all(&result, " | strip($1)").to_string();
    
    let lstrip_re = Regex::new(r"\.lstrip\(([^)]*)\)").unwrap();
    result = lstrip_re.replace_all(&result, " | lstrip($1)").to_string();
    
    let rstrip_re = Regex::new(r"\.rstrip\(([^)]*)\)").unwrap();
    result = rstrip_re.replace_all(&result, " | rstrip($1)").to_string();
    
    result
}
[content] crates/cli/src/main.rs (rust)
/// Chat template format with known token overhead values.
/// 
/// Different models use different chat templates with different special tokens.
/// This enum defines the per-message token overhead for accurate counting.
#[derive(Debug, Clone, Copy, ValueEnum, Default)]
pub enum ChatTemplateFormat {
    /// ChatML format (Qwen, many others): 
role
content


    /// Overhead per message: ~4 tokens (im_start + role + im_end + newlines)
    #[default]
    Chatml,
    
    /// Llama 3 format: 
role


content

    /// Overhead per message: ~5 tokens
    Llama3,
    
    /// Mistral format: 
 content 

    /// Overhead per message: ~4 tokens
    Mistral,
    
    /// No chat template - just count raw content tokens
    None,
}

impl ChatTemplateFormat {
    /// Get the token overhead per message for this template format.
    pub fn tokens_per_message(&self) -> usize {
        match self {
            ChatTemplateFormat::Chatml => 4,   // 
 + role + 
 + 

            ChatTemplateFormat::Llama3 => 5,   // 
 + role + 
 + 

 + 

            ChatTemplateFormat::Mistral => 4,  // 
 + 
 or similar
            ChatTemplateFormat::None => 0,
        }
    }
    
    /// Get any fixed overhead tokens at the start of the conversation.
    pub fn conversation_start_overhead(&self) -> usize {
        match self {
            ChatTemplateFormat::Chatml => 0,
            ChatTemplateFormat::Llama3 => 1,   // 

            ChatTemplateFormat::Mistral => 1,  // 

            ChatTemplateFormat::None => 0,
        }
    }
}

impl FromStr for ChatTemplateFormat {
    type Err = String;
    
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s.to_lowercase().as_str() {
            "chatml" | "qwen" | "qwen2" => Ok(ChatTemplateFormat::Chatml),
            "llama3" | "llama-3" => Ok(ChatTemplateFormat::Llama3),
            "mistral" => Ok(ChatTemplateFormat::Mistral),
            "none" => Ok(ChatTemplateFormat::None),
            _ => Err(format!("Unknown chat template format: {}", s)),
        }
    }
}
[content] crates/cli/src/main.rs (rust)
    let tokenizer = RustTokenizer::load(&args.tokenizer, args.chat_template)?;

    // Report chat template format
    println!("  Chat template format: {:?}", args.chat_template);
    println!("  Per-message overhead: {} tokens", args.chat_template.tokens_per_message());

    // Determine system prompt early so it can be included in token counting
    let default_prompt = default_system_prompt(args.viewport_radius);
    let system_prompt = args.system_prompt.clone().unwrap_or(default_prompt.clone());
[content] crates/cli/src/main.rs (rust)
        // Use offsets to slice original string precisely
        let offsets = encoding.get_offsets();
        if let Some((_, end)) = offsets.get(max_tokens.saturating_sub(1)) {
            text[..*end].to_string()
        } else {
            // Fallback to decode method
            let truncated_ids: Vec<u32> = ids[..max_tokens].to_vec();
            self.inner
                .decode(&truncated_ids, true)
                .expect("Failed to decode truncated tokens")
        }
    }

    fn count_conversation_tokens(&self, messages: &[ChatMessage]) -> usize {
        // Simple overhead-based approach:
        // total = conversation_start_overhead 
        //       + sum(content_tokens) 
        //       + (num_messages * per_message_overhead)
        
        let content_tokens: usize = messages.iter()
            .map(|m| self.count_tokens(&m.content))
            .sum();
        
        let per_message_overhead = self.chat_template_format.tokens_per_message();
        let message_overhead = messages.len() * per_message_overhead;
        
        let start_overhead = self.chat_template_format.conversation_start_overhead();
        
        start_overhead + content_tokens + message_overhead
    }

    fn supports_chat_template(&self) -> bool {
        !matches!(self.chat_template_format, ChatTemplateFormat::None)
[content] crates/cli/src/main.rs (rust)
    /// Chat template format for accurate token counting.
    /// Options: chatml (default, for Qwen), llama3, mistral, none
    #[arg(long, default_value = "chatml")]
    chat_template: ChatTemplateFormat,
[content] crates/cli/src/main.rs (rust)
            "qwen3" | "qwen-3" | "qwen" => Ok(ChatTemplateFormat::Qwen3),
            "glm45" | "glm-4.5" | "glm4.5" | "glm" => Ok(ChatTemplateFormat::Glm45),
            _ => Err(format!("Unknown chat template format: {}. Supported: qwen3, glm45", s)),
[content] crates/cli/src/main.rs (rust)
        // Use offsets to slice original string precisely
        let offsets = encoding.get_offsets();
        if let Some((_, end)) = offsets.get(max_tokens.saturating_sub(1)) {
            text[..*end].to_string()
        } else {
            // Fallback to decode method
            let truncated_ids: Vec<u32> = ids[..max_tokens].to_vec();
            self.inner
                .decode(&truncated_ids, true)
                .expect("Failed to decode truncated tokens")
        }
    }

    fn count_conversation_tokens(&self, messages: &[ChatMessage]) -> usize {
        // Simple overhead-based approach:
        // total = conversation_start_overhead 
        //       + sum(content_tokens) 
        //       + (num_messages * per_message_overhead)
        
        let content_tokens: usize = messages.iter()
            .map(|m| self.count_tokens(&m.content))
            .sum();
        
        let per_message_overhead = self.chat_template_format.tokens_per_message();
        let message_overhead = messages.len() * per_message_overhead;
        
        let start_overhead = self.chat_template_format.conversation_start_overhead();
        
        start_overhead + content_tokens + message_overhead
    }

    fn supports_chat_template(&self) -> bool {
        true  // All supported formats have chat template token counting
[content] crates/cli/src/main.rs (rust)
    fn count_conversation_tokens(&self, messages: &[ChatMessage]) -> usize {
        // Simple overhead-based approach:
        // total = conversation_start_overhead 
        //       + sum(content_tokens) 
        //       + (num_messages * per_message_overhead)
        
        let content_tokens: usize = messages.iter()
            .map(|m| self.count_tokens(&m.content))
            .sum();
        
        let per_message_overhead = self.chat_template_format.tokens_per_message();
        let message_overhead = messages.len() * per_message_overhead;
        
        let start_overhead = self.chat_template_format.conversation_start_overhead();
        
        start_overhead + content_tokens + message_overhead
    }

    fn supports_chat_template(&self) -> bool {
        true  // All supported formats have chat template token counting
    }
}
[content] crates/cli/src/main.rs (rust)
        // Use offsets to slice original string precisely
        let offsets = encoding.get_offsets();
        if let Some((_, end)) = offsets.get(max_tokens.saturating_sub(1)) {
            text[..*end].to_string()
        } else {
            // Fallback to decode method
            let truncated_ids: Vec<u32> = ids[..max_tokens].to_vec();
            self.inner
                .decode(&truncated_ids, true)
                .expect("Failed to decode truncated tokens")
        }
    }
[content] crates/cli/src/main.rs (rust)
        // Use offsets to slice original string precisely
        let offsets = encoding.get_offsets();
        let (_, end) = offsets[max_tokens - 1];
        text[..end].to_string()
    }

    fn count_conversation_tokens(&self, messages: &[ChatMessage]) -> usize {
        // Simple overhead-based approach:
        // total = conversation_start_overhead 
        //       + sum(content_tokens) 
        //       + (num_messages * per_message_overhead)
        
        let content_tokens: usize = messages.iter()
            .map(|m| self.count_tokens(&m.content))
            .sum();
        
        let per_message_overhead = self.chat_template_format.tokens_per_message();
        let message_overhead = messages.len() * per_message_overhead;
        
        let start_overhead = self.chat_template_format.conversation_start_overhead();
        
        start_overhead + content_tokens + message_overhead
    }

    fn supports_chat_template(&self) -> bool {
        true  // All supported formats have chat template token counting
[content] crates/core/src/conversation.rs (rust)
            if self.current_tokens + tokens > max_tokens && !self.messages.is_empty() {
                self.finalize_current_conversation();
                // After starting a new conversation, we need to re-capture file states
                // This will happen naturally as files are accessed
            }
        }

        self.messages.push(message);
        self.current_tokens += tokens;
[content] crates/cli/src/main.rs (rust)
/// Special token counts for known chat templates: (per_message, conversation_start)
fn chat_template_overhead(name: &str) -> Option<(usize, usize)> {
    match name.to_lowercase().as_str() {
        // Qwen3: 
role
content
 per message
        "qwen3" => Some((2, 0)),
        // GLM-4.5: [gMASK]<sop> at start, then 
 per message
        "glm45" => Some((1, 2)),
        _ => None,
[content] crates/cli/src/main.rs (rust)
/// Chat template format with known token overhead values.
/// 
/// Different models use different chat templates with different special tokens.
/// This enum defines the per-message token overhead for accurate counting.
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum ChatTemplateFormat {
    /// Qwen3 format: 
role
content

    /// Special tokens per message: 2 (
 + 
)
    Qwen3,
    
    /// GLM-4.5 format: [gMASK]<sop>
content...
    /// Special tokens per message: 1 (role token)
    /// Conversation start: 2 ([gMASK] + <sop>)
    Glm45,
}

impl ChatTemplateFormat {
    /// Get the special token overhead per message for this template format.
    pub fn tokens_per_message(&self) -> usize {
        match self {
            ChatTemplateFormat::Qwen3 => 2,  // 
 + 

            ChatTemplateFormat::Glm45 => 1,  // 
 or 
 or 

        }
    }
    
    /// Get any fixed overhead tokens at the start of the conversation.
    pub fn conversation_start_overhead(&self) -> usize {
        match self {
            ChatTemplateFormat::Qwen3 => 0,
            ChatTemplateFormat::Glm45 => 2,  // [gMASK] + <sop>
        }
    }
}

impl FromStr for ChatTemplateFormat {
    type Err = String;
    
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        match s.to_lowercase().as_str() {
            "qwen3" => Ok(ChatTemplateFormat::Qwen3),
            "glm45" => Ok(ChatTemplateFormat::Glm45),
            _ => Err(format!("Unknown chat template format: {}. Supported: qwen3, glm45", s)),
        }
[content] crates/cli/src/main.rs (rust)
    println!("  Chat template: {}", args.chat_template);
    println!("  Per-message overhead: {} tokens", special_tokens_per_message);
    println!("  Conversation start overhead: {} tokens", conversation_start_tokens);
[content] crates/cli/src/main.rs (rust)
    let (special_tokens_per_message, conversation_start_tokens) =
        chat_template_overhead(&args.chat_template).ok_or_else(|| {
            format!(
                "Unknown chat template: '{}'. Supported: qwen3, glm45",
                args.chat_template
            )
        })?;
[tab] /fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/metadata.json (json)
{
  "config": {
    "coalesce_radius": 5,
    "csv_root": "/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/",
    "max_tokens_per_conversation": 4096,
    "max_tokens_per_message": 2048,
    "min_conversation_messages": 5,
    "output_dir": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/",
    "tokenizer": "Qwen/Qwen3-8B",
    "val_ratio": 0.1,
    "viewport_radius": 10
  },
  "counts": {
    "total_conversations": 10609,
    "total_sessions": 349,
    "train_conversations": 9652,
    "val_conversations": 957
  },
  "files": {
    "train_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/training.jsonl",
    "val_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/validation.jsonl"
  },
  "stats": {
    "avg_messages_per_conversation": 12.384579130926571,
    "avg_tokens_per_conversation": 3527.8928268451314,
    "total_messages": 131388,
    "total_tokens": 37427415
  }
}
[tab] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_8k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_8k_tokens/" \
--max-tokens-per-conversation 8192 \
--tokenizer="Qwen/Qwen3-8B"
[content] test.py (python)
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("/fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-4B", trust_remote_code=True)

# Empty content to isolate overhead
for role in ["system", "user", "assistant"]:
    msgs = [{"role": role, "content": ""}]
    print(f"{role}: {len(tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False))} tokens")

# Check incremental cost
msgs = [{"role": "system", "content": ""}, {"role": "user", "content": ""}, {"role": "assistant", "content": ""}]
print(f"3 empty messages: {len(tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False))} tokens")
[content] test.py (python)
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("/fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-4B", trust_remote_code=True)

# Single messages
for role in ["system", "user", "assistant"]:
    msgs = [{"role": role, "content": ""}]
    token_ids = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
    decoded = tokenizer.decode(token_ids)
    print(f"
{role}: {len(token_ids)} tokens")
    print(f"Decoded: {repr(decoded)}")

# Multiple messages
msgs = [{"role": "system", "content": ""}, {"role": "user", "content": ""}, {"role": "assistant", "content": ""}]
token_ids = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
decoded = tokenizer.decode(token_ids)
print(f"
3 empty messages: {len(token_ids)} tokens")
print(f"Decoded: {repr(decoded)}")
[content] test.py (python)
# Test with actual content
msgs = [
    {"role": "system", "content": ""}, 
    {"role": "user", "content": "test"}, 
    {"role": "assistant", "content": ""}
]
token_ids = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
decoded = tokenizer.decode(token_ids)
print(f"
With user content: {len(token_ids)} tokens")
print(f"Decoded: {repr(decoded)}")
[content] test.py (python)
# Test with multiple turns
msgs = [
    {"role": "system", "content": ""},
    {"role": "user", "content": "hello"},
    {"role": "assistant", "content": "hi"},
    {"role": "user", "content": "how are you"},
    {"role": "assistant", "content": "good"},
]
token_ids = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
decoded = tokenizer.decode(token_ids)
print(f"
5 messages: {len(token_ids)} tokens")
print(f"Decoded: {repr(decoded)}")

# Count by role
system_count = sum(1 for m in msgs if m["role"] == "system")
user_count = sum(1 for m in msgs if m["role"] == "user")
assistant_count = sum(1 for m in msgs if m["role"] == "assistant")
expected = system_count * 5 + user_count * 5 + assistant_count * 9
print(f"Expected: {expected} (system: {system_count}×5 + user: {user_count}×5 + assistant: {assistant_count}×9)")
[content] test.py (python)
Single messages
for role in ["system", "user", "assistant"]:
    msgs = [{"role": role, "content": ""}]
    token_ids = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
    decoded = tokenizer.decode(token_ids)
    print(f"
{role}: {len(token_ids)} tokens")
    print(f"Decoded: {repr(decoded)}")

# Multiple messages
msgs = [{"role": "system", "content": ""}, {"role": "user", "content": ""}, {"role": "assistant", "content": ""}]
token_ids = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
decoded = tokenizer.decode(token_ids)
print(f"
3 empty messages: {len(token_ids)} tokens")
print(f"Decoded: {repr(decoded)}")

# Test with actual content
msgs = [
    {"role": "system", "content": ""}, 
    {"role": "user", "content": "test"}, 
    {"role": "assistant", "content": ""}
]
token_ids = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
decoded = tokenizer.decode(token_ids)
print(f"
With user content: {len(token_ids)} tokens")
print(f"Decoded: {repr(decoded)}")

# Test with multiple turns
msgs = [
    {"role": "system", "content": ""},
    {"role": "user", "content": "hello"},
    {"role": "assistant", "content": "hi"},
    {"role": "user", "content": "how are you"},
    {"role": "assistant", "content": "good"},
]
token_ids = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
decoded = tokenizer.decode(token_ids)
print(f"
5 messages: {len(token_ids)} tokens")
print(f"Decoded: {repr(decoded)}")

#
[content] test.py (python)
token_ids = tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
decoded = tokenizer.decode(token_ids)
print(f"
5 messages: {len(token_ids)} tokens")
print(f"Decoded: {repr(decoded)}")

# Count by role
[content] test.py (python)

print("
" + "=" * 60)
print("Testing GLM-4.5")
print("=" * 60)
glm_tokenizer = AutoTokenizer.from_pretrained("/fast/project/HFMI_SynergyUnit/tab_model/huggingface/GLM-4.5-9B", trust_remote_code=True)

# Single messages
for role in ["system", "user", "assistant"]:
    msgs = [{"role": role, "content": ""}]
    token_ids = glm_tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
    decoded = glm_tokenizer.decode(token_ids)
    print(f"
{role}: {len(token_ids)} tokens")
    print(f"Decoded: {repr(decoded)}")

# Multiple messages
msgs = [
    {"role": "system", "content": ""},
    {"role": "user", "content": "hello"},
    {"role": "assistant", "content": "hi"},
    {"role": "user", "content": "how are you"},
    {"role": "assistant", "content": "good"},
]
token_ids = glm_tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
decoded = glm_tokenizer.decode(token_ids)
print(f"
5 messages: {len(token_ids)} tokens")
print(f"Decoded: {repr(decoded)}")

# Count by role (GLM-4.5: 2 start + 1 per message)
system_count = sum(1 for m in msgs if m["role"] == "system")
user_count = sum(1 for m in msgs if m["role"] == "user")
assistant_count = sum(1 for m in msgs if m["role"] == "assistant")
total_messages = len(msgs)
expected = 2 + total_messages  # 2 start tokens + 1 per message
print(f"Expected: {expected} (2 start + {total_messages} messages × 1 = {2 + total_messages})")
[content] test.py (python)
token_ids = qwen_tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
decoded = qwen_tokenizer.decode(token_ids)
print(f"
5 messages: {len(token_ids)} tokens")
print(f"Decoded: {repr(decoded)}")
[content] test.py (python)
print("=" * 60)
print("Testing Qwen3")
print("=" * 60)
qwen_tokenizer = AutoTokenizer.from_pretrained("/fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-4B", trust_remote_code=True)

# Single messages
for role in ["system", "user", "assistant"]:
    msgs = [{"role": role, "content": ""}]
    token_ids = qwen_tokenizer.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False)
    decoded = qwen_tokenizer.decode(token_ids)
    print(f"
{role}: {len(token_ids)} tokens")
    print(f"Decoded: {repr(decoded)}")

# Multiple messages
[content] test_token_count.py (python)
import json
from transformers import AutoTokenizer
import numpy as np

model_path = "/fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-4B"
val_path = "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/validation.jsonl"

print(f"Loading tokenizer from {model_path}...")
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)

lengths = []
exceed_4096 = 0
exceed_4999 = 0

print(f"Reading validation file {val_path}...")
with open(val_path, 'r') as f:
    for line in f:
        data = json.loads(line)
        messages = data.get("messages")
        if messages:
            # Apply chat template
            token_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False)
            length = len(token_ids)
            lengths.append(length)
            if length > 4096:
                exceed_4096 += 1
            if length > 4999:
                exceed_4999 += 1

lengths = np.array(lengths)
print("
--- Statistics ---")
print(f"Total samples: {len(lengths)}")
print(f"Mean length: {np.mean(lengths):.2f}")
print(f"Median length: {np.median(lengths)}")
print(f"Max length: {np.max(lengths)}")
print(f"Min length: {np.min(lengths)}")
print(f"Samples > 4096: {exceed_4096} ({100 * exceed_4096 / len(lengths):.2f}%)")
print(f"Samples > 4999: {exceed_4999} ({100 * exceed_4999 / len(lengths):.2f}%)")

# Percentiles
for p in [10, 25, 50, 75, 90, 95, 99]:
    print(f"{p}th percentile: {np.percentile(lengths, p)}")
[tab] /fast/project/HFMI_SynergyUnit/tab_model/data/glm/miles_hf_part_jsonl_4k_tokens/metadata.json (json)
{
  "config": {
    "coalesce_radius": 5,
    "csv_root": "/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/",
    "max_tokens_per_conversation": 4096,
    "max_tokens_per_message": 2048,
    "min_conversation_messages": 5,
    "output_dir": "/fast/project/HFMI_SynergyUnit/tab_model/data/glm/miles_hf_part_jsonl_4k_tokens/",
    "tokenizer": "zai-org/GLM-4.5-Air",
    "val_ratio": 0.1,
    "viewport_radius": 10
  },
  "counts": {
    "total_conversations": 11830,
    "total_sessions": 349,
    "train_conversations": 10754,
    "val_conversations": 1076
  },
  "files": {
    "train_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/glm/miles_hf_part_jsonl_4k_tokens/training.jsonl",
    "val_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/glm/miles_hf_part_jsonl_4k_tokens/validation.jsonl"
  },
  "stats": {
    "avg_messages_per_conversation": 11.157734573119189,
    "avg_tokens_per_conversation": 3672.8300929839393,
    "total_messages": 131996,
    "total_tokens": 43449580
  }
}
[content] test.py (python)
from transformers import AutoTokenizer
glm = AutoTokenizer.from_pretrained("zai-org/GLM-4.5-Air", trust_remote_code=True)

# Test with known content lengths
msgs = [
    {"role": "system", "content": "X"},
    {"role": "user", "content": "X"},
    {"role": "assistant", "content": "X"},
]

# Count content tokens
content_tokens = sum(len(glm.encode(m["content"], add_special_tokens=False)) for m in msgs)
print(f"Content tokens: {content_tokens}")

# Count total with template
total = len(glm.apply_chat_template(msgs, tokenize=True, add_generation_prompt=False))
print(f"Total with template: {total}")

# Overhead
overhead = total - content_tokens
print(f"Overhead: {overhead}")

# Expected: 2 (start) + 2 (system) + 2 (user) + 4 (assistant) = 10
print(f"Expected overhead: 10 (2 start + 2 system + 2 user + 4 assistant)")
[content] /fast/project/HFMI_SynergyUnit/tab_model/data/glm/miles_hf_part_jsonl_4k_tokens/metadata.json (json)
    "total_conversations": 11970,
    "total_sessions": 349,
    "train_conversations": 10883,
    "val_conversations": 1087
  },
  "files": {
    "train_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/glm/miles_hf_part_jsonl_4k_tokens/training.jsonl",
    "val_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/glm/miles_hf_part_jsonl_4k_tokens/validation.jsonl"
  },
  "stats": {
    "avg_messages_per_conversation": 11.041102756892231,
    "avg_tokens_per_conversation": 3679.2641604010023,
    "total_messages": 132162,
    "total_tokens": 44040792
[content] crates/cli/src/main.rs (rust)
//! Miles SFT training. It uses the HuggingFace tokenizers Rust library for
//! accurate token counting.

use std::path::PathBuf;

use clap::Parser;
use tokenizers::Tokenizer as HfTokenizer;

use crowd_pilot_serializer_core::{
    default_system_prompt,
    pipeline::{PipelineConfig, PipelineResult},
    process_all_sessions, write_jsonl_output, Tokenizer,
};

/// Serialize crowd-pilot CSV sessions to Miles JSONL format.
[content] crates/napi/src/lib.rs (rust)
                // Extension doesn't need chat template counting
                system_prompt: None,
                special_tokens_per_user_message: 0,
                special_tokens_per_assistant_message: 0,
                conversation_start_tokens: 0,