p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] crates/cli/src/main.rs (rust)
        let encoding = self.inner
            .encode(text, false)
            .expect("Failed to encode text with tokenizer");
        
        let ids = encoding.get_ids();
        if ids.len() <= max_tokens {
            return text.to_string();
        }
        
        let truncated_ids: Vec<u32> = ids[..max_tokens].to_vec();
        self.inner
            .decode(&truncated_ids, true)
            .expect("Failed to decode truncated tokens")
[content] crates/cli/src/main.rs (rust)
        match self.inner.encode(text, false) {
            Ok(encoding) => {
                let ids = encoding.get_ids();
                if ids.len() <= max_tokens {
                    return text.to_string();
                }
                let truncated_ids: Vec<u32> = ids[..max_tokens].to_vec();
                self.inner
                    .decode(&truncated_ids, true)
                    .unwrap_or_else(|_| text.chars().take(max_tokens * 4).collect())
            }
            Err(_) => text.chars().take(max_tokens * 4).collect(),
        }
[tab] crates/cli/Cargo.toml (plaintext)
[package]
name = "crowd-pilot-serialize"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "CLI tool for serializing crowd-pilot's IDE interaction data"

[[bin]]
name = "crowd-pilot-serialize"
path = "src/main.rs"

[dependencies]
crowd-pilot-serializer-core = { path = "../core" }
clap = { version = "4.5", features = ["derive"] }
tokenizers = { version = "0.21", features = ["http"] }
serde_json = { workspace = true }