[tab] crates/cli/src/main.rs (rust)
//! CLI tool for serializing crowd-pilot IDE interaction data.
//!
//! This tool processes CSV session files and outputs JSONL format suitable for
//! Miles SFT training. It uses the HuggingFace tokenizers Rust library for
//! accurate token counting.

use std::path::PathBuf;

use clap::{Parser, Subcommand};
use tokenizers::Tokenizer as HfTokenizer;

use crowd_pilot_serializer_core::{
    default_system_prompt,
    git_pull_analysis::{analyze_all_sessions, AnalysisResult},
    pipeline::{PipelineConfig, PipelineResult},
    process_all_sessions, write_jsonl_output, Tokenizer,
};

/// CLI tool for crowd-pilot IDE interaction data.
#[derive(Parser, Debug)]
#[command(name = "crowd-pilot-serialize")]
#[command(author, version, about, long_about = None)]
struct Cli {
    #[command(subcommand)]
    command: Commands,
}

#[derive(Subcommand, Debug)]
enum Commands {
    /// Serialize CSV sessions to Miles JSONL format
    Serialize(SerializeArgs),
    /// Analyze sessions for git pull data corruption issues
    AnalyzeGitPull(AnalyzeGitPullArgs),
}

/// Arguments for the serialize subcommand.
#[derive(Parser, Debug)]
struct SerializeArgs {
    /// Root directory containing CSV session files
    #[arg(long)]
    csv_root: PathBuf,

    /// Output directory for JSONL files
    #[arg(long)]
    output_dir: PathBuf,

    /// HuggingFace tokenizer model name or path
    #[arg(long)]
    tokenizer: String,

    /// Maximum tokens per conversation chunk
    #[arg(long, default_value = "8192")]
    max_tokens_per_conversation: usize,

    /// Maximum tokens per message
    #[arg(long, default_value = "2048")]
    max_tokens_per_message: usize,

    /// Minimum messages required to keep a conversation
    #[arg(long, default_value = "5")]
    min_conversation_messages: usize,

    /// Viewport radius (lines above/below cursor)
    #[arg(long, default_value = "10")]
    viewport_radius: usize,

    /// Coalesce radius for grouping nearby edits
    #[arg(long, default_value = "5")]
    coalesce_radius: usize,

    /// Fraction of sessions for validation (0.0-1.0)
    #[arg(long, default_value = "0.1")]
    val_ratio: f64,

    /// Custom system prompt (optional)
    #[arg(long)]
    system_prompt: Option<String>,
}

/// Arguments for the analyze-git-pull subcommand.
#[derive(Parser, Debug)]
struct AnalyzeGitPullArgs {
    /// Root directory containing CSV session files
    #[arg(long)]
    csv_root: PathBuf,

    /// Output JSON file for detailed results (optional)
    #[arg(long)]
    output: Option<PathBuf>,

    /// Show detailed per-session information
    #[arg(long, default_value = "false")]
    verbose: bool,
}

/// Wrapper around HuggingFace tokenizers for token counting and truncation.
///
/// This uses the Rust-native tokenizers library, which is `Send + Sync`
/// and enables true parallel tokenization without the Python GIL.
struct RustTokenizer {
    inner: HfTokenizer,
}

impl RustTokenizer {
    /// Load a HuggingFace tokenizer from a model name or path.
    fn load(model_name: &str) -> Result<Self, Box<dyn std::error::Error>> {
        let inner = HfTokenizer::from_pretrained(model_name, None)
            .map_err(|e| e as Box<dyn std::error::Error>)?;
        Ok(Self { inner })
    }
}

impl Tokenizer for RustTokenizer {
    fn count_tokens(&self, text: &str) -> usize {
        self.inner
            .encode(text, false)
            .expect("Failed to encode text with tokenizer")
            .get_ids()
            .len()
    }

    fn truncate_to_max_tokens(&self, text: &str, max_tokens: usize) -> String {
        let encoding = self.inner
            .encode(text, false)
            .expect("Failed to encode text with tokenizer");
        
        let ids = encoding.get_ids();
        if ids.len() <= max_tokens {
            return text.to_string();
        }
        
        let truncated_ids: Vec<u32> = ids[..max_tokens].to_vec();
        self.inner
            .decode(&truncated_ids, true)
            .expect("Failed to decode truncated tokens")
    }
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let cli = Cli::parse();

    match cli.command {
        Commands::Serialize(args) => run_serialize(args),
        Commands::AnalyzeGitPull(args) => run_analyze_git_pull(args),
    }
}

fn run_serialize(args: SerializeArgs) -> Result<(), Box<dyn std::error::Error>> {
    println!("Loading tokenizer from {}...", args.tokenizer);
    let tokenizer = RustTokenizer::load(&args.tokenizer)?;

    let config = PipelineConfig {
        max_tokens_per_conversation: args.max_tokens_per_conversation,
        max_tokens_per_message: args.max_tokens_per_message,
        min_conversation_messages: args.min_conversation_messages,
        viewport_radius: args.viewport_radius,
        coalesce_radius: args.coalesce_radius,
        val_ratio: args.val_ratio,
    };

    println!("Processing CSV files from {:?}...", args.csv_root);
    let session_results = process_all_sessions(
        &args.csv_root,
        &tokenizer,
        &config,
    )?;

    let total_sessions = session_results.len();
    println!("Processed {} sessions", total_sessions);

    let default_prompt = default_system_prompt(args.viewport_radius);
    let system_prompt = args.system_prompt.as_deref().unwrap_or(&default_prompt);

    println!("Writing output to {:?}...", args.output_dir);
    let result: PipelineResult = write_jsonl_output(
        session_results,
        &args.output_dir,
        args.val_ratio,
        system_prompt,
    )?;

    let metadata_path = args.output_dir.join("metadata.json");
    let metadata = serde_json::json!({
        "config": {
            "csv_root": args.csv_root.to_string_lossy(),
            "output_dir": args.output_dir.to_string_lossy(),
            "tokenizer": args.tokenizer,
            "max_tokens_per_conversation": args.max_tokens_per_conversation,
            "max_tokens_per_message": args.max_tokens_per_message,
            "min_conversation_messages": args.min_conversation_messages,
            "viewport_radius": args.viewport_radius,
            "coalesce_radius": args.coalesce_radius,
            "val_ratio": args.val_ratio,
        },
        "counts": {
            "total_sessions": result.total_sessions,
            "total_conversations": result.total_conversations,
            "train_conversations": result.train_conversations,
            "val_conversations": result.val_conversations,
        },
        "stats": {
            "total_messages": result.total_messages,
            "total_tokens": result.total_tokens,
            "avg_messages_per_conversation": if result.total_conversations > 0 {
                result.total_messages as f64 / result.total_conversations as f64
            } else {
                0.0
            },
            "avg_tokens_per_conversation": if result.total_conversations > 0 {
                result.total_tokens as f64 / result.total_conversations as f64
            } else {
                0.0
            },
        },
        "files": {
            "train_path": args.output_dir.join("training.jsonl").to_string_lossy(),
            "val_path": args.output_dir.join("validation.jsonl").to_string_lossy(),
        },
    });
    std::fs::write(&metadata_path, serde_json::to_string_pretty(&metadata)?)?;

    println!("
[summary]");
    println!("  Total sessions processed: {}", result.total_sessions);
    println!("  Train conversations: {}", result.train_conversations);
    println!("  Val conversations: {}", result.val_conversations);
    println!("  Total messages: {}", result.total_messages);
    println!("  Total tokens: {}", result.total_tokens);
    println!("  Output: {:?}/{{training,validation}}.jsonl", args.output_dir);
    println!("  Metadata: {:?}", metadata_path);

    Ok(())
}

fn run_analyze_git_pull(args: AnalyzeGitPullArgs) -> Result<(), Box<dyn std::error::Error>> {
    println!("Analyzing sessions for git pull corruption...");
    println!("CSV root: {:?}", args.csv_root);
    println!();

    let result: AnalysisResult = analyze_all_sessions(&args.csv_root)?;

    // Print summary
    println!("
╔══════════════════════════════════════════════════════════════╗");
    println!("║           GIT PULL CORRUPTION ANALYSIS RESULTS              ║");
    println!("╠══════════════════════════════════════════════════════════════╣");
    println!("║ Total sessions analyzed:        {:>8}                    ║", result.total_sessions);
    println!("║ Sessions with terminal git pull:{:>8}                    ║", result.sessions_with_git_pull);
    println!("║ Corrupted sessions (detected):  {:>8}                    ║", result.corrupted_sessions);
    println!("╠══════════════════════════════════════════════════════════════╣");
    println!("║ Total git pull commands:        {:>8}                    ║", result.total_git_pulls);
    println!("║ Git pulls with parseable output:{:>8}                    ║", result.parseable_git_pulls);
    println!("╠══════════════════════════════════════════════════════════════╣");
    println!("║ Total affected files:           {:>8}                    ║", result.total_affected_files);
    println!("║ Total affected edit events:     {:>8}                    ║", result.total_affected_edits);
    println!("╠══════════════════════════════════════════════════════════════╣");
    println!("║ CORRUPTION RATE (lower bound):  {:>7.2}%                    ║", result.corruption_rate * 100.0);
    println!("╚══════════════════════════════════════════════════════════════╝");

    println!("
[note] This analysis only detects terminal-based git pulls.");
    println!("       VS Code UI git pulls (~33% of users) are undetectable.");
    println!("       Actual corruption rate may be ~1.5x higher.");

    // Optionally write detailed results
    if let Some(output_path) = &args.output {
        let json = serde_json::to_string_pretty(&result)?;
        std::fs::write(output_path, json)?;
        println!("
Detailed results written to: {:?}", output_path);
    }

    // Optionally print corrupted session paths
    if args.verbose && !result.corrupted_session_paths.is_empty() {
        println!("
Corrupted sessions:");
        for path in &result.corrupted_session_paths {
            println!("  - {}", path);
        }
    }

    Ok(())
}