[tab] crowd-pilot-extension/src/serialize-cli.ts (typescript)
#!/usr/bin/env npx ts-node
/**
 * CLI tool for batch serialization of crowd-code CSV files to JSONL conversations.
 * 
 * This tool uses the same ConversationStateManager as the extension, ensuring
 * identical serialization between training data preprocessing and runtime inference.
 * 
 * Usage:
 *   npx ts-node src/serialize-cli.ts input.csv > output.jsonl
 *   npx ts-node src/serialize-cli.ts input1.csv input2.csv > combined.jsonl
 *   cat input.csv | npx ts-node src/serialize-cli.ts - > output.jsonl
 */

import * as fs from 'fs';
import * as readline from 'readline';
import { ConversationStateManager, ConversationMessage } from './serializer';

// -------------------- CSV Parsing --------------------

interface CrowdCodeEvent {
	sequence: number;
	timestamp: number;
	file: string;
	type: string;
	rangeOffset: number;
	rangeLength: number;
	text: string;
}

/**
 * Parse a CSV line, handling quoted fields with escaped quotes.
 */
function parseCSVLine(line: string): string[] {
	const result: string[] = [];
	let current = '';
	let inQuotes = false;
	let i = 0;

	while (i < line.length) {
		const char = line[i];

		if (inQuotes) {
			if (char === '"') {
				// Check for escaped quote
				if (i + 1 < line.length && line[i + 1] === '"') {
					current += '"';
					i += 2;
					continue;
				} else {
					// End of quoted field
					inQuotes = false;
					i++;
					continue;
				}
			} else {
				current += char;
				i++;
			}
		} else {
			if (char === '"') {
				inQuotes = true;
				i++;
			} else if (char === ',') {
				result.push(current);
				current = '';
				i++;
			} else {
				current += char;
				i++;
			}
		}
	}
	result.push(current);
	return result;
}

/**
 * Parse a crowd-code CSV file and return events.
 * 
 * Expected columns: Sequence, Timestamp, File, Type, RangeOffset, RangeLength, Text
 */
function parseCSVFile(content: string): CrowdCodeEvent[] {
	const lines = content.split(/
?
/).filter(line => line.trim());
	if (lines.length === 0) {
		return [];
	}

	// Parse header to get column indices
	const header = parseCSVLine(lines[0]);
	const colIndex: Record<string, number> = {};
	header.forEach((col, idx) => {
		colIndex[col.toLowerCase().trim()] = idx;
	});

	// Validate required columns
	const requiredCols = ['file', 'type'];
	for (const col of requiredCols) {
		if (!(col in colIndex)) {
			throw new Error(`Missing required column: ${col}`);
		}
	}

	const events: CrowdCodeEvent[] = [];
	for (let i = 1; i < lines.length; i++) {
		const fields = parseCSVLine(lines[i]);
		if (fields.length === 0) continue;

		const event: CrowdCodeEvent = {
			sequence: colIndex['sequence'] !== undefined ? parseInt(fields[colIndex['sequence']] || '0', 10) : i,
			timestamp: colIndex['timestamp'] !== undefined ? parseInt(fields[colIndex['timestamp']] || '0', 10) : 0,
			file: fields[colIndex['file']] || '',
			type: (fields[colIndex['type']] || '').toLowerCase(),
			rangeOffset: colIndex['rangeoffset'] !== undefined ? parseInt(fields[colIndex['rangeoffset']] || '0', 10) : 0,
			rangeLength: colIndex['rangelength'] !== undefined ? parseInt(fields[colIndex['rangelength']] || '0', 10) : 0,
			text: fields[colIndex['text']] || '',
		};
		events.push(event);
	}

	return events;
}

// -------------------- Event Processing --------------------

/**
 * Process events through the ConversationStateManager and return serialized messages.
 */
function processEvents(events: CrowdCodeEvent[]): ConversationMessage[] {
	const manager = new ConversationStateManager();

	for (const event of events) {
		switch (event.type) {
			case 'tab':
				// Tab event: file path with optional content
				const tabContent = event.text || null;
				manager.handleTabEvent(event.file, tabContent);
				break;

			case 'content':
				// Content change event
				manager.handleContentEvent(
					event.file,
					event.rangeOffset,
					event.rangeLength,
					event.text
				);
				break;

			case 'selection_command':
			case 'selection_mouse':
			case 'selection_keyboard':
				// Selection event
				manager.handleSelectionEvent(event.file, event.rangeOffset);
				break;

			case 'terminal_command':
				// Terminal command event
				manager.handleTerminalCommandEvent(event.text);
				break;

			case 'terminal_output':
				// Terminal output event
				manager.handleTerminalOutputEvent(event.text);
				break;

			case 'terminal_focus':
				// Terminal focus event
				manager.handleTerminalFocusEvent();
				break;

			case 'git_branch_checkout':
				// Git branch checkout event
				manager.handleGitBranchCheckoutEvent(event.text);
				break;

			default:
				console.error(`[serialize-cli] Unknown event type: ${event.type}`);
		}
	}

	return manager.finalizeForModel();
}

// -------------------- Output Formatting --------------------

interface ConversationOutput {
	messages: Array<{ from: string; value: string }>;
	metadata?: {
		eventCount: number;
		sourceFile?: string;
	};
}

/**
 * Format messages as a JSONL line.
 */
function formatAsJSONL(messages: ConversationMessage[], metadata?: { eventCount: number; sourceFile?: string }): string {
	const output: ConversationOutput = {
		messages: messages.map(m => ({ from: m.from, value: m.value })),
	};
	if (metadata) {
		output.metadata = metadata;
	}
	return JSON.stringify(output);
}

// -------------------- Main --------------------

async function readStdin(): Promise<string> {
	return new Promise((resolve, reject) => {
		const chunks: string[] = [];
		const rl = readline.createInterface({
			input: process.stdin,
			crlfDelay: Infinity,
		});

		const lines: string[] = [];
		rl.on('line', (line) => {
			lines.push(line);
		});

		rl.on('close', () => {
			resolve(lines.join('
'));
		});

		rl.on('error', (err) => {
			reject(err);
		});
	});
}

async function main() {
	const args = process.argv.slice(2);

	if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
		console.log(`
Usage: npx ts-node serialize-cli.ts [options] <input.csv> [input2.csv ...]

Serialize crowd-code CSV files to JSONL conversation format.

Arguments:
  <input.csv>       CSV file(s) to process. Use '-' to read from stdin.

Options:
  -h, --help        Show this help message
  --per-file        Output one JSONL line per input file (default)
  --combined        Combine all input files into one conversation

Examples:
  npx ts-node serialize-cli.ts session.csv > output.jsonl
  npx ts-node serialize-cli.ts *.csv > all_sessions.jsonl
  cat session.csv | npx ts-node serialize-cli.ts - > output.jsonl
`);
		process.exit(0);
	}

	const combined = args.includes('--combined');
	const inputFiles = args.filter(arg => !arg.startsWith('--'));

	if (inputFiles.length === 0) {
		console.error('Error: No input files specified');
		process.exit(1);
	}

	if (combined) {
		// Process all files as one combined session
		const allEvents: CrowdCodeEvent[] = [];
		for (const inputFile of inputFiles) {
			let content: string;
			if (inputFile === '-') {
				content = await readStdin();
			} else {
				if (!fs.existsSync(inputFile)) {
					console.error(`Error: File not found: ${inputFile}`);
					process.exit(1);
				}
				content = fs.readFileSync(inputFile, 'utf-8');
			}
			const events = parseCSVFile(content);
			allEvents.push(...events);
		}

		const messages = processEvents(allEvents);
		if (messages.length > 0) {
			console.log(formatAsJSONL(messages, { eventCount: allEvents.length }));
		}
	} else {
		// Process each file separately (default)
		for (const inputFile of inputFiles) {
			let content: string;
			if (inputFile === '-') {
				content = await readStdin();
			} else {
				if (!fs.existsSync(inputFile)) {
					console.error(`Error: File not found: ${inputFile}`);
					process.exit(1);
				}
				content = fs.readFileSync(inputFile, 'utf-8');
			}

			const events = parseCSVFile(content);
			const messages = processEvents(events);

			if (messages.length > 0) {
				const sourceFile = inputFile === '-' ? undefined : inputFile;
				console.log(formatAsJSONL(messages, { eventCount: events.length, sourceFile }));
			}
		}
	}
}

main().catch(err => {
	console.error('Error:', err.message);
	process.exit(1);
});