[tab] crowd-pilot-extension/src/serializer.ts (typescript)
/**
 * Serialization utilities for converting IDE events into conversation format.
 * This module is the single source of truth for serialization logic,
 * used by both the extension (real-time inference) and CLI (batch processing).
 */

// -------------------- Constants --------------------
export const ANSI_CSI_RE = /\x1b\[[0-9;?]*[ -/]*[@-~]/g;
export const ANSI_OSC_TERMINATED_RE = /\x1b\][\s\S]*?(?:\x07|\x1b\\)/g;
export const ANSI_OSC_LINE_FALLBACK_RE = /\x1b\][^
]*$/g;

// NOTE (f.srambical): Make sure that these are the parameters that were used during serialization
export const VIEWPORT_RADIUS = 10;
export const COALESCE_RADIUS = 5;
// FIXME (f.srambical): this slightly differs from the logic in @file:crowd-pilot/crowd_pilot/serialization_utils.py
// as we only approximate the token count
export const MAX_TOKENS_PER_MESSAGE = 2048;
export const MAX_TOKENS_PER_TERMINAL_OUTPUT = 256;

// -------------------- Types --------------------
export interface ConversationMessage {
	from: 'User' | 'Assistant';
	value: string;
}

// -------------------- Helper Functions --------------------

/**
 * Estimate token count from text using character-based heuristic.
 * Uses ~4 characters per token as a rough approximation for most LLMs.
 */
export function estimateTokens(text: string): number {
	return Math.ceil(text.length / 4);
}

export function cleanText(text: string): string {
	return text.replace(/
/g, '
').replace(/
/g, '
').trimEnd();
}

export function fencedBlock(language: string | null, content: string): string {
	const lang = (language || '').toLowerCase();
	return `\`\`\`${lang}
${content}
\`\`\`
`;
}

export function applyChange(content: string, offset: number, length: number, newText: string): string {
	let base = String(content);
	const text = (newText ?? '').replace(/\
/g, '
').replace(/\
/g, '
');
	if (offset > base.length) {
		base = base + ' '.repeat(offset - base.length);
	}
	return base.slice(0, offset) + text + base.slice(offset + length);
}

export function applyBackspaces(text: string): string {
	const out: string[] = [];
	for (const ch of text) {
		if (ch === '\b') {
			if (out.length > 0) {
				out.pop();
			}
		} else {
			out.push(ch);
		}
	}
	return out.join('');
}

export function normalizeTerminalOutput(raw: string): string {
	if (!raw) { return raw; }
	let s = applyBackspaces(raw);
	s = s.replace(ANSI_OSC_TERMINATED_RE, '');
	s = s.split('
').map(line => line.replace(ANSI_OSC_LINE_FALLBACK_RE, '')).join('
');
	const resolvedLines: string[] = [];
	for (const seg of s.split('
')) {
		const parts = seg.split('
');
		let chosen = '';
		for (let i = parts.length - 1; i >= 0; i--) {
			if (parts[i] !== '') {
				chosen = parts[i];
				break;
			}
		}
		if (chosen === '' && parts.length > 0) {
			chosen = parts[parts.length - 1];
		}
		resolvedLines.push(chosen);
	}
	s = resolvedLines.join('
');
	s = s.replace(ANSI_CSI_RE, '');
	s = s.replace(/\x07/g, '');
	return s;
}

export function lineNumberedOutput(content: string, startLine?: number, endLine?: number): string {
	const lines = content.split(/
?
/);
	const total = (lines.length === 1 && lines[0] === '') ? 0 : lines.length;
	if (total === 0) { return ''; }
	const s = startLine !== undefined ? Math.max(1, Math.min(startLine, total)) : 1;
	const e = endLine !== undefined ? Math.max(1, Math.min(endLine, total)) : total;
	const buf: string[] = [];
	for (let idx = s; idx <= e; idx++) {
		const lineText = lines[idx - 1] ?? '';
		buf.push(`${idx.toString().padStart(6, ' ')}	${lineText}`);
	}
	return buf.join('
');
}

export function serializeComputeViewport(totalLines: number, centerLine: number, radius: number): { start: number; end: number } {
	if (totalLines <= 0) { return { start: 1, end: 0 }; }
	const start = Math.max(1, centerLine - radius);
	const end = Math.min(totalLines, centerLine + radius);
	return { start, end };
}

export function escapeSingleQuotesForSed(text: string): string {
	return text.replace(/'/g, "'\"'\"'");
}

// -------------------- SequenceMatcher --------------------

type OpcodeTag = "replace" | "delete" | "insert" | "equal";
export type Opcode = [OpcodeTag, number, number, number, number];

export class SequenceMatcher {
	private a: string[];
	private b: string[];
	private b2j: Map<string, number[]> | null = null;
	private matchingBlocks: Array<{ i: number; j: number; n: number }> | null = null;
	private opcodesCache: Opcode[] | null = null;

	constructor(a: string[], b: string[]) {
		this.a = a;
		this.b = b;
		this.chainB();
	}

	// Exact port of difflib's __chain_b assuming junk=None and autojunk=False
	private chainB() {
		this.b2j = new Map();
		for (let i = 0; i < this.b.length; i++) {
			const elt = this.b[i];
			let indices = this.b2j.get(elt);
			if (!indices) {
				indices = [];
				this.b2j.set(elt, indices);
			}
			indices.push(i);
		}
	}

	// Exact port of difflib's find_longest_match assuming junk=None
	private findLongestMatch(alo: number, ahi: number, blo: number, bhi: number) {
		const b2j = this.b2j!;

		let besti = alo;
		let bestj = blo;
		let bestsize = 0;

		let j2len = new Map<number, number>();

		for (let i = alo; i < ahi; i++) {
			const newj2len = new Map<number, number>();
			const elt = this.a[i];
			const indices = b2j.get(elt);
			if (indices) {
				for (const j of indices) {
					if (j < blo) { continue; }
					if (j >= bhi) { break; }
					const k = (j2len.get(j - 1) ?? 0) + 1;
					newj2len.set(j, k);
					if (k > bestsize) {
						besti = i - k + 1;
						bestj = j - k + 1;
						bestsize = k;
					}
				}
			}
			j2len = newj2len;
		}

		// bjunk is always empty since we assume junk=None
		while (besti > alo && bestj > blo && this.a[besti - 1] === this.b[bestj - 1]) {
			besti--;
			bestj--;
			bestsize++;
		}
		while (
			besti + bestsize < ahi &&
			bestj + bestsize < bhi &&
			this.a[besti + bestsize] === this.b[bestj + bestsize]
		) {
			bestsize++;
		}

		return { i: besti, j: bestj, n: bestsize };
	}

	// Exact port of difflib's get_matching_blocks
	private getMatchingBlocks() {
		if (this.matchingBlocks !== null) { return this.matchingBlocks; }

		const la = this.a.length;
		const lb = this.b.length;

		const queue: Array<[number, number, number, number]> = [[0, la, 0, lb]];
		const matchingBlocks: Array<{ i: number; j: number; n: number }> = [];

		while (queue.length) {
			const [alo, ahi, blo, bhi] = queue.pop()!;
			const match = this.findLongestMatch(alo, ahi, blo, bhi);
			const { i, j, n } = match;
			if (n > 0) {
				matchingBlocks.push(match);
				if (alo < i && blo < j) {
					queue.push([alo, i, blo, j]);
				}
				if (i + n < ahi && j + n < bhi) {
					queue.push([i + n, ahi, j + n, bhi]);
				}
			}
		}

		// Sort lexicographically by (i, j, n) to match Python's tuple sort behavior
		matchingBlocks.sort((a, b) => {
			if (a.i !== b.i) { return a.i - b.i; }
			if (a.j !== b.j) { return a.j - b.j; }
			return a.n - b.n;
		});

		let i1 = 0;
		let j1 = 0;
		let k1 = 0;
		const result: Array<{ i: number; j: number; n: number }> = [];

		for (const m of matchingBlocks) {
			if (i1 + k1 === m.i && j1 + k1 === m.j) {
				k1 += m.n;
			} else {
				if (k1 > 0) {
					result.push({ i: i1, j: j1, n: k1 });
				}
				i1 = m.i;
				j1 = m.j;
				k1 = m.n;
			}
		}
		if (k1 > 0) {
			result.push({ i: i1, j: j1, n: k1 });
		}

		result.push({ i: la, j: lb, n: 0 });

		this.matchingBlocks = result;
		return result;
	}

	// Exact port of difflib's get_opcodes
	getOpcodes(): Opcode[] {
		if (this.opcodesCache) { return this.opcodesCache; }

		const opcodes: Opcode[] = [];
		let i = 0;
		let j = 0;

		for (const m of this.getMatchingBlocks()) {
			const tag: OpcodeTag = "equal";
			const ai = m.i;
			const bj = m.j;
			const n = m.n;

			let tagToUse: OpcodeTag | null = null;

			if (i < ai && j < bj) {
				tagToUse = "replace";
			} else if (i < ai) {
				tagToUse = "delete";
			} else if (j < bj) {
				tagToUse = "insert";
			}

			if (tagToUse) {
				opcodes.push([tagToUse, i, ai, j, bj]);
			}
			if (n > 0) {
				opcodes.push([tag, ai, ai + n, bj, bj + n]);
			}
			i = ai + n;
			j = bj + n;
		}

		this.opcodesCache = opcodes;
		return opcodes;
	}
}

// -------------------- Diff Computation --------------------

export function computeChangedBlockLines(before: string, after: string): {
	startBefore: number;
	endBefore: number;
	startAfter: number;
	endAfter: number;
	replacementLines: string[];
} {
	const beforeLines = before.split(/
?
/);
	const afterLines = after.split(/
?
/);

	const sm = new SequenceMatcher(beforeLines, afterLines);
	const allOpcodes = sm.getOpcodes();
	const nonEqual = allOpcodes.filter(op => op[0] !== "equal");

	if (nonEqual.length === 0) {
		throw new Error(
			"Opcode list cannot be empty! Likely a bug in the diff computation."
		);
	}

	const first = nonEqual[0];
	const last = nonEqual[nonEqual.length - 1];

	// i1/i2 refer to 'before' indices, j1/j2 to 'after'
	const startBefore = Math.max(1, first[1] + 1);
	const endBefore = last[2];
	const startAfter = Math.max(1, first[3] + 1);
	const endAfter = last[4];
	const replacementLines = afterLines.slice(first[3], last[4]);

	return { startBefore, endBefore, startAfter, endAfter, replacementLines };
}

// -------------------- Conversation State Manager --------------------

export class ConversationStateManager {
	private messages: ConversationMessage[] = [];
	private fileStates: Map<string, string> = new Map();
	private perFileViewport: Map<string, { start: number; end: number } | null> = new Map();
	private filesOpenedInConversation: Set<string> = new Set();
	private terminalOutputBuffer: string[] = [];
	private pendingEditsBefore: Map<string, string | null> = new Map();
	private pendingEditRegions: Map<string, { start: number; end: number } | null> = new Map();

	constructor() {}

	reset(): void {
		this.messages = [];
		this.fileStates.clear();
		this.perFileViewport.clear();
		this.filesOpenedInConversation.clear();
		this.terminalOutputBuffer = [];
		this.pendingEditsBefore.clear();
		this.pendingEditRegions.clear();
	}

	getMessages(): ConversationMessage[] {
		return [...this.messages];
	}

	getFileContent(filePath: string): string {
		return this.fileStates.get(filePath) ?? '';
	}

	private appendMessage(message: ConversationMessage): void {
		const tokens = estimateTokens(message.value);
		if (tokens > MAX_TOKENS_PER_MESSAGE) {
			const maxChars = MAX_TOKENS_PER_MESSAGE * 4;
			message.value = message.value.slice(0, maxChars);
		}
		this.messages.push(message);
	}

	private maybeCaptureFileContents(filePath: string, content: string): void {
		if (this.filesOpenedInConversation.has(filePath)) {
			return;
		}
		const cmd = `cat -n ${filePath}`;
		this.appendMessage({
			from: 'Assistant',
			value: fencedBlock('bash', cleanText(cmd)),
		});
		const output = lineNumberedOutput(content);
		this.appendMessage({
			from: 'User',
			value: `<stdout>
${output}
</stdout>`,
		});
		this.filesOpenedInConversation.add(filePath);
	}

	flushTerminalOutputBuffer(): void {
		if (this.terminalOutputBuffer.length === 0) {
			return;
		}
		const aggregated = this.terminalOutputBuffer.join('');
		const out = normalizeTerminalOutput(aggregated);
		let cleaned = cleanText(out);
		
		const tokens = estimateTokens(cleaned);
		if (tokens > MAX_TOKENS_PER_TERMINAL_OUTPUT) {
			const maxChars = MAX_TOKENS_PER_TERMINAL_OUTPUT * 4;
			cleaned = cleaned.slice(0, maxChars) + '
... [truncated]';
		}
		
		if (cleaned.trim()) {
			this.appendMessage({
				from: 'User',
				value: `<stdout>
${cleaned}
</stdout>`,
			});
		}
		this.terminalOutputBuffer = [];
	}

	flushPendingEditForFile(targetFile: string): void {
		const beforeSnapshot = this.pendingEditsBefore.get(targetFile);
		if (beforeSnapshot === null || beforeSnapshot === undefined) {
			return;
		}
		const afterState = this.fileStates.get(targetFile) ?? '';
		if (beforeSnapshot.replace(/
+$/, '') === afterState.replace(/
+$/, '')) {
			this.pendingEditsBefore.set(targetFile, null);
			this.pendingEditRegions.set(targetFile, null);
			return;
		}

		const {
			startBefore,
			endBefore,
			startAfter,
			endAfter,
			replacementLines,
		} = computeChangedBlockLines(beforeSnapshot, afterState);

		const beforeTotalLines = beforeSnapshot.split(/
?
/).length;
		let sedCmd: string;

		if (endBefore < startBefore) {
			// Pure insertion
			const escapedLines = replacementLines.map(line => escapeSingleQuotesForSed(line));
			const sedPayload = escapedLines.join('
');
			if (startBefore <= Math.max(1, beforeTotalLines)) {
				sedCmd = `sed -i '${startBefore}i\\
${sedPayload}' ${targetFile}`;
			} else {
				sedCmd = `sed -i '$a\\
${sedPayload}' ${targetFile}`;
			}
		} else if (replacementLines.length === 0) {
			// Pure deletion
			sedCmd = `sed -i '${startBefore},${endBefore}d' ${targetFile}`;
		} else {
			// Replacement
			const escapedLines = replacementLines.map(line => escapeSingleQuotesForSed(line));
			const sedPayload = escapedLines.join('
');
			sedCmd = `sed -i '${startBefore},${endBefore}c\\
${sedPayload}' ${targetFile}`;
		}

		const totalLines = afterState.split(/
?
/).length;
		const center = Math.floor((startAfter + endAfter) / 2);
		const vp = serializeComputeViewport(totalLines, center, VIEWPORT_RADIUS);
		this.perFileViewport.set(targetFile, vp);

		this.maybeCaptureFileContents(targetFile, beforeSnapshot);

		const chainedCmd = `${sedCmd} && cat -n ${targetFile} | sed -n '${vp.start},${vp.end}p'`;
		this.appendMessage({
			from: 'Assistant',
			value: fencedBlock('bash', cleanText(chainedCmd)),
		});

		const viewportOutput = lineNumberedOutput(afterState, vp.start, vp.end);
		this.appendMessage({
			from: 'User',
			value: `<stdout>
${viewportOutput}
</stdout>`,
		});

		this.pendingEditsBefore.set(targetFile, null);
		this.pendingEditRegions.set(targetFile, null);
	}

	flushAllPendingEdits(): void {
		for (const fname of this.pendingEditsBefore.keys()) {
			this.flushPendingEditForFile(fname);
		}
	}

	handleTabEvent(filePath: string, textContent: string | null): void {
		this.flushAllPendingEdits();
		this.flushTerminalOutputBuffer();

		if (textContent !== null) {
			const content = textContent.replace(/\
/g, '
').replace(/\
/g, '
');
			this.fileStates.set(filePath, content);

			const cmd = `cat -n ${filePath}`;
			this.appendMessage({
				from: 'Assistant',
				value: fencedBlock('bash', cleanText(cmd)),
			});
			const output = lineNumberedOutput(content);
			this.appendMessage({
				from: 'User',
				value: `<stdout>
${output}
</stdout>`,
			});
			this.filesOpenedInConversation.add(filePath);
		} else {
			// File switch without content snapshot: show current viewport only
			const content = this.fileStates.get(filePath) ?? '';
			const totalLines = content.split(/
?
/).length;
			let vp = this.perFileViewport.get(filePath);
			if (!vp || vp.end === 0) {
				vp = serializeComputeViewport(totalLines, 1, VIEWPORT_RADIUS);
				this.perFileViewport.set(filePath, vp);
			}
			if (vp && vp.end >= vp.start) {
				this.maybeCaptureFileContents(filePath, content);
				const cmd = `cat -n ${filePath} | sed -n '${vp.start},${vp.end}p'`;
				this.appendMessage({
					from: 'Assistant',
					value: fencedBlock('bash', cleanText(cmd)),
				});
				const viewportOutput = lineNumberedOutput(content, vp.start, vp.end);
				this.appendMessage({
					from: 'User',
					value: `<stdout>
${viewportOutput}
</stdout>`,
				});
			}
		}
	}

	handleContentEvent(filePath: string, offset: number, length: number, newText: string): void {
		this.flushTerminalOutputBuffer();

		const before = this.fileStates.get(filePath) ?? '';
		const newTextStr = newText ?? '';

		// Approximate current edit region in line space
		const startLineCurrent = before.slice(0, offset).split('
').length;
		const deletedContent = before.slice(offset, offset + length);
		const linesAdded = (newTextStr.match(/
/g) || []).length;
		const linesDeleted = (deletedContent.match(/
/g) || []).length;
		const regionStart = startLineCurrent;
		const regionEnd = startLineCurrent + Math.max(linesAdded, linesDeleted, 0);

		// Flush pending edits if this edit is far from the pending region
		let currentRegion = this.pendingEditRegions.get(filePath);
		if (currentRegion !== null && currentRegion !== undefined) {
			const { start: rstart, end: rend } = currentRegion;
			if (regionStart < (rstart - COALESCE_RADIUS) || regionStart > (rend + COALESCE_RADIUS)) {
				this.flushPendingEditForFile(filePath);
				currentRegion = null;
			}
		}

		const after = applyChange(before, offset, length, newText);

		if (this.pendingEditsBefore.get(filePath) === null || this.pendingEditsBefore.get(filePath) === undefined) {
			this.pendingEditsBefore.set(filePath, before);
		}

		// Update/initialize region union
		if (currentRegion === null || currentRegion === undefined) {
			this.pendingEditRegions.set(filePath, { start: regionStart, end: Math.max(regionStart, regionEnd) });
		} else {
			const { start: rstart, end: rend } = currentRegion;
			this.pendingEditRegions.set(filePath, {
				start: Math.min(rstart, regionStart),
				end: Math.max(rend, regionEnd),
			});
		}

		this.fileStates.set(filePath, after);
	}

	handleSelectionEvent(filePath: string, offset: number): void {
		if (this.pendingEditsBefore.get(filePath) !== null && this.pendingEditsBefore.get(filePath) !== undefined) {
			return;
		}

		this.flushTerminalOutputBuffer();

		const content = this.fileStates.get(filePath) ?? '';
		const totalLines = content.split(/
?
/).length;
		const targetLine = content.slice(0, offset).split('
').length;

		let vp = this.perFileViewport.get(filePath);
		let shouldEmit = false;

		if (!vp || vp.end === 0) {
			vp = serializeComputeViewport(totalLines, targetLine, VIEWPORT_RADIUS);
			this.perFileViewport.set(filePath, vp);
			shouldEmit = true;
		} else {
			if (targetLine < vp.start || targetLine > vp.end) {
				vp = serializeComputeViewport(totalLines, targetLine, VIEWPORT_RADIUS);
				this.perFileViewport.set(filePath, vp);
				shouldEmit = true;
			}
		}

		if (shouldEmit && vp && vp.end >= vp.start) {
			this.maybeCaptureFileContents(filePath, content);
			const cmd = `cat -n ${filePath} | sed -n '${vp.start},${vp.end}p'`;
			this.appendMessage({
				from: 'Assistant',
				value: fencedBlock('bash', cleanText(cmd)),
			});
			const viewportOutput = lineNumberedOutput(content, vp.start, vp.end);
			this.appendMessage({
				from: 'User',
				value: `<stdout>
${viewportOutput}
</stdout>`,
			});
		}
	}

	handleTerminalCommandEvent(command: string): void {
		this.flushAllPendingEdits();
		this.flushTerminalOutputBuffer();

		const commandStr = command.replace(/\
/g, '
').replace(/\
/g, '
');
		this.appendMessage({
			from: 'Assistant',
			value: fencedBlock('bash', cleanText(commandStr)),
		});
	}

	handleTerminalOutputEvent(output: string): void {
		const rawOutput = output.replace(/\
/g, '
').replace(/\
/g, '
');
		this.terminalOutputBuffer.push(rawOutput);
	}

	handleTerminalFocusEvent(): void {
		this.flushAllPendingEdits();
		this.flushTerminalOutputBuffer();
		// No-op for bash transcript; focus changes don't emit commands/output
	}

	handleGitBranchCheckoutEvent(branchInfo: string): void {
		this.flushAllPendingEdits();
		this.flushTerminalOutputBuffer();

		const branchStr = branchInfo.replace(/\
/g, '
').replace(/\
/g, '
');
		const cleaned = cleanText(branchStr);
		const m = cleaned.match(/to '([^']+)'/);
		if (!m) {
			console.warn(`[crowd-pilot] Could not extract branch name from git checkout message: ${cleaned}`);
			return;
		}
		let branchName = m[1].trim();
		// Safe-quote branch if it contains special characters
		if (/[^A-Za-z0-9._/\\-]/.test(branchName)) {
			branchName = "'" + branchName.replace(/'/g, "'\"'\"'") + "'";
		}
		const cmd = `git checkout ${branchName}`;
		this.appendMessage({
			from: 'Assistant',
			value: fencedBlock('bash', cleanText(cmd)),
		});
	}

	// Finalize and get conversation ready for model
	finalizeForModel(): ConversationMessage[] {
		this.flushAllPendingEdits();
		this.flushTerminalOutputBuffer();
		return this.getMessages();
	}
}