p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] crowd-pilot-extension/src/extension.ts (typescript)
// -------------------- Context Window Management --------------------

/**
 * Estimate token count from text using character-based heuristic.
 * Uses ~4 characters per token as a rough approximation for most LLMs.
 */
function estimateTokens(text: string): number {
	return Math.ceil(text.length / 4);
}

/**
 * Truncate conversation messages to fit within the context window.
 * Preserves the system prompt and drops oldest messages first.
 * Returns the truncated messages array.
 */
function truncateToContextLimit(
	messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
	maxTokens: number
): Array<{ role: 'system' | 'user' | 'assistant'; content: string }> {
	// Calculate total tokens
	let totalTokens = 0;
	for (const msg of messages) {
		totalTokens += estimateTokens(msg.content);
	}

	// If within limit, return as-is
	if (totalTokens <= maxTokens) {
		return messages;
	}

	// Separate system message from conversation
	const systemMessages = messages.filter(m => m.role === 'system');
	const conversationMessages = messages.filter(m => m.role !== 'system');

	// Calculate system prompt tokens (these are preserved)
	let systemTokens = 0;
	for (const msg of systemMessages) {
		systemTokens += estimateTokens(msg.content);
	}

	// Available tokens for conversation
	const availableTokens = maxTokens - systemTokens;
	if (availableTokens <= 0) {
		console.warn('[crowd-pilot] System prompt exceeds context limit');
		return systemMessages;
	}

	// Drop oldest messages until we fit
	const truncatedConversation: typeof conversationMessages = [];
	let conversationTokens = 0;

	// Start from the end (most recent) and work backwards
	for (let i = conversationMessages.length - 1; i >= 0; i--) {
		const msgTokens = estimateTokens(conversationMessages[i].content);
		if (conversationTokens + msgTokens <= availableTokens) {
			truncatedConversation.unshift(conversationMessages[i]);
			conversationTokens += msgTokens;
		} else {
			// Log how many messages were dropped
			const droppedCount = i + 1;
			console.log(`[crowd-pilot] Truncated ${droppedCount} oldest messages to fit context window (${totalTokens} -> ${systemTokens + conversationTokens} tokens)`);
			break;
		}
	}

	return [...systemMessages, ...truncatedConversation];
}
[content] crowd-pilot-extension/src/extension.ts (typescript)
// -------------------- Context Window Management --------------------

/**
 * Estimate token count from text using character-based heuristic.
 * Uses ~4 characters per token as a rough approximation for most LLMs.
 */
function estimateTokens(text: string): number {
	return Math.ceil(text.length / 4);
}

/**
 * Truncate conversation messages to fit within the context window.
 * Preserves the system prompt and drops oldest messages first.
 * Single-pass: computes token counts and separates messages simultaneously.
 */
function truncateToContextLimit(
	messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
	maxTokens: number
): Array<{ role: 'system' | 'user' | 'assistant'; content: string }> {
	type Message = { role: 'system' | 'user' | 'assistant'; content: string };
	const systemMessages: Message[] = [];
	const conversationWithTokens: Array<{ msg: Message; tokens: number }> = [];
	let systemTokens = 0;
	let totalTokens = 0;

	for (const msg of messages) {
		const tokens = estimateTokens(msg.content);
		totalTokens += tokens;
		if (msg.role === 'system') {
			systemMessages.push(msg);
			systemTokens += tokens;
		} else {
			conversationWithTokens.push({ msg, tokens });
		}
	}

	if (totalTokens <= maxTokens) {
		return messages;
	}

	const availableTokens = maxTokens - systemTokens;
	if (availableTokens <= 0) {
		console.warn('[crowd-pilot] System prompt exceeds context limit');
		return systemMessages;
	}

	// Find cutoff index by iterating from most recent backwards
	let keptTokens = 0;
	let cutoffIndex = conversationWithTokens.length;
	for (let i = conversationWithTokens.length - 1; i >= 0; i--) {
		if (keptTokens + conversationWithTokens[i].tokens <= availableTokens) {
			keptTokens += conversationWithTokens[i].tokens;
			cutoffIndex = i;
		} else {
			break;
		}
	}

	if (cutoffIndex > 0) {
		console.log(`[crowd-pilot] Truncated ${cutoffIndex} oldest messages to fit context window (${totalTokens} -> ${systemTokens + keptTokens} tokens)`);
	}

	const result: Message[] = [...systemMessages];
	for (let i = cutoffIndex; i < conversationWithTokens.length; i++) {
		result.push(conversationWithTokens[i].msg);
	}
	return result;
}
[content] crowd-pilot-extension/src/extension.ts (typescript)
// -------------------- Context Window Management --------------------

/**
 * Estimate token count from text using character-based heuristic.
 * Uses ~4 characters per token as a rough approximation for most LLMs.
 */
function estimateTokens(text: string): number {
	return Math.ceil(text.length / 4);
}

/**
 * Truncate conversation messages to fit within the context window.
 * Assumes system prompt is the first message. Drops oldest conversation messages first.
 */
function truncateToContextLimit(
	messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
	maxTokens: number
): Array<{ role: 'system' | 'user' | 'assistant'; content: string }> {
	if (messages.length === 0) { return messages; }

	const systemTokens = estimateTokens(messages[0].content);
	const availableTokens = maxTokens - systemTokens;

	// Compute token counts for conversation messages (skip system at index 0)
	const tokenCounts: number[] = [];
	let totalConversationTokens = 0;
	for (let i = 1; i < messages.length; i++) {
		const t = estimateTokens(messages[i].content);
		tokenCounts.push(t);
		totalConversationTokens += t;
	}

	if (totalConversationTokens <= availableTokens) {
		return messages;
	}

	// Find cutoff by iterating backwards from most recent
	let keptTokens = 0;
	let cutoffIndex = tokenCounts.length;
	for (let i = tokenCounts.length - 1; i >= 0; i--) {
		if (keptTokens + tokenCounts[i] <= availableTokens) {
			keptTokens += tokenCounts[i];
			cutoffIndex = i;
		} else {
			break;
		}
	}

	console.log(`[crowd-pilot] Truncated ${cutoffIndex} oldest messages (${systemTokens + totalConversationTokens} -> ${systemTokens + keptTokens} tokens)`);
	return [messages[0], ...messages.slice(cutoffIndex + 1)];
}
[content] crowd-pilot-extension/src/extension.ts (typescript)
// -------------------- Context Window Management --------------------

/**
 * Estimate token count from text using character-based heuristic.
 * Uses ~4 characters per token as a rough approximation for most LLMs.
 */
function estimateTokens(text: string): number {
	return Math.ceil(text.length / 4);
}

/**
 * Truncate conversation messages to fit within the context window.
 * Assumes system prompt is the first message. Drops oldest conversation messages first.
 */
function truncateToContextLimit(
	messages: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>,
	maxTokens: number
): Array<{ role: 'system' | 'user' | 'assistant'; content: string }> {
	if (messages.length === 0) { return messages; }

	const systemTokens = estimateTokens(messages[0].content);
	const availableTokens = maxTokens - systemTokens;

	const tokenCounts = messages.slice(1).map(m => estimateTokens(m.content));
	const totalConversationTokens = tokenCounts.reduce((a, b) => a + b, 0);

	if (totalConversationTokens <= availableTokens) {
		return messages;
	}

	// Find cutoff by iterating backwards from most recent
	let keptTokens = 0;
	let cutoffIndex = tokenCounts.length;
	for (let i = tokenCounts.length - 1; i >= 0; i--) {
		if (keptTokens + tokenCounts[i] <= availableTokens) {
			keptTokens += tokenCounts[i];
			cutoffIndex = i;
		} else {
			break;
		}
	}

	console.log(`[crowd-pilot] Truncated ${cutoffIndex} oldest messages (${systemTokens + totalConversationTokens} -> ${systemTokens + keptTokens} tokens)`);
	return [messages[0], ...messages.slice(cutoffIndex + 1)];
}
[content] crowd-pilot-extension/package.json (json)
,
        "crowd-pilot.maxContextTokens": {
          "type": "number",
          "default": 120000,
          "description": "Context length (in tokens). Older messages are truncated to fit. Set below your model's limit to leave room for the response."
        }
[tab] crowd-pilot-extension/package.json (json)
{
  "name": "crowd-pilot",
  "displayName": "crowd-pilot-extension",
  "description": "Teaching language models to code like humans.",
  "publisher": "p-doom",
  "version": "0.0.1",
  "repository": {
    "type": "git",
    "url": "https://github.com/p-doom/crowd-pilot-extension"
  },
  "engines": {
    "vscode": "^1.99.3"
  },
  "categories": [
    "Other"
  ],
  "activationEvents": [
    "onStartupFinished"
  ],
  "main": "./out/extension.js",
  "contributes": {
    "commands": [
      {
        "command": "crowd-pilot.toggleSuggestions",
        "title": "crowd-pilot: Toggle Tab Suggestions"
      },
      {
        "command": "crowd-pilot.hideUi",
        "title": "crowd-pilot: Hide Preview"
      },
      {
        "command": "crowd-pilot.sglangTest",
        "title": "crowd-pilot: Test SGLang"
      },
      {
        "command": "crowd-pilot.modelRun",
        "title": "crowd-pilot: Model Plan & Run"
      }
    ],
    "configuration": {
      "title": "crowd-pilot",
      "properties": {
        "crowd-pilot.hostname": {
          "type": "string",
          "default": "hai002",
          "description": "Hostname of the SGLang server"
        },
        "crowd-pilot.port": {
          "type": "number",
          "default": 30000,
          "description": "Port of the SGLang server"
        },
        "crowd-pilot.basePath": {
          "type": "string",
          "default": "/v1/chat/completions",
          "description": "Base path for the SGLang API endpoint"
        },
        "crowd-pilot.modelName": {
          "type": "string",
          "default": "qwen/qwen3-8b",
          "description": "Model name to use for completions"
        },
        "crowd-pilot.minAvgLogprob": {
          "type": "number",
          "default": -1.0,
          "description": "Minimum average log-probability per token for displaying suggestions. Higher values (closer to 0) require more confidence. -1.0 ≈ perplexity 2.7"
        }
      }
    },
    "keybindings": [
      {
        "command": "crowd-pilot.modelRun",
        "key": "tab",
        "mac": "tab",
        "when": "editorTextFocus && crowdPilot.uiVisible"
      },
      {
        "command": "crowd-pilot.modelRun",
        "key": "tab",
        "mac": "tab",
        "when": "inQuickOpen && crowdPilot.uiVisible"
      },
      {
        "command": "crowd-pilot.hideUi",
        "key": "escape",
        "mac": "escape",
        "when": "crowdPilot.uiVisible"
      }
    ]
  },
  "scripts": {
    "vscode:prepublish": "npm run compile",
    "compile": "tsc -p ./",
    "watch": "tsc -watch -p ./",
    "pretest": "npm run compile && npm run lint",
    "lint": "eslint src",
    "test": "vscode-test"
  },
  "devDependencies": {
    "@types/vscode": "^1.99.3",
    "@types/mocha": "^10.0.10",
    "@types/node": "22.x",
    "@typescript-eslint/eslint-plugin": "^8.45.0",
    "@typescript-eslint/parser": "^8.45.0",
    "eslint": "^9.36.0",
    "typescript": "^5.9.3",
    "@vscode/test-cli": "^0.0.11",
    "@vscode/test-electron": "^2.5.2"
  }
}