p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] src/types.ts (typescript)
/**
 * crowd-code 2.0 Type Definitions
 * Observation-Action schema for state-based capture
 */


export type ConsentStatus = 'pending' | 'accepted' | 'declined'

export type ActionSource = 'user' | 'external' | 'git' | 'git_checkout'

export interface CursorPosition {
	line: number
	character: number
}

export interface ViewportState {
	file: string
	startLine: number
	endLine: number
	content: string
	cursorPosition: CursorPosition | null
}

export interface TerminalViewport {
	id: string
	name: string
	viewport: string[]
}

export interface Observation {
	viewport: ViewportState | null
	activeTerminal: TerminalViewport | null
}

export interface EditDiff {
	rangeOffset: number
	rangeLength: number
	text: string
}

export interface EditAction {
	kind: 'edit'
	source: ActionSource
	file: string
	diff: EditDiff
}

export interface SelectionAction {
	kind: 'selection'
	source: ActionSource
	file: string
	selectionStart: CursorPosition
	selectionEnd: CursorPosition
	selectedText: string
}

export interface TabSwitchAction {
	kind: 'tab_switch'
	source: ActionSource
	file: string
	previousFile: string | null
}

export interface TerminalFocusAction {
	kind: 'terminal_focus'
	source: ActionSource
	terminalId: string
	terminalName: string
}

export interface TerminalCommandAction {
	kind: 'terminal_command'
	source: ActionSource
	terminalId: string
	terminalName: string
	command: string
}

export interface TerminalOutputAction {
	kind: 'terminal_output'
	source: ActionSource
	terminalId: string
	terminalName: string
	output: string
}

export interface FileChangeAction {
	kind: 'file_change'
	source: ActionSource
	file: string
	changeType: 'create' | 'change' | 'delete'
	diff: string | null
}

export interface ScrollAction {
	kind: 'scroll'
	source: ActionSource
	file: string
}

export type Action =
	| EditAction
	| SelectionAction
	| TabSwitchAction
	| TerminalFocusAction
	| TerminalCommandAction
	| TerminalOutputAction
	| FileChangeAction
	| ScrollAction

export interface ObservationEvent {
	sequence: number
	timestamp: number
	type: 'observation'
	observation: Observation
}

export interface ActionEvent {
	sequence: number
	timestamp: number
	type: 'action'
	action: Action
}

export type RecordingEvent = ObservationEvent | ActionEvent

export interface RecordingSession {
	version: '2.0'
	sessionId: string
	startTime: number
	events: RecordingEvent[]
}

export interface RecordingState {
	isRecording: boolean
	startDateTime: Date | null
	endDateTime: Date | null
	sequence: number
	sessionId: string
	events: RecordingEvent[]
}
[content] /home/franz.srambical/test_extension/test/src/extension.ts (typescript)

    // ---- Terminal focus testing ----
    context.subscriptions.push(
        vscode.window.onDidChangeActiveTerminal(terminal => {
            log(
                `TERMINAL_ACTIVE_CHANGE
  terminal=${terminal ? terminal.name : 'undefined'}
  processId=${terminal?.processId ?? 'N/A'}
  activeTerminal=${vscode.window.activeTerminal?.name ?? 'undefined'}`
            )
        })
    )

    // ---- Editor focus testing ----
    context.subscriptions.push(
        vscode.window.onDidChangeActiveTextEditor(editor => {
            log(
                `EDITOR_ACTIVE_CHANGE
  editor=${editor ? vscode.workspace.asRelativePath(editor.document.fileName) : 'undefined'}
  activeTerminal=${vscode.window.activeTerminal?.name ?? 'undefined'}`
            )
        })
    )

    // ---- Terminal shell execution ----
    context.subscriptions.push(
        vscode.window.onDidStartTerminalShellExecution(event => {
            log(
                `TERMINAL_EXEC_START
  terminal=${event.terminal.name}
  command=${event.execution.commandLine.value}`
            )
        })
    )
[tab] src/capture/index.ts (typescript)
/**
 * Capture Module Index
 * Exports all capture-related functionality
 */

// Viewport capture
export {
	captureViewportState,
	captureObservation,
	captureNow,
	initializeViewportCapture,
	cleanupViewportCapture,
	resetObservationState
} from './viewportCapture'

// Terminal capture
export {
	getActiveTerminalViewport,
	initializeTerminalCapture,
	cleanupTerminalCapture,
	resetTerminalState
} from './terminalCapture'
export type { TerminalCallbacks, TerminalViewport } from './terminalCapture'

// Filesystem watcher
export {
	initializeFilesystemWatcher,
	cleanupFilesystemWatcher,
	resetFilesystemState
} from './filesystemWatcher'
export type { FileChangeCallback } from './filesystemWatcher'
[content] src/capture/viewportCapture.ts (typescript)
export function captureNow(): Observation | null {
	const observation = captureObservation()
	const hash = computeObservationHash(observation)

	if (hash === lastObservationHash) return null

	lastObservationHash = hash
	return observation
[content] src/capture/viewportCapture.ts (typescript)
/**
 * Compute a hash of the viewport state for deduplication
 */
function computeObservationHash(observation: Observation): string {
	const data = JSON.stringify({
		viewport: observation.viewport,
		terminalId: observation.activeTerminal?.id ?? null
	})
	return crypto.createHash('md5').update(data).digest('hex')
}
[content] src/capture/terminalCapture.ts (typescript)
	const combined = existing + content
	const lines = combined.split('
')
	
	// Keep only the last TERMINAL_VIEWPORT_LINES lines
	if (lines.length > TERMINAL_VIEWPORT_LINES) {
		terminalContent.set(terminalId, lines.slice(-TERMINAL_VIEWPORT_LINES).join('
'))
	} else {
		terminalContent.set(terminalId, combined)
	}
[tab] src/capture/viewportCapture.ts (typescript)
/**
 * Viewport Capture Module
 * Captures the current visible editor state (viewport content, cursor position, line ranges)
 */

import * as vscode from 'vscode'
import type { ViewportState, Observation } from '../types'
import { getActiveTerminalViewport } from './terminalCapture'

const POLL_INTERVAL_MS = 100 // 10Hz

let viewportChanged = false

let onObservationCallback: ((observation: Observation) => void) | null = null

let visibleRangesDisposable: vscode.Disposable | null = null
let pollInterval: NodeJS.Timeout | null = null

/**
 * Capture the current viewport state from the active editor
 */
export function captureViewportState(): ViewportState | null {
	const editor = vscode.window.activeTextEditor
	if (!editor) {
		return null
	}

	const document = editor.document
	const visibleRanges = editor.visibleRanges

	if (visibleRanges.length === 0) {
		return null
	}

	const firstRange = visibleRanges[0]
	const lastRange = visibleRanges[visibleRanges.length - 1]
	
	const startLine = firstRange.start.line
	const endLine = lastRange.end.line

	const fullVisibleRange = new vscode.Range(
		new vscode.Position(startLine, 0),
		new vscode.Position(endLine, document.lineAt(endLine).text.length)
	)

	const content = document.getText(fullVisibleRange)

	const selection = editor.selection
	const cursorPosition = selection ? {
		line: selection.active.line,
		character: selection.active.character
	} : null

	const file = vscode.workspace.asRelativePath(document.fileName)

	return {
		file,
		startLine: startLine + 1,
		endLine: endLine + 1,
		content,
		cursorPosition
	}
}

/**
 * Capture a full observation (viewport + active terminal viewport)
 */
export function captureObservation(): Observation {
	const viewport = captureViewportState()
	const activeTerminal = getActiveTerminalViewport()

	return {
		viewport,
		activeTerminal
	}
}

/**
 * Poll handler - captures observation if viewport changed
 */
function pollViewport(): void {
	if (!viewportChanged) {
		return
	}
	viewportChanged = false

	if (!onObservationCallback) {
		return
	}

	onObservationCallback(captureObservation())
}

/**
 * Initialize the viewport capture module
 */
export function initializeViewportCapture(
	context: vscode.ExtensionContext,
	onScrollObservation: (observation: Observation) => void
): void {
	onObservationCallback = onScrollObservation

	visibleRangesDisposable = vscode.window.onDidChangeTextEditorVisibleRanges(() => {
		viewportChanged = true
	})
	context.subscriptions.push(visibleRangesDisposable)

	pollInterval = setInterval(pollViewport, POLL_INTERVAL_MS)
}

/**
 * Cleanup the viewport capture module
 */
export function cleanupViewportCapture(): void {
	if (visibleRangesDisposable) {
		visibleRangesDisposable.dispose()
		visibleRangesDisposable = null
	}
	if (pollInterval) {
		clearInterval(pollInterval)
		pollInterval = null
	}
	viewportChanged = false
}

/**
 * Reset the observation state (useful when switching recordings)
 */
export function resetObservationState(): void {
	viewportChanged = false
}
[tab] src/gitProvider.ts (typescript)
/**
 * Git Provider for crowd-code 2.0
 * Detects git operations to annotate filesystem changes
 */

import * as vscode from 'vscode'
import { logToOutput } from './utilities'

// Track recent git operations for filesystem change attribution
let sawHeadChange = false
let lastGitOperationTime = 0
const GIT_OPERATION_WINDOW_MS = 5000

// File system watchers
let gitHeadWatcher: vscode.FileSystemWatcher | undefined
let gitRefsWatcher: vscode.FileSystemWatcher | undefined

/**
 * Check if there was a recent git operation
 * Returns 'git_checkout' if HEAD changed, 'git' for other operations, null otherwise
 */
export function getRecentGitOperation(): 'git' | 'git_checkout' | null {
	if (Date.now() - lastGitOperationTime > GIT_OPERATION_WINDOW_MS) {
		return null
	}
	const result = sawHeadChange ? 'git_checkout' : 'git'
	sawHeadChange = false
	return result
}

/**
 * Setup git file watchers for the current workspace
 */
function setupGitWatchers(): void {
	// Cleanup any existing watchers first
	disposeWatchers()

	const workspaceFolder = vscode.workspace.workspaceFolders?.[0]
	if (!workspaceFolder) {
		logToOutput('No workspace folder found', 'info')
		return
	}

	const gitDir = vscode.Uri.joinPath(workspaceFolder.uri, '.git')
	vscode.workspace.fs.stat(gitDir).then(
		() => {
			logToOutput('Git repository found', 'info')

			// Watch .git/HEAD for branch changes
			gitHeadWatcher = vscode.workspace.createFileSystemWatcher(
				new vscode.RelativePattern(workspaceFolder, '.git/HEAD')
			)
			gitHeadWatcher.onDidChange(() => {
				logToOutput('Git checkout detected', 'info')
				sawHeadChange = true
				lastGitOperationTime = Date.now()
			})

			// Watch .git/refs for other git operations (pull, stash, etc.)
			gitRefsWatcher = vscode.workspace.createFileSystemWatcher(
				new vscode.RelativePattern(workspaceFolder, '.git/refs/**/*')
			)
			gitRefsWatcher.onDidChange(() => {
				logToOutput('Git refs changed', 'info')
				lastGitOperationTime = Date.now()
			})
			gitRefsWatcher.onDidCreate(() => {
				logToOutput('Git refs created', 'info')
				lastGitOperationTime = Date.now()
			})
			gitRefsWatcher.onDidDelete(() => {
				logToOutput('Git refs deleted', 'info')
				lastGitOperationTime = Date.now()
			})

			logToOutput('Git provider initialized', 'info')
		},
		() => {
			logToOutput('Not a git repository', 'info')
		}
	)
}

function disposeWatchers(): void {
	gitHeadWatcher?.dispose()
	gitHeadWatcher = undefined
	gitRefsWatcher?.dispose()
	gitRefsWatcher = undefined
}

/**
 * Initialize the git provider
 */
export function initializeGitProvider(context: vscode.ExtensionContext): void {
	logToOutput('Initializing git provider...', 'info')

	setupGitWatchers()

	// Reinitialize on workspace changes
	context.subscriptions.push(
		vscode.workspace.onDidChangeWorkspaceFolders(() => {
			logToOutput('Workspace changed, reinitializing git provider...', 'info')
			setupGitWatchers()
		})
	)
}

/**
 * Cleanup the git provider
 */
export function cleanupGitProvider(): void {
	disposeWatchers()
	sawHeadChange = false
	lastGitOperationTime = 0
}
[tab] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_4k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 4096 \
--tokenizer="Qwen/Qwen3-8B"
[tab] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_8k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 8192 \
--tokenizer="Qwen/Qwen3-8B"
[tab] /fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/metadata.json (json)
{
  "config": {
    "coalesce_radius": 5,
    "csv_root": "/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/",
    "max_tokens_per_conversation": 4096,
    "max_tokens_per_message": 2048,
    "min_conversation_messages": 5,
    "output_dir": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/",
    "tokenizer": "Qwen/Qwen3-8B",
    "val_ratio": 0.1,
    "viewport_radius": 10
  },
  "counts": {
    "total_conversations": 10514,
    "total_sessions": 349,
    "train_conversations": 9569,
    "val_conversations": 945
  },
  "files": {
    "train_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/training.jsonl",
    "val_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/validation.jsonl"
  },
  "stats": {
    "avg_messages_per_conversation": 12.480311964999048,
    "avg_tokens_per_conversation": 3519.9015598249953,
    "total_messages": 131218,
    "total_tokens": 37008245
  }
}
[tab] /fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_8k_tokens/metadata.json (json)
{
  "config": {
    "coalesce_radius": 5,
    "csv_root": "/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/",
    "max_tokens_per_conversation": 8192,
    "max_tokens_per_message": 2048,
    "min_conversation_messages": 5,
    "output_dir": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_8k_tokens/",
    "tokenizer": "Qwen/Qwen3-8B",
    "val_ratio": 0.1,
    "viewport_radius": 10
  },
  "counts": {
    "total_conversations": 4446,
    "total_sessions": 349,
    "train_conversations": 4043,
    "val_conversations": 403
  },
  "files": {
    "train_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_8k_tokens/training.jsonl",
    "val_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_8k_tokens/validation.jsonl"
  },
  "stats": {
    "avg_messages_per_conversation": 29.13697705802969,
    "avg_tokens_per_conversation": 7527.038461538462,
    "total_messages": 129543,
    "total_tokens": 33465213
  }
}
[content] /fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/metadata.json (json)
    "total_conversations": 12942,
    "total_sessions": 349,
    "train_conversations": 11760,
    "val_conversations": 1182
  },
  "files": {
    "train_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/training.jsonl",
    "val_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/validation.jsonl"
  },
  "stats": {
    "avg_messages_per_conversation": 10.277005099675476,
    "avg_tokens_per_conversation": 3705.9932004326997,
    "total_messages": 133005,
    "total_tokens": 47962964
[content] src/capture/viewportCapture.ts (typescript)
/**
 * Viewport Capture Module
 * Captures the current visible editor state (viewport content, cursor position, line ranges)
 */

import * as vscode from 'vscode'
import type { ViewportState, Observation } from '../types'
import { getActiveTerminalViewport } from './terminalCapture'

const POLL_INTERVAL_MS = 100 // 10Hz

let viewportChanged = false

let onObservationCallback: ((observation: Observation) => void) | null = null

let visibleRangesDisposable: vscode.Disposable | null = null
let pollInterval: NodeJS.Timeout | null = null

/**
 * Capture the current viewport state from the active editor
 */
export function captureViewportState(): ViewportState | null {
	const editor = vscode.window.activeTextEditor
	if (!editor) {
		return null
	}

	const document = editor.document
	const visibleRanges = editor.visibleRanges

	if (visibleRanges.length === 0) {
		return null
	}

	const firstRange = visibleRanges[0]
	const lastRange = visibleRanges[visibleRanges.length - 1]
	
	const startLine = firstRange.start.line
	const endLine = lastRange.end.line

	const fullVisibleRange = new vscode.Range(
		new vscode.Position(startLine, 0),
		new vscode.Position(endLine, document.lineAt(endLine).text.length)
	)

	const content = document.getText(fullVisibleRange)

	const selection = editor.selection
	const cursorPosition = selection ? {
		line: selection.active.line,
		character: selection.active.character
	} : null

	const file = vscode.workspace.asRelativePath(document.fileName)

	return {
		file,
		startLine: startLine + 1,
		endLine: endLine + 1,
		content,
		cursorPosition
	}
}

/**
 * Capture a full observation (viewport + active terminal viewport)
 */
export function captureObservation(): Observation {
	const viewport = captureViewportState()
	const activeTerminal = getActiveTerminalViewport()

	return {
		viewport,
		activeTerminal
	}
}

/**
 * Poll handler - captures observation if viewport changed
 */
function pollViewport(): void {
	if (!viewportChanged) {
		return
	}
	viewportChanged = false

	if (!onObservationCallback) {
		return
	}

	onObservationCallback(captureObservation())
}

/**
 * Initialize the viewport capture module
 */
export function initializeViewportCapture(
	context: vscode.ExtensionContext,
	onScrollObservation: (observation: Observation) => void
): void {
	onObservationCallback = onScrollObservation

	visibleRangesDisposable = vscode.window.onDidChangeTextEditorVisibleRanges(() => {
		viewportChanged = true
	})
	context.subscriptions.push(visibleRangesDisposable)

	pollInterval = setInterval(pollViewport, POLL_INTERVAL_MS)
}

/**
 * Cleanup the viewport capture module
 */
export function cleanupViewportCapture(): void {
	if (visibleRangesDisposable) {
		visibleRangesDisposable.dispose()
		visibleRangesDisposable = null
	}
	if (pollInterval) {
		clearInterval(pollInterval)
		pollInterval = null
	}
	viewportChanged = false
}

/**
 * Reset the observation state (useful when switching recordings)
 */
export function resetObservationState(): void {
	viewportChanged = false
}