p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] /home/franz.srambical/crowd-pilot-serializer/crates/python/Cargo.toml (plaintext)
[package]
name = "crowd-pilot-serializer-python"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Python bindings for crowd-pilot serializer"

[lib]
name = "crowd_pilot_serializer"
crate-type = ["cdylib"]

[dependencies]
crowd-pilot-serializer-core = { workspace = true }
pyo3 = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
[tab] /home/franz.srambical/crowd-pilot-serializer/crates/core/Cargo.toml (plaintext)
[package]
name = "crowd-pilot-serializer-core"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Core serialization logic for crowd-pilot's IDE interaction data"

[dependencies]
regex = { workspace = true }
thiserror = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
csv = "1.3"
walkdir = "2.5"

[dev-dependencies]
tempfile = "3.10"
[tab] crowd-pilot-extension/LICENSE (plaintext)
Copyright 2025 Franz Srambical/p(doom)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
[tab] /home/franz.srambical/crowd-pilot-serializer/Cargo.toml (plaintext)
[workspace]
resolver = "2"
members = [
    "crates/core",
    "crates/napi",
    "crates/cli",
]

[workspace.package]
version = "0.1.0"
edition = "2021"
license = "MIT"
repository = "https://github.com/p-doom/crowd-pilot-serializer"

[workspace.dependencies]
# Core dependencies
regex = "1.10"
thiserror = "1.0"

# Serialization
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"

# Python bindings
pyo3 = { version = "0.20", features = ["extension-module"] }

# The core crate
crowd-pilot-serializer-core = { path = "crates/core" }
[tab] crowd-pilot-extension/.vscode/tasks.json (jsonc)
// See https://go.microsoft.com/fwlink/?LinkId=733558
// for the documentation about the tasks.json format
{
	"version": "2.0.0",
	"tasks": [
		{
			"label": "watch",
			"type": "shell",
			"command": "module load nodejs && npm run watch",
			"problemMatcher": "$tsc-watch",
			"isBackground": true,
			"presentation": {
				"reveal": "never"
			},
			"group": {
				"kind": "build",
				"isDefault": true
			}
		}
	]
}
[tab] crowd-pilot-extension/.vscode/tasks.json (jsonc)
// See https://go.microsoft.com/fwlink/?LinkId=733558
// for the documentation about the tasks.json format
{
	"version": "2.0.0",
	"tasks": [
		{
			"label": "watch",
			"type": "shell",
			// FIXME (f.srambical): We should make this cluster-agnostic
			"command": "module load nodejs && npm run watch",
			"problemMatcher": "$tsc-watch",
			"isBackground": true,
			"presentation": {
				"reveal": "never"
			},
			"group": {
				"kind": "build",
				"isDefault": true
			}
		}
	]
}
[content] /home/franz.srambical/crowd-pilot-serializer/crates/core/src/conversation.rs (rust)
            let cmd = format!("cat -n {}", file_path);
            self.append_message(ConversationMessage::assistant(fenced_block(
                Some("bash"),
                &clean_text(&cmd),
            )));
            let output = line_numbered_output(&content, None, None);
            self.append_message(ConversationMessage::user(format!(
                "<stdout>
{}
</stdout>",
                output
            )));
            self.files_opened_in_conversation.insert(file_path.to_string());
[content] /home/franz.srambical/crowd-pilot-serializer/README.md (markdown)
### Node.js/TypeScript (Usage in crowd-pilot-extension)

```typescript
import { ConversationStateManager } from '@crowd-pilot/serializer';

const manager = new ConversationStateManager({
  viewportRadius: 10,
  coalesceRadius: 5,
  maxTokensPerMessage: 2048,
  maxTokensPerTerminalOutput: 256,
});

manager.handleTabEvent('/path/to/file.ts', 'file contents...');
manager.handleContentEvent('/path/to/file.ts', 10, 0, 'inserted text');

const messages = manager.finalizeForModel();
```

### CLI (Preprocessing)

```bash
crowd-pilot-serialize \
    --csv-root ./data/sessions \
    --output-dir ./output \
    --tokenizer "Qwen/Qwen2-7B" \
    --max-tokens-per-conversation 8192 \
    --max-tokens-per-message 2048 \
    --val-ratio 0.1
```

This reads CSV session files, processes them through the Rust serializer, and outputs `training.jsonl` and `validation.jsonl` in NeMo's conversation format.

#### CLI Options

| Option | Default | Description |
|--------|---------|-------------|
| `--csv-root` | required | Root directory containing per-session CSV files |
| `--output-dir` | required | Output directory for JSONL files |
| `--tokenizer` | required | HuggingFace tokenizer name or path |
| `--max-tokens-per-conversation` | 8192 | Maximum tokens per conversation chunk |
| `--max-tokens-per-message` | 2048 | Maximum tokens per message |
| `--max-tokens-per-terminal-output` | 256 | Maximum tokens for terminal output |
| `--min-conversation-messages` | 5 | Minimum messages to keep a conversation |
| `--viewport-radius` | 10 | Lines above/below cursor to show |
| `--coalesce-radius` | 5 | Radius for grouping nearby edits |
| `--val-ratio` | 0.10 | Fraction of sessions for validation |

## License

Apache 2.0
[content] crowd-pilot-extension/src/extension.ts (typescript)
			const currentFileUri = editor.document.uri.toString();
			let tabEventText: string | null = null;

			if (!activatedFiles.has(currentFileUri)) {
				tabEventText = editor.document.getText();
				activatedFiles.add(currentFileUri);
			}

			conversationManager.handleTabEvent(filePath, tabEventText);
[content] crowd-pilot-extension/src/extension.ts (typescript)
		const currentFileUri = initialEditor.document.uri.toString();
		const tabEventText = initialEditor.document.getText();
		activatedFiles.add(currentFileUri);
		conversationManager.handleTabEvent(filePath, tabEventText);