p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] crates/core/src/pipeline.rs (rust)
            // Build messages array: system message first, then conversation messages
            let mut messages = Vec::new();
            
            // Add system message
            messages.push(HuggingFaceMessage {
                role: "system".to_string(),
                content: system_prompt.to_string(),
            });
            
            // Convert conversation messages to HuggingFace format
            for m in &conv.messages {
                let role = match m.from.as_str() {
                    "User" => "user",
                    "Assistant" => "assistant",
                    _ => continue, // Skip unknown message types
                };
                
                messages.push(HuggingFaceMessage {
                    role: role.to_string(),
                    content: m.value.clone(),
                });
            }
[content] crates/core/src/pipeline.rs (rust)
            let nemo_messages: Vec<NemoMessage> = conv
                .messages
                .iter()
                .map(|m| NemoMessage {
                    from: m.from.clone(),
                    value: m.value.clone(),
                })
                .collect();
[content] crates/core/src/pipeline.rs (rust)
            let mut messages = Vec::new();
            
            messages.push(HuggingFaceMessage {
                role: "system".to_string(),
                content: system_prompt.to_string(),
            });
            
            for m in &conv.messages {
                let role = match m.from.as_str() {
                    "User" => "user",
                    "Assistant" => "assistant",
                    other => {
                        eprintln!("Warning: Unknown message type '{}', skipping", other);
                        continue;
                    }
                };
                
                messages.push(HuggingFaceMessage {
                    role: role.to_string(),
                    content: m.value.clone(),
                });
            }
[content] crates/core/src/pipeline.rs (rust)
            let mut messages = Vec::new();
            
            messages.push(HuggingFaceMessage {
                role: "system".to_string(),
                content: system_prompt.to_string(),
            });
            
            for m in &conv.messages {
                let role = match m.from.as_str() {
                    "User" => "user",
                    "Assistant" => "assistant",
                    _ => continue, // Skip unknown message types
                };
                
                messages.push(HuggingFaceMessage {
                    role: role.to_string(),
                    content: m.value.clone(),
                });
            }
[content] crates/core/src/pipeline.rs (rust)
            // Build messages: system prompt first, then conversation messages
            let mut messages = vec![HuggingFaceMessage {
                role: "system".to_string(),
                content: system_prompt.to_string(),
            }];
            
            messages.extend(conv.messages.iter().map(|m| HuggingFaceMessage {
                role: m.role,
                content: m.content.clone(),
            }));
[content] crates/core/src/pipeline.rs (rust)
            let nemo_messages: Vec<NemoMessage> = conv
                .messages
                .iter()
                .map(|m| NemoMessage {
                    from: m.from.clone(),
                    value: m.value.clone(),
                })
                .collect();

            let record = NemoRecord {
                mask: "User".to_string(),
                system: system_prompt.to_string(),
                conversations: nemo_messages,
            };
[content] crates/core/src/pipeline.rs (rust)
            // Build messages: system prompt first, then conversation messages
            let mut messages = vec![HuggingFaceMessage {
                role: Role::System,
                content: system_prompt.to_string(),
            }];
            
            messages.extend(conv.messages.iter().map(|m| HuggingFaceMessage {
                role: m.role,
                content: m.content.clone(),
            }));
[tab] README.md (markdown)
# crowd-pilot-serializer

Core serialization library for crowd-pilot's IDE interaction data. Converts IDE events (tab switches, edits, terminal commands, etc.) into conversation format for training language models.

## Architecture

This is a Rust library with:
- **Node.js/TypeScript bindings** (via napi-rs) - for the VS Code extension (uses character approximation for token counting)
- **CLI binary** - for batch preprocessing (uses HuggingFace tokenizer via embedded Python)

The serialization logic is the single source of truth, ensuring consistency between runtime inference and training data preprocessing.

## Crates

- `crates/core` - Core serialization logic
- `crates/napi` - Node.js bindings (`@crowd-pilot/serializer` npm package)
- `crates/cli` - CLI binary for preprocessing (`crowd-pilot-serialize`)

## Building

### Prerequisites

- Rust 1.70+
- Node.js 18+ (for napi bindings)
- Python 3.9+ with `transformers` installed (for CLI tokenizer)

### Build all

```bash
cargo build --release
```

### Build Node.js bindings

```bash
cd crates/napi
npm install
npm run build
```

### Build CLI

```bash
cargo build --release -p crowd-pilot-serialize
```

## Usage

### Node.js/TypeScript (Usage in crowd-pilot-extension)

```typescript
import { ConversationStateManager } from '@crowd-pilot/serializer';

const manager = new ConversationStateManager({
  viewportRadius: 10,
  coalesceRadius: 5,
  maxTokensPerMessage: 2048,
  maxTokensPerTerminalOutput: 256,
});

manager.handleTabEvent('/path/to/file.ts', 'file contents...');
manager.handleContentEvent('/path/to/file.ts', 10, 0, 'inserted text');

const messages = manager.finalizeForModel();
```

### CLI (Preprocessing)

```bash
crowd-pilot-serialize \
    --csv-root ./data/sessions \
    --output-dir ./output \
    --tokenizer "Qwen/Qwen2-7B" \
    --max-tokens-per-conversation 8192 \
    --max-tokens-per-message 2048 \
    --val-ratio 0.1
```

This reads CSV session files, processes them through the Rust serializer, and outputs `training.jsonl` and `validation.jsonl` in Miles conversation format.

#### CLI Options

| Option | Default | Description |
|--------|---------|-------------|
| `--csv-root` | required | Root directory containing per-session CSV files |
| `--output-dir` | required | Output directory for JSONL files |
| `--tokenizer` | required | HuggingFace tokenizer name or path |
| `--max-tokens-per-conversation` | 8192 | Maximum tokens per conversation chunk |
| `--max-tokens-per-message` | 2048 | Maximum tokens per message |
| `--max-tokens-per-terminal-output` | 256 | Maximum tokens for terminal output |
| `--min-conversation-messages` | 5 | Minimum messages to keep a conversation |
| `--viewport-radius` | 10 | Lines above/below cursor to show |
| `--coalesce-radius` | 5 | Radius for grouping nearby edits |
| `--val-ratio` | 0.10 | Fraction of sessions for validation |

## License

Apache 2.0
[tab] .github/workflows/ci.yml (yaml)
name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

env:
  CARGO_TERM_COLOR: always

jobs:
  test:
    name: Test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - name: Run tests
        run: cargo test --all

  lint:
    name: Lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy
      - uses: Swatinem/rust-cache@v2
      - name: Check formatting
        run: cargo fmt --all -- --check
      - name: Clippy
        run: cargo clippy --all -- -D warnings

  build-napi:
    name: Build Node.js (${{ matrix.target }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            target: x86_64-unknown-linux-gnu
          - os: ubuntu-latest
            target: aarch64-unknown-linux-gnu
          - os: macos-latest
            target: x86_64-apple-darwin
          - os: macos-latest
            target: aarch64-apple-darwin
          - os: windows-latest
            target: x86_64-pc-windows-msvc
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
        with:
          targets: ${{ matrix.target }}
      - uses: Swatinem/rust-cache@v2
      - uses: actions/setup-node@v4
        with:
          node-version: 20
      - name: Install dependencies
        working-directory: crates/napi
        run: npm install
      - name: Build
        working-directory: crates/napi
        run: npm run build -- --target ${{ matrix.target }}
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: bindings-${{ matrix.target }}
          path: crates/napi/*.node

  build-python:
    name: Build Python (${{ matrix.target }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            target: x86_64
          - os: ubuntu-latest
            target: aarch64
          - os: macos-latest
            target: x86_64
          - os: macos-latest
            target: aarch64
          - os: windows-latest
            target: x64
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install maturin
        run: pip install maturin
      - name: Build wheel
        working-directory: crates/python
        run: maturin build --release
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: wheel-${{ matrix.os }}-${{ matrix.target }}
          path: crates/python/target/wheels/*.whl