p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] /home/franz.srambical/crowd-pilot-serializer/Cargo.toml (plaintext)
[workspace]
resolver = "2"
members = [
    "crates/core",
    "crates/napi",
    "crates/python",
]

[workspace.package]
version = "0.1.0"
edition = "2021"
license = "MIT"
repository = "https://github.com/your-org/crowd-pilot-serializer"

[workspace.dependencies]
# Core dependencies
regex = "1.10"
thiserror = "1.0"

# Serialization
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"

# The core crate
crowd-pilot-serializer-core = { path = "crates/core" }
[tab] /home/franz.srambical/crowd-pilot-serializer/crates/napi/Cargo.toml (plaintext)
[package]
name = "crowd-pilot-serializer-napi"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Node.js bindings for crowd-pilot serializer"

[lib]
crate-type = ["cdylib"]

[dependencies]
crowd-pilot-serializer-core = { workspace = true }
napi = { version = "2", features = ["napi4"] }
napi-derive = "2"
serde = { workspace = true }
serde_json = { workspace = true }

[build-dependencies]
napi-build = "2"
[tab] /home/franz.srambical/crowd-pilot-serializer/README.md (markdown)
# crowd-pilot-serializer

Core serialization library for crowd-pilot IDE interaction data. Converts IDE events (tab switches, edits, terminal commands, etc.) into conversation format for training language models.

## Architecture

This is a Rust library with bindings for:
- **Node.js/TypeScript** (via napi-rs) - for the VS Code extension
- **Python** (via PyO3) - for batch preprocessing

The serialization logic is the single source of truth, ensuring consistency between runtime inference and training data preprocessing.

## Crates

- `crates/core` - Core serialization logic
- `crates/napi` - Node.js bindings (`@crowd-pilot/serializer` npm package)
- `crates/python` - Python bindings (`crowd-pilot-serializer` PyPI package)

## Building

### Prerequisites

- Rust 1.70+
- Node.js 18+ (for napi bindings)
- Python 3.9+ (for Python bindings)
- maturin (`pip install maturin`)

### Build all

```bash
cargo build --release
```

### Build Node.js bindings

```bash
cd crates/napi
npm install
npm run build
```

### Build Python bindings

```bash
cd crates/python
maturin develop
```

## Usage

### Node.js/TypeScript

```typescript
import { ConversationStateManager } from '@crowd-pilot/serializer';

const manager = new ConversationStateManager({
  countTokens: (text) => Math.ceil(text.length / 4),
  maxTokensPerConversation: 8192,
  maxTokensPerMessage: 2048,
});

manager.handleTabEvent('/path/to/file.ts', 'file contents...');
manager.handleContentEvent('/path/to/file.ts', 10, 0, 'inserted text');

const messages = manager.finalizeForModel();
```

### Python

```python
from crowd_pilot_serializer import ConversationStateManager
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-7B")

manager = ConversationStateManager(
    count_tokens=lambda text: len(tokenizer.encode(text)),
    max_tokens_per_conversation=8192,
    max_tokens_per_message=2048,
)

manager.handle_tab_event("/path/to/file.ts", "file contents...")
manager.handle_content_event("/path/to/file.ts", 10, 0, "inserted text")

messages = manager.finalize_for_model()
```

## License

MIT
[tab] /home/franz.srambical/crowd-pilot-serializer/Cargo.toml (plaintext)
[workspace]
resolver = "2"
members = [
    "crates/core",
    "crates/napi",
    "crates/python",
]

[workspace.package]
version = "0.1.0"
edition = "2021"
license = "MIT"
repository = "https://github.com/p-doom/crowd-pilot-serializer"

[workspace.dependencies]
# Core dependencies
regex = "1.10"
thiserror = "1.0"

# Serialization
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"

# The core crate
crowd-pilot-serializer-core = { path = "crates/core" }
[tab] /home/franz.srambical/crowd-pilot-serializer/crates/core/Cargo.toml (plaintext)
[package]
name = "crowd-pilot-serializer-core"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Core serialization logic for crowd-pilot IDE interaction data"

[dependencies]
regex = { workspace = true }
thiserror = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
[tab] /home/franz.srambical/crowd-pilot-serializer/crates/python/Cargo.toml (plaintext)
[package]
name = "crowd-pilot-serializer-python"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Python bindings for crowd-pilot serializer"

[lib]
name = "crowd_pilot_serializer"
crate-type = ["cdylib"]

[dependencies]
crowd-pilot-serializer-core = { workspace = true }
pyo3 = { version = "0.20", features = ["extension-module"] }
serde = { workspace = true }
serde_json = { workspace = true }
[tab] /home/franz.srambical/crowd-pilot-serializer/crates/napi/package.json (json)
{
  "name": "@crowd-pilot/serializer",
  "version": "0.1.0",
  "description": "Node.js bindings for crowd-pilot serializer",
  "main": "index.js",
  "types": "index.d.ts",
  "napi": {
    "name": "serializer",
    "triples": {
      "defaults": true,
      "additional": [
        "aarch64-apple-darwin",
        "aarch64-unknown-linux-gnu",
        "aarch64-unknown-linux-musl",
        "x86_64-unknown-linux-musl"
      ]
    }
  },
  "files": [
    "index.js",
    "index.d.ts"
  ],
  "devDependencies": {
    "@napi-rs/cli": "^2.18.0"
  },
  "scripts": {
    "artifacts": "napi artifacts",
    "build": "napi build --platform --release",
    "build:debug": "napi build --platform",
    "prepublishOnly": "napi prepublish -t npm",
    "universal": "napi universal",
    "version": "napi version"
  },
  "license": "MIT",
  "repository": {
    "type": "git",
    "url": "https://github.com/your-org/crowd-pilot-serializer"
  }
}
[tab] /home/franz.srambical/crowd-pilot-serializer/crates/python/pyproject.toml (plaintext)
[build-system]
requires = ["maturin>=1.4,<2.0"]
build-backend = "maturin"

[project]
name = "crowd-pilot-serializer"
version = "0.1.0"
description = "Python bindings for crowd-pilot serializer"
readme = "README.md"
license = { text = "MIT" }
requires-python = ">=3.9"
classifiers = [
    "Programming Language :: Rust",
    "Programming Language :: Python :: Implementation :: CPython",
    "Programming Language :: Python :: Implementation :: PyPy",
]

[tool.maturin]
features = ["pyo3/extension-module"]
module-name = "crowd_pilot_serializer"
[tab] /home/franz.srambical/crowd-pilot-serializer/.gitignore (ignore)
# Rust
/target/
Cargo.lock

# Node.js
node_modules/
*.node
*.js
*.d.ts
!crates/napi/index.js
!crates/napi/index.d.ts

# Python
__pycache__/
*.py[cod]
*$py.class
*.so
*.egg-info/
dist/
build/
.eggs/

# IDE
.idea/
.vscode/
*.swp
*.swo

# OS
.DS_Store
Thumbs.db
[tab] /home/franz.srambical/crowd-pilot-serializer/.github/workflows/ci.yml (yaml)
name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

env:
  CARGO_TERM_COLOR: always

jobs:
  test:
    name: Test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - name: Run tests
        run: cargo test --all

  lint:
    name: Lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy
      - uses: Swatinem/rust-cache@v2
      - name: Check formatting
        run: cargo fmt --all -- --check
      - name: Clippy
        run: cargo clippy --all -- -D warnings

  build-napi:
    name: Build Node.js (${{ matrix.target }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            target: x86_64-unknown-linux-gnu
          - os: ubuntu-latest
            target: aarch64-unknown-linux-gnu
          - os: macos-latest
            target: x86_64-apple-darwin
          - os: macos-latest
            target: aarch64-apple-darwin
          - os: windows-latest
            target: x86_64-pc-windows-msvc
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@stable
        with:
          targets: ${{ matrix.target }}
      - uses: Swatinem/rust-cache@v2
      - uses: actions/setup-node@v4
        with:
          node-version: 20
      - name: Install dependencies
        working-directory: crates/napi
        run: npm install
      - name: Build
        working-directory: crates/napi
        run: npm run build -- --target ${{ matrix.target }}
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: bindings-${{ matrix.target }}
          path: crates/napi/*.node

  build-python:
    name: Build Python (${{ matrix.target }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            target: x86_64
          - os: ubuntu-latest
            target: aarch64
          - os: macos-latest
            target: x86_64
          - os: macos-latest
            target: aarch64
          - os: windows-latest
            target: x64
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install maturin
        run: pip install maturin
      - name: Build wheel
        working-directory: crates/python
        run: maturin build --release
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: wheel-${{ matrix.os }}-${{ matrix.target }}
          path: crates/python/target/wheels/*.whl
[tab] /home/franz.srambical/crowd-pilot-serializer/crates/core/src/lib.rs (rust)
//! Core serialization logic for crowd-pilot IDE interaction data.
//!
//! This crate provides the `ConversationStateManager` which converts IDE events
//! (tab switches, edits, terminal commands, etc.) into conversation format
//! suitable for training language models.

mod conversation;
mod diff;
mod helpers;
pub mod pipeline;

pub use conversation::{ConversationMessage, ConversationStateManager, ConversationStateManagerConfig, FinalizedConversation};
pub use pipeline::{
    discover_csv_files, process_all_sessions, process_session, write_jsonl_output,
    NemoMessage, NemoRecord, PipelineConfig, PipelineResult, SessionResult,
};
pub use diff::{compute_changed_block_lines, ChangedBlock};
pub use helpers::{
    apply_backspaces, apply_change, clean_text, escape_single_quotes_for_sed, fenced_block,
    line_numbered_output, normalize_terminal_output, serialize_compute_viewport, Viewport,
};

/// Default viewport radius (lines above/below cursor to show)
pub const VIEWPORT_RADIUS: usize = 10;

/// Default coalesce radius for grouping nearby edits
pub const COALESCE_RADIUS: usize = 5;

/// Default maximum tokens per message (approximate)
pub const MAX_TOKENS_PER_MESSAGE: usize = 2048;

/// Default maximum tokens per terminal output
pub const MAX_TOKENS_PER_TERMINAL_OUTPUT: usize = 256;
[tab] /home/franz.srambical/crowd-pilot-serializer/crates/cli/Cargo.toml (plaintext)
[package]
name = "crowd-pilot-serialize"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "CLI tool for serializing crowd-pilot IDE interaction data"

[[bin]]
name = "crowd-pilot-serialize"
path = "src/main.rs"

[dependencies]
crowd-pilot-serializer-core = { path = "../core" }
clap = { version = "4.5", features = ["derive"] }
pyo3 = { version = "0.20", features = ["auto-initialize"] }  # Different features from workspace
serde_json = { workspace = true }