p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] crowd-pilot/serialize_dataset.py (python)
        offset = int(ro) if pd.notna(ro) else 0
        length = int(rl) if pd.notna(rl) else 0

        # Handle selection/cursor movement events
        if evt_type == "selection_command":
            old_cursor = per_file_cursor_positions.get(file_path, (0, 0))
            new_cursor = (offset, length)
            per_file_cursor_positions[file_path] = new_cursor
            
            # Emit cursor movement observation if position changed
            if old_cursor != new_cursor:
                parts.append(f"<act cursor file=\"{file_path}\" offset=\"{offset}\" len=\"{length}\" />")
            continue

        # Handle file edit events (must have type 'content')
        if evt_type != "content":
            continue
[content] crowd-pilot/serialize_dataset.py (python)
        # Selection/move events (optional tokenization)
        evt_type = str(row.get("Type", ""))
        if evt_type == "selection_command":
            parts.append(f"<act select file=\"{file_path}\" offset=\"{offset}\" len=\"{length}\" />")
[content] crowd-pilot/serialize_dataset.py (python)
                # Update cursor position after edit (cursor moves to end of inserted/replaced text)
                per_file_cursor_positions[file_path] = (offset + len(new_text_str), 0)

                if per_file_event_counts[file_path] % max(1, keyframe_every_events) == 0:
                    emit_keyframe_for(file_path, language)
[content] crowd-pilot/serialize_dataset.py (python)
                before = file_states.get(file_path, "")
                after = _apply_change(before, offset, length, new_text)
                file_states[file_path] = after
                per_file_event_counts[file_path] = per_file_event_counts.get(file_path, 0) + 1
[content] crowd-pilot/serialize_dataset.py (python)
                operation = "noop"
                if length == 0 and new_text_str:
                    operation = "insert"
                elif length > 0 and not new_text_str:
                    operation = "delete"
                elif length > 0 and new_text_str:
                    operation = "replace"
[content] crowd-pilot/serialize_dataset.py (python)
        # Handle different event types
        match event_type:
            case "selection_command":
                # Handle cursor movement
                old_cursor = per_file_cursor_positions.get(file_path, (0, 0))
                new_cursor = (offset, length)
                per_file_cursor_positions[file_path] = new_cursor
                
                # Emit cursor movement observation if position changed
                if old_cursor != new_cursor:
                    parts.append(f"<act cursor file=\"{file_path}\" offset=\"{offset}\" len=\"{length}\" />")
[content] crowd-pilot/serialize_dataset.py (python)
        before = file_states.get(file_path, "")
        after = _apply_change(before, offset, length, new_text)
        file_states[file_path] = after
        per_file_event_counts[file_path] = per_file_event_counts.get(file_path, 0) + 1
[content] crowd-pilot/serialize_dataset.py (python)
        operation = "noop"
        if length == 0 and new_text_str:
            operation = "insert"
        elif length > 0 and not new_text_str:
            operation = "delete"
        elif length > 0 and new_text_str:
            operation = "replace"
[content] crowd-pilot/serialize_dataset.py (python)
        # Handle file edit events (must have type 'content')
        if event_type != "content":
            continue

        new_text = row.get("Text", "")
        new_text_str = str(new_text) if pd.notna(new_text) else ""
[content] crowd-pilot/serialize_dataset.py (python)
        # Handle selection/cursor movement events
        if event_type == "selection_command":
            old_cursor = per_file_cursor_positions.get(file_path, (0, 0))
            new_cursor = (offset, length)
            per_file_cursor_positions[file_path] = new_cursor
            
            # Emit cursor movement observation if position changed
            if old_cursor != new_cursor:
                parts.append(f"<act cursor file=\"{file_path}\" offset=\"{offset}\" len=\"{length}\" />")
            continue
[content] crowd-pilot/serialize_dataset.py (python)
            case "content":
                # Handle file edit events
                offset = row["RangeOffset"]
                length = row["RangeLength"]
                new_text = row.get("Text", "")
                new_text_str = str(new_text) if pd.notna(new_text) else ""
[content] crowd-pilot/serialize_dataset.py (python)
            case "selection_command":
                # Handle cursor movement
                offset = row["RangeOffset"]
                length = row["RangeLength"]
                old_cursor = per_file_cursor_positions.get(file_path, (0, 0))
                new_cursor = (offset, length)
                per_file_cursor_positions[file_path] = new_cursor
                
                # Emit cursor movement observation if position changed
                if old_cursor != new_cursor:
                    parts.append(f"<act cursor file=\"{file_path}\" offset=\"{offset}\" len=\"{length}\" />")
[content] crowd-pilot/serialize_dataset.py (python)
            case "terminal_command":
                # Terminal command execution
                command = row.get("Text", "")
                if pd.notna(command):
                    command_str = str(command).replace("\
", "
").replace("\
", "
")
                    parts.append(f"<act terminal_command />")
                    parts.append(_fenced_block("TERMINAL", "bash", _clean_text(command_str)))

            case "terminal_output":
                # Terminal output capture
                output = row.get("Text", "")
                if pd.notna(output):
                    output_str = str(output).replace("\
", "
").replace("\
", "
")
                    parts.append(f"<obs terminal_output />")
                    parts.append(_fenced_block("TERMINAL", None, _clean_text(output_str)))
[content] crowd-pilot/serialize_dataset.py (python)
        # Get event type and process based on type
        event_type = row["Type"]

        # Handle different event types
        match event_type:
            case "tab":
                # File switch event
                active_file = file_path
                parts.append(f"<act focus file=\"{file_path}\" />")
[content] crowd-pilot/serialize_dataset.py (python)
        # Get event type and range information
        event_type = row["Type"]
        offset = row["RangeOffset"]
        length = row["RangeLength"]

        # Handle selection/cursor movement events
        if event_type == "selection_command":
            old_cursor = per_file_cursor_positions.get(file_path, (0, 0))
            new_cursor = (offset, length)
            per_file_cursor_positions[file_path] = new_cursor
            
            # Emit cursor movement observation if position changed
            if old_cursor != new_cursor:
                parts.append(f"<act cursor file=\"{file_path}\" offset=\"{offset}\" len=\"{length}\" />")
            continue

        # Handle file edit events (must have type 'content')
        if event_type != "content":
            continue
[content] crowd-pilot/serialize_dataset.py (python)
        # Terminal input/output events
        if file_path == "TERMINAL":
            raw_text = row.get("Text", "")
            term_fragment = str(raw_text) if pd.notna(raw_text) else ""
            term_fragment = term_fragment.replace("\
", "
").replace("\
", "
")
            parts.append("<act terminal_input />")
            prev_len = len(terminal_state)
            # Append plus newline as in the replayer
            terminal_state = terminal_state + term_fragment + "
"
            appended = terminal_state[prev_len:]
            if appended.strip():
                parts.append("// observation: terminal_delta")
                parts.append(_fenced_block("TERMINAL", None, _clean_text(appended)))
            continue
[content] crowd-pilot/serialize_dataset.py (python)
        # Focus / switch
        if active_file != file_path:
            active_file = file_path
            if active_file == "TERMINAL":
                parts.append("<act focus target=\"terminal\" />")
            else:
                parts.append(f"<act focus file=\"{active_file}\" />")
            emit_keyframe_for(active_file, language)
[content] crowd-pilot/serialize_dataset.py (python)
                
                # If Text is present, this is the first time opening the file
                # and the entire file content is captured
                text = row.get("Text", "")
                if pd.notna(text) and text:
                    file_content = str(text).replace("\
", "
").replace("\
", "
")
                    file_states[file_path] = file_content
                    language = row.get("Language", "")
                    lang = (language or "") if isinstance(language, str) else ""
                    parts.append(f"// observation: file={file_path}")
                    parts.append(_fenced_block(file_path, lang, _clean_text(file_content)))
[content] crowd-pilot/serialize_dataset.py (python)
                
                # If Text is present, this is the first time opening the file
                # and the entire file content is captured
                text = row["Text"]
                if pd.notna(text):
                    file_content = str(text).replace("\
", "
").replace("\
", "
")
                    file_states[file_path] = file_content
                    language = row.get("Language", "")
                    lang = (language or "") if isinstance(language, str) else ""
                    parts.append(f"// observation: file={file_path}")
                    parts.append(_fenced_block(file_path, lang, _clean_text(file_content)))
[content] crowd-pilot/serialize_dataset.py (python)
                # If text was inserted or replaced, also serialize the new text content
                if new_text_str and (operation == "insert" or operation == "replace"):
                    parts.append(f"// text: {_clean_text(new_text_str)}")
[content] crowd-pilot/serialize_dataset.py (python)
def emit_keyframe_for(file_path: str, language: Optional[str]) -> None:
        if file_path == "TERMINAL":
            if not terminal_state:
                return
            parts.append("// observation: terminal")
            parts.append(_fenced_block("TERMINAL", None, _clean_text(terminal_state)))
        else:
            # FIXME (f.srambical): this yields the entire file content, but we likely only want to emit the changes, or a local viewport
            content = file_states.get(file_path, "")
            if not content:
                return
            lang = (language or "") if isinstance(language, str) else ""
            parts.append(f"// observation: file={file_path}")
            parts.append(_fenced_block(file_path, lang, _clean_text(content)))
[tab] pyproject.toml (plaintext)
[project]
name = "crowd-pilot"
description = "Teaching language models to code like humans."
version = "0.1.0"
requires-python = ">=3.12"
dependencies = [
    "datasets>=2.19.0",
    "pandas>=2.2.2",
    "pyarrow>=15.0.2",
    "tqdm>=4.66.4",
    "regex>=2024.5.15",
]

[project.urls]
Repository = "https://github.com/p-doom/crowd-pilot"
[content] crowd-pilot/serialize_dataset.py (python)

                # Git branch checkout event
                branch = row["Text"]
                branch_str = str(branch).replace("\
", "
").replace("\
", "
")
                parts.append(f"<act git_branch_checkout branch=\"{branch_str}\" />")
[content] crowd-pilot/serialize_dataset.py (python)
                branch_info = row["Text"]
                branch_str = str(branch_info).replace("\
", "
").replace("\
", "
")
                parts.append(f"<act git_branch_checkout />")
                parts.append(f"// git: {_clean_text(branch_str)}")
[content] crowd-pilot/read_dataset.py (python)
#!/usr/bin/env python3
"""
Read and display parquet shards from serialize_dataset.py output.
"""

import argparse
import sys
from pathlib import Path
from typing import List

import pandas as pd  # type: ignore


def read_parquet_shards(input_dir: str, shard_pattern: str = "shard_*.parquet") -> List[str]:
    """Read all parquet shards from input directory and return text content."""
    input_path = Path(input_dir)
    
    if not input_path.exists():
        print(f"Error: Directory {input_dir} does not exist", file=sys.stderr)
        sys.exit(1)
    
    # Find all parquet files matching the pattern
    parquet_files = list(input_path.glob(shard_pattern))
    
    if not parquet_files:
        print(f"Error: No parquet files found in {input_dir} matching {shard_pattern}", file=sys.stderr)
        sys.exit(1)
    
    # Sort files by name for consistent ordering
    parquet_files.sort()
    
    all_texts = []
    for parquet_file in parquet_files:
        try:
            df = pd.read_parquet(parquet_file)
            if 'text' in df.columns:
                all_texts.extend(df['text'].tolist())
            else:
                print(f"Warning: No 'text' column found in {parquet_file}", file=sys.stderr)
        except Exception as e:
            print(f"Error reading {parquet_file}: {e}", file=sys.stderr)
            continue
    
    return all_texts


def main():
    parser = argparse.ArgumentParser(description="Read and display parquet shards from serialize_dataset.py")
    parser.add_argument("input_dir", help="Directory containing parquet shards")
    parser.add_argument("--shard-pattern", default="shard_*.parquet", 
                       help="Pattern to match parquet files (default: shard_*.parquet)")
    parser.add_argument("--limit", type=int, help="Limit number of documents to display")
    
    args = parser.parse_args()
    
    try:
        texts = read_parquet_shards(args.input_dir, args.shard_pattern)
        
        if args.limit:
            texts = texts[:args.limit]
        
        # Print each text document
        for i, text in enumerate(texts):
            print(f"=== Document {i+1} ===")
            print(text)
            print()  # Empty line between documents
            
    except KeyboardInterrupt:
        sys.exit(0)
    except Exception as e:
        print(f"Error: {e}", file=sys.stderr)
        sys.exit(1)


if __name__ == "__main__":
    main()
[content] crowd-pilot/read_dataset.py (python)
#!/usr/bin/env python3
import sys
from pathlib import Path
import pandas as pd

input_dir = sys.argv[1]
for parquet_file in sorted(Path(input_dir).glob("shard_*.parquet")):
    df = pd.read_parquet(parquet_file)
    for text in df['text']:
        print(text)