p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] crowd-pilot/serialize_dataset.py (python)
    p.add_argument("--keyframe_every_events", type=int, default=40, help="Emit an observation snapshot every N edit events per file")
    p.add_argument("--long_pause_threshold_ms", type=int, default=120000, help="Threshold (ms) to annotate long pauses and emit a keyframe")
[content] crowd-pilot/serialize_dataset.py (python)
        if transcript:
            h = _hash(transcript)
            if h not in seen:
                seen.add(h)
                buffer.append(transcript)
                if len(buffer) >= cfg.shard_size:
                    flush_buffer(shard_idx)
                    shard_idx += 1
[content] crowd-pilot/serialize_dataset.py (python)

        # File edit events
        ro = row.get("RangeOffset", 0)
        rl = row.get("RangeLength", 0)
        new_text = row.get("Text", "")
        offset = _safe_int(ro, 0)
        length = _safe_int(rl, 0)
        new_text_str = str(new_text) if pd.notna(new_text) else ""

        operation = "noop"
        if length == 0 and new_text_str:
            operation = "insert"
        elif length > 0 and not new_text_str:
            operation = "delete"
        elif length > 0 and new_text_str:
            operation = "replace"

        parts.append(f"<act {operation} file=\"{file_path}\" offset=\"{offset}\" len=\"{length}\" />")

        before = file_states.get(file_path, "")
        after = _apply_change(before, offset, length, new_text)
        file_states[file_path] = after
        per_file_event_counts[file_path] = per_file_event_counts.get(file_path, 0) + 1
[content] crowd-pilot/serialize_dataset.py (python)
        language = row.get("Language", "")
        event_time = _safe_int(row.get("Time", None), None) if "Time" in events.columns else None

        # Long pause detection
        if last_time_ms is not None and event_time is not None:
            delta = event_time - last_time_ms
            if delta is not None and delta > long_pause_threshold_ms:
                parts.append(f"<obs long_pause ms=\"{delta}\" />")
                if active_file is not None:
                    emit_keyframe_for(active_file, language)
        if event_time is not None:
            last_time_ms = event_time

        # Focus / switch
        if active_file != file_path:
            active_file = file_path
            if active_file == "TERMINAL":
                parts.append("<act focus target=\"terminal\" />")
            else:
                parts.append(f"<act focus file=\"{active_file}\" />")
            emit_keyframe_for(active_file, language)

        # Terminal input/output events
        if file_path == "TERMINAL":
            raw_text = row.get("Text", "")
            term_fragment = str(raw_text) if pd.notna(raw_text) else ""
            term_fragment = term_fragment.replace("\
", "
").replace("\
", "
")
            parts.append("<act terminal_input />")
            prev_len = len(terminal_state)
            # Append plus newline as in the replayer
            terminal_state = terminal_state + term_fragment + "
"
            appended = terminal_state[prev_len:]
            if appended.strip():
                parts.append("// observation: terminal_delta")
                parts.append(_fenced_block("TERMINAL", None, _clean_text(appended)))
[content] crowd-pilot/serialize_dataset.py (python)

    def emit_keyframe_for(file_path: str, language: Optional[str]) -> None:
        if file_path == "TERMINAL":
            if not terminal_state:
                return
            parts.append("// observation: terminal")
            parts.append(_fenced_block("TERMINAL", None, _clean_text(terminal_state)))
            return
        content = file_states.get(file_path, "")
        if not content:
            return
        lang = (language or "") if isinstance(language, str) else ""
        parts.append(f"// observation: file={file_path}")
        parts.append(_fenced_block(file_path, lang, _clean_text(content)))

    for i in range(len(events)):
        row = events.iloc[i]
[content] crowd-pilot/serialize_dataset.py (python)
def _apply_change(content: str, offset: object, length: object, new_text: object) -> str:
    # Mirrors crowd_code_player.replay_file.apply_change
    base = str(content)
    text = str(new_text) if pd.notna(new_text) else ""
    off = _safe_int(offset, 0)
    ln = _safe_int(length, 0)
    text = text.replace("\
", "
").replace("\
", "
")
    if off > len(base):
        base = base + (" " * (off - len(base)))
    return base[:off] + text + base[off + ln:]


def _session_to_transcript(
    df: pd.DataFrame,
    keyframe_every_events: int,
    long_pause_threshold_ms: int,
) -> str:
    # Build a single chronological transcript mixing action tokens and observation keyframes.
    if df.empty:
        return ""

    # Prefer Time if present, else Sequence
    sort_cols: List[str] = [c for c in ["Time", "Sequence"] if c in df.columns]
    if not sort_cols:
        sort_cols = df.columns[:1].tolist()  # fallback deterministic order
    events = df.sort_values(sort_cols).reset_index(drop=True)

    file_states: Dict[str, str] = {}
    terminal_state: str = ""
    active_file: Optional[str] = None
    per_file_event_counts: Dict[str, int] = {}
    last_time_ms: Optional[int] = None
[content] crowd-pilot/serialize_dataset.py (python)
def _safe_int(value: object, default: int = 0) -> int:
    try:
        if value is None or (isinstance(value, float) and pd.isna(value)):
            return default
        if isinstance(value, (int,)):
            return int(value)
        if isinstance(value, float):
            return int(value)
        s = str(value).strip()
        if s == "":
            return default
        return int(float(s))
    except Exception:
        return default
[content] crowd-pilot/serialize_dataset.py (python)
        for d in docs:
            h = _hash(d)
            if h in seen:
                continue
            seen.add(h)
            buffer.append(d)
            if cfg.max_docs and len(seen) >= cfg.max_docs:
                break
            if len(buffer) >= cfg.shard_size:
                flush_buffer(shard_idx)
                shard_idx += 1
[content] crowd-pilot/serialize_dataset.py (python)
    # Chunk by characters with overlap
    docs: List[str] = []
    start = 0
    n = len(session_doc)
    while start < n:
        end = min(start + target_chars, n)
        chunk = session_doc[start:end].strip()
        if len(chunk) >= min_doc_chars:
            docs.append(chunk)
        if end == n:
            break
        start = max(end - overlap_chars, start + 1)
    return docs
[content] crowd-pilot/serialize_dataset.py (python)
    session_doc = "

".join(parts).strip()
    if selection_parts:
        session_doc = session_doc + "

// selections
" + "

".join(selection_parts)
    if len(session_doc) < min_doc_chars:
        # Small sessions can still be useful; return single doc if not empty
        return [session_doc] if session_doc else []
[content] crowd-pilot/serialize_dataset.py (python)
    # Append selection events explicitly so the LM can learn cursor/mouse behavior.
    # We attempt to extract the selected substring from the latest snapshot when possible; otherwise we log metadata only.
    selection_rows = df.sort_values(["Sequence"]).query("Type == 'selection_command'")
    selection_parts: List[str] = []
    for _, srow in selection_rows.iterrows():
        file_path = str(srow.get("File", ""))
        lang_val = srow.get("Language", "")
        lang = lang_val if isinstance(lang_val, str) else ""
        ro = srow.get("RangeOffset", 0)
        rl = srow.get("RangeLength", 0)
        offset = int(ro) if isinstance(ro, (int, float, str)) and str(ro).isdigit() else 0
        length = int(rl) if isinstance(rl, (int, float, str)) and str(rl).isdigit() else 0
        header = f"// selection: file={file_path} offset={offset} length={length}"
        snippet = ""
        base_text = latest_map.get(file_path)
        if base_text and length > 0 and 0 <= offset < len(base_text):
            end = min(offset + length, len(base_text))
            snippet = base_text[offset:end]
        if snippet:
            selection_parts.append(header + "
" + _fenced_block(file_path, lang, snippet))
        else:
            selection_parts.append(header)
[content] crowd-pilot/serialize_dataset.py (python)
        lang_val = row.get("Language", "")
        lang = lang_val if isinstance(lang_val, str) else ""
        text = _clean_text(str(row.get("Text", "")))
        if not text:
            continue
        # Drop extremely small snippets to avoid noise
        if len(text) < 16:
[content] crowd-pilot/serialize_dataset.py (python)
    # Latest content per file by Sequence (assumed monotonic per CSV); groupby File and take last row
    df_sorted = df.sort_values(["File", "Sequence"])  # ensure order within file
    latest_per_file = df_sorted.groupby("File", as_index=False).tail(1)
[content] crowd-pilot/serialize_dataset.py (python)
def _session_to_documents(df: pd.DataFrame, target_chars: int, overlap_chars: int, min_doc_chars: int) -> List[str]:
    """
    Convert a single session (CSV) into one or more training documents.
    Strategy: gather latest content per file, then assemble a session doc and chunk.
    """
[content] crowd-pilot/serialize_dataset.py (python)
        language = row.get("Language", "")
        event_time_raw = row.get("Time", None) if "Time" in events.columns else None
        event_time = _safe_int(event_time_raw, 0) if event_time_raw is not None else None

        # Long pause detection
        if last_time_ms is not None and event_time is not None:
            delta = event_time - last_time_ms
            if delta is not None and delta > long_pause_threshold_ms:
                parts.append(f"<obs long_pause ms=\"{delta}\" />")
                if active_file is not None:
                    emit_keyframe_for(active_file, language)
        if event_time is not None:
            last_time_ms = event_time

        # Focus / switch
        if active_file != file_path:
            active_file = file_path
            if active_file == "TERMINAL":
                parts.append("<act focus target=\"terminal\" />")
            else:
                parts.append(f"<act focus file=\"{active_file}\" />")
            emit_keyframe_for(active_file, language)

        # Terminal input/output events
        if file_path == "TERMINAL":
            raw_text = row.get("Text", "")
            term_fragment = str(raw_text) if pd.notna(raw_text) else ""
            term_fragment = term_fragment.replace("\
", "
").replace("\
", "
")
            parts.append("<act terminal_input />")
            prev_len = len(terminal_state)
            # Append plus newline as in the replayer
            terminal_state = terminal_state + term_fragment + "
"
            appended = terminal_state[prev_len:]
            if appended.strip():
                parts.append("// observation: terminal_delta")
                parts.append(_fenced_block("TERMINAL", None, _clean_text(appended)))
[content] crowd-pilot/serialize_dataset.py (python)

        # File edit events
        ro = row.get("RangeOffset", 0)
        rl = row.get("RangeLength", 0)
        new_text = row.get("Text", "")
        offset = _safe_int(ro, 0)
        length = _safe_int(rl, 0)
        new_text_str = str(new_text) if pd.notna(new_text) else ""

        operation = "noop"
        if length == 0 and new_text_str:
            operation = "insert"
        elif length > 0 and not new_text_str:
            operation = "delete"
        elif length > 0 and new_text_str:
            operation = "replace"

        parts.append(f"<act {operation} file=\"{file_path}\" offset=\"{offset}\" len=\"{length}\" />")

        before = file_states.get(file_path, "")
        after = _apply_change(before, offset, length, new_text)
        file_states[file_path] = after
        per_file_event_counts[file_path] = per_file_event_counts.get(file_path, 0) + 1

        if per_file_event_counts[file_path] % max(1, keyframe_every_events) == 0:
            emit_keyframe_for(file_path, language)