p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] crowd-pilot/serialization_utils.py (python)
_ANSI_CSI_RE = re.compile(r"\x1b\[[0-9;?]*[ -/]*[@-~]")
_ANSI_OSC_RE = re.compile(r"\x1b\].*?(?:\x07|\x1b\\)")


def _apply_backspaces(text: str) -> str:
    out: List[str] = []
    for ch in text:
        if ch == "\b":  # \x08
            if out:
                out.pop()
        else:
            out.append(ch)
    return "".join(out)


def _normalize_terminal_output(raw: str) -> str:
    """
    Normalize PTY/terminal output for training:
      - Apply backspaces (\x08)
      - Resolve carriage returns (
) by keeping the last rewrite per line
      - Strip ANSI/OSC escape sequences and BEL (\x07)
    """
    if not raw:
        return raw
    s = raw.replace("\x07", "")
    s = _apply_backspaces(s)
    # Resolve carriage returns per line (keep last rewrite)
    lines: List[str] = []
    for seg in s.split("
"):
        lines.append(seg.split("
")[-1])
    s = "
".join(lines)
    # Strip ANSI escape sequences
    s = _ANSI_OSC_RE.sub("", s)
    s = _ANSI_CSI_RE.sub("", s)
    return s
[content] crowd-pilot/serialization_utils.py (python)
    s = _apply_backspaces(raw)
    # Remove OSC sequences that are properly terminated (BEL or ST)
    s = _ANSI_OSC_TERMINATED_RE.sub("", s)
    # Fallback: drop any unterminated OSC up to end-of-line only
    s = "
".join(_ANSI_OSC_LINE_FALLBACK_RE.sub("", line) for line in s.split("
"))
[content] crowd-pilot/serialization_utils.py (python)
        parts = seg.split("
")
        chosen = ""
        # pick last non-empty part if available; else last part
        for p in reversed(parts):
            if p != "":
                chosen = p
                break
        if chosen == "" and parts:
            chosen = parts[-1]
        resolved_lines.append(chosen)
    s = "
".join(resolved_lines)
[content] crowd-pilot/serialization_utils.py (python)
    s = _apply_backspaces(raw)
    # Remove OSC sequences that are properly terminated (BEL or ST)
    s = _ANSI_OSC_TERMINATED_RE.sub("", s)
    # Fallback: drop any unterminated OSC up to end-of-line only
    s = "
".join(_ANSI_OSC_LINE_FALLBACK_RE.sub("", line) for line in s.split("
"))
    # Resolve carriage returns per line:
    # - If there are multiple rewrites, keep the last non-empty chunk
    # - If it's CRLF (ending with '
' before '
'), keep the content before '
'
    resolved_lines: List[str] = []
[content] crowd-pilot/serialization_utils.py (python)
def _is_probable_prompt(raw_before_norm: str, cleaned_after_norm: str) -> bool:
    """
    Heuristic prompt detection without hardcoding user-specific PS1 formats.
    - Uses structural cues commonly emitted around prompts:
      - Bracketed paste enable/disable sequences
      - Shell integration OSC 633 markers
      - Window title OSC 0 right before prompt draw
    - Requires the visible text to be a single line without newline (prompts rarely end with newline).
    """
    if not cleaned_after_norm:
        return False
    if "
" in cleaned_after_norm:
        return False
    # Very long single-line outputs without newline are unlikely to be prompts; cap length
    if len(cleaned_after_norm) > 300:
        return False
    raw = raw_before_norm or ""
    has_prompt_markers = (
        (_BRACKETED_PASTE_ENABLE in raw) or
        (_OSC_633 in raw) or
        (_OSC_0 in raw)
    )
    return has_prompt_markers
[content] crowd-pilot/serialization_utils.py (python)
    terminal_output_buffer: List[str] = []

    def _flush_terminal_output_buffer() -> None:
        if not terminal_output_buffer:
            return
        aggregated = "".join(terminal_output_buffer)
        out = aggregated
        if normalize_terminal_output:
            out = _normalize_terminal_output(out)
        cleaned = _clean_text(out)
        if cleaned.strip():
            parts.append(f"<stdout>
{cleaned}
</stdout>")
        terminal_output_buffer.clear()
[content] crowd-pilot/serialization_utils.py (python)
                vp = per_file_viewport.get(file_path)
                if not vp or vp[1] == 0:
                    center = (start_line + end_line) // 2
                    vp = _compute_viewport(total_lines, center, viewport_radius)
                    per_file_viewport[file_path] = vp
[content] crowd-pilot/serialization_utils.py (python)
    def _flush_pending_edit_for_file(target_file: str) -> None:
        before_snapshot = pending_edits_before.get(target_file)
        if before_snapshot is None:
            return
        after_state = file_states.get(target_file, "")
        try:
            start_line, end_line, repl_lines = _compute_changed_block_lines(before_snapshot, after_state)
        except ValueError:
            pending_edits_before[target_file] = None
            return
        before_total_lines = len(before_snapshot.splitlines())
        if end_line < start_line:
            escaped_lines = [_escape_single_quotes_for_sed(line) for line in repl_lines]
            sed_payload = "
".join(escaped_lines)
            if start_line <= max(1, before_total_lines):
                sed_cmd = f"sed -i '{start_line}i\\
{sed_payload}' {target_file}"
            else:
                sed_cmd = f"sed -i '$a\\
{sed_payload}' {target_file}"
        elif not repl_lines:
            sed_cmd = f"sed -i '{start_line},{end_line}d' {target_file}"
        else:
            escaped_lines = [_escape_single_quotes_for_sed(line) for line in repl_lines]
            sed_payload = "
".join(escaped_lines)
            sed_cmd = f"sed -i '{start_line},{end_line}c\\
{sed_payload}' {target_file}"
        total_lines = len(after_state.splitlines())
        center = (start_line + end_line) // 2
        vp = _compute_viewport(total_lines, center, viewport_radius)
        per_file_viewport[target_file] = vp
        vstart, vend = vp
        chained_cmd = f"{sed_cmd} && cat -n {target_file} | sed -n '{vstart},{vend}p'"
        parts.append(_fenced_block(target_file, "bash", _clean_text(chained_cmd)))
        viewport_output = _line_numbered_output(after_state, vstart, vend)
        parts.append(f"<stdout>
{viewport_output}
</stdout>")
        pending_edits_before[target_file] = None

    def _flush_all_pending_edits() -> None:
        for fname in list(pending_edits_before.keys()):
            _flush_pending_edit_for_file(fname)