p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] crowd-pilot/serialization_utils.py (python)
def _line_numbered_output(content: str, start_line: Optional[int] = None, end_line: Optional[int] = None) -> str:
    lines = content.splitlines()
    total = len(lines)
    if total == 0:
        return ""
    s = 1 if start_line is None else max(1, min(start_line, total))
    e = total if end_line is None else max(1, min(end_line, total))
    if e < s:
        e = s
    buf: List[str] = []
    for idx in range(s, e + 1):
        buf.append(f"{idx:6}	{lines[idx - 1]}")
    return "
".join(buf)


def _offset_to_line(content: str, offset: int) -> int:
    if offset <= 0:
        return 1
    if offset > len(content):
        offset = len(content)
    # Count newlines before the offset
    return content[:offset].count("
") + 1


def _compute_viewport(total_lines: int, center_line: int, radius: int) -> Tuple[int, int]:
    if total_lines <= 0:
        return (1, 0)
    start = max(1, center_line - radius)
    end = min(total_lines, center_line + radius)
    if end < start:
        end = start
    return (start, end)


def _escape_single_quotes_for_sed(text: str) -> str:
    # Close quote, add an escaped single quote, reopen quote: '"'"'
    return text.replace("'", "'\"'\"'")


def _compute_changed_block_lines(before: str, after: str) -> Tuple[int, int, List[str]]:
    """
    Return 1-based start and end line numbers in 'before' that should be replaced,
    and the replacement lines from 'after'.
    Always returns at least one line.
    """
    before_lines = before.splitlines()
    after_lines = after.splitlines()
    sm = difflib.SequenceMatcher(a=before_lines, b=after_lines, autojunk=False)
    opcodes = [op for op in sm.get_opcodes() if op[0] != "equal"]
    if not opcodes:
        # No visible change; choose a safe single-line replace at end of file
        start_line = max(1, len(before_lines))
        end_line = start_line
        repl = after_lines[start_line - 1:start_line] if after_lines else [""]
        return (start_line, end_line, repl)
    first = opcodes[0]
    last = opcodes[-1]
    # i1/i2 refer to 'before' indices, j1/j2 to 'after'
    start_line = (first[1] + 1) if (first[1] + 1) > 0 else 1
    end_line = last[2]
    if end_line < start_line:
        end_line = start_line
    repl = after_lines[first[3]:last[4]]
    # Ensure at least one replacement line
    if not repl:
        repl = [""]
    return (start_line, end_line, repl)
[content] crowd-pilot/serialization_utils.py (python)
def _line_numbered_output(content: str, start_line: Optional[int] = None, end_line: Optional[int] = None) -> str:
    # TODO (f.srambical): check whether this corresponds **exactly** to the output of cat -n {file_path} | sed -n '{vstart},{vend}p'
    lines = content.splitlines()
    total = len(lines)
    if total == 0:
        return ""
    s = 1 if start_line is None else max(1, min(start_line, total))
    e = total if end_line is None else max(1, min(end_line, total))
    if e < s:
        e = s
    buf: List[str] = []
    for idx in range(s, e + 1):
        buf.append(f"{idx:6}	{lines[idx - 1]}")
    return "
".join(buf)


def _compute_viewport(total_lines: int, center_line: int, radius: int) -> Tuple[int, int]:
    if total_lines <= 0:
        return (1, 0)
    # Clamp center_line to valid range to ensure viewport is always valid
    center_line = max(1, min(total_lines, center_line))
    start = max(1, center_line - radius)
    end = min(total_lines, center_line + radius)
    return (start, end)


def _escape_single_quotes_for_sed(text: str) -> str:
    # Close quote, add an escaped single quote, reopen quote: '"'"'
    return text.replace("'", "'\"'\"'")


def _compute_changed_block_lines(before: str, after: str) -> Tuple[int, int, List[str]]:
    """
    Return 1-based start and end line numbers in 'before' that should be replaced,
    and the replacement lines from 'after'.
    Always returns at least one line.
    """
    before_lines = before.splitlines()
    after_lines = after.splitlines()
    sm = difflib.SequenceMatcher(a=before_lines, b=after_lines, autojunk=False)
    opcodes = [op for op in sm.get_opcodes() if op[0] != "equal"]
    if not opcodes:
        # FIXME (f.srambical): clean this up
        raise ValueError("This should never happen!")

        # No visible change; choose a safe single-line replace at end of file
        start_line = max(1, len(before_lines))
        end_line = start_line
        repl = after_lines[start_line - 1:start_line] if after_lines else [""]
        return (start_line, end_line, repl)
    first = opcodes[0]
    last = opcodes[-1]
    # i1/i2 refer to 'before' indices, j1/j2 to 'after'
    start_line = (first[1] + 1) if (first[1] + 1) > 0 else 1
    end_line = last[2]
    if end_line < start_line:
        end_line = start_line
    repl = after_lines[first[3]:last[4]]
    # Ensure at least one replacement line
    if not repl:
        repl = [""]
    return (start_line, end_line, repl)
[content] crowd-pilot/serialization_utils.py (python)
def _offset_to_line(content: str, offset: int) -> int:
    if offset <= 0:
        return 1
    if offset > len(content):
        offset = len(content)
    # Count newlines before the offset
    return content[:offset].count("
") + 1
[tab] crowd-pilot/read_dataset.py (python)
#!/usr/bin/env python3
import sys
from pathlib import Path
import pandas as pd

input_dir = sys.argv[1]
for parquet_file in sorted(Path(input_dir).glob("shard_*.parquet")):
    df = pd.read_parquet(parquet_file)
    for text in df['text']:
        print(text)