p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
    # Determine expected column count from the first non-empty line (likely the header)
    expected_columns: Optional[int] = None
    for header_line in original_lines:
        if header_line.strip() == "":
            continue
        header_tokens = tokenize_csv_line_with_indices(header_line)
        if header_tokens:
            expected_columns = len(header_tokens)
            break
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
    # Determine split indices at the start of timestamp tokens in first-column positions
    split_starts: List[int] = []
    for idx, (start, _end, tok) in enumerate(tokens):
        if idx % expected_columns == 0 and is_timestamp_token(tok, cfg.timestamp_regexes):
            split_starts.append(start)

    # Need at least two timestamp tokens to split
    if len(split_starts) <= 1:
        return [line]

    # Build chunk boundaries: from first effective start to each subsequent split start
    # If there is any preface before the first timestamp token that is only separators,
    # we drop it; otherwise, keep from 0.
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
    - Tokenize with quote-awareness to find column boundaries.
    - Identify tokens at FIRST-COLUMN positions that match timestamp regexes.
    - Start a new CSV row at each such timestamp except the first.
    - Keep delimiters and content from each start to right before the next split point.
    - Trim leading separators between chunks and strip existing trailing newlines.
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
def tokenize_csv_line_with_indices(line: str) -> List[Tuple[int, int, str]]:
    """
    Split a CSV line into tokens while tracking start/end indices in the original string.
    - Handles quotes ("") and escaped quotes within quoted fields.
    - Does not strip whitespace or quotes; caller can normalize token text as needed.
    Returns a list of (start_index, end_index, token_text) tuples.
    """
    tokens: List[Tuple[int, int, str]] = []
    in_quotes = False
    token_start = 0
    i = 0
    n = len(line)
    while i < n:
        ch = line[i]
        if in_quotes:
            if ch == '"':
                # Handle escaped quote inside quoted field
                nxt = line[i + 1] if i + 1 < n else ''
                if nxt == '"':
                    i += 2
                    continue
                # Closing quote
                in_quotes = False
                i += 1
                continue
            i += 1
            continue
        else:
            if ch == '"':
                in_quotes = True
                i += 1
                continue
            if ch == ',':
                tokens.append((token_start, i, line[token_start:i]))
                i += 1
                token_start = i
                continue
            i += 1
    # Last token (may include trailing newline) 
    tokens.append((token_start, n, line[token_start:n]))
    return tokens


def normalize_csv_token(token_text: str) -> str:
    s = token_text.strip().rstrip("
")
    if len(s) >= 2 and s[0] == '"' and s[-1] == '"':
        # Unquote and unescape double quotes
        inner = s[1:-1]
        return inner.replace('""', '"')
    return s


def is_timestamp_token(token_text: str, regexes: Sequence[Pattern[str]]) -> bool:
    value = normalize_csv_token(token_text)
    for rx in regexes:
        if rx.fullmatch(value) is not None:
            return True
    return False


def needs_split(line: str, cfg: HeuristicConfig, expected_columns: Optional[int]) -> bool:
    """
    Decide whether a line needs splitting based on presence of two or more timestamp
    tokens specifically in FIRST-COLUMN positions, using the expected column count.
    If expected_columns is unknown, be conservative and do not split.
    """
    if not expected_columns or expected_columns <= 0:
        return False
    tokens = tokenize_csv_line_with_indices(line)
    if not tokens:
        return False
    first_col_timestamp_indices = []
    for idx, (_s, _e, tok) in enumerate(tokens):
        if idx % expected_columns == 0:
            if is_timestamp_token(tok, cfg.timestamp_regexes):
                first_col_timestamp_indices.append(idx)
        if len(first_col_timestamp_indices) >= 2:
            return True
    return False
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
        # Clean up leading separators carried over when we started mid-line
        segment = segment.lstrip(" 	,;|
")
        # Also strip trailing newline characters; we'll re-add newline at write time
        segment = segment.rstrip("
")
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
    # Build chunks: [0:first_ts_start) is kept with first chunk if it's not just separators
    chunks: List[str] = []
    start_indices: List[int] = [spans[0][0]] + [s for s, _ in spans[1:]]
    # Pre-chunk content
    prefix = line[: spans[0][0]]
    # If prefix has non-separator characters, keep it attached to the first chunk.
    # Otherwise, drop it.
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
    - Detect all timestamp spans (merged across patterns).
    - If multiple spans exist, start a new CSV row at each timestamp except the first.
    - Keep delimiters and content from each start to right before the next timestamp.
    - Trim leading whitespace/separators between chunks.
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
    # Build chunk boundaries: from first effective start to each subsequent split start
    # If there is any preface before the first timestamp token that is only separators,
    # we drop it; otherwise, keep from 0.
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
    tokens = tokenize_csv_line_with_indices(line)
    if not tokens:
        return [line]
    # Require first field to be a timestamp
    if not is_timestamp_token(tokens[0][2], cfg.timestamp_regexes):
        return [line]
    spans = timestamp_spans_outside_quotes(line, cfg.timestamp_regexes)
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
def _is_inside_quotes(line: str, idx: int) -> bool:
    """Return True if the character position idx is inside a CSV quoted field.

    CSV quoting uses double quotes (") and doubles them ("") to escape.
    We scan from start to idx (exclusive) and toggle quote-state, skipping escaped quotes.
    """
    in_quotes = False
    i = 0
    # We only need quote state up to the index where a match begins
    while i < idx and i < len(line):
        ch = line[i]
        if ch == '"':
            # Escaped quote inside a quoted field: ""
            if in_quotes and i + 1 < idx and line[i + 1] == '"':
                i += 2
                continue
            in_quotes = not in_quotes
        i += 1
    return in_quotes


def find_row_start_indices(line: str, cfg: HeuristicConfig) -> List[int]:
    """Find indices where a new CSV row likely starts within a (possibly merged) line.

    Heuristic:
    - A row start looks like digits,digits (first two numeric columns)
    - It must be OUTSIDE quoted fields
    - It must be either at the start of the line, or NOT immediately preceded by a comma
      (to avoid matching numeric pairs that are simply subsequent columns like 0,0)
    - It should be immediately followed by a comma (end of second numeric column)
    """
    indices: List[int] = []
    for rx in cfg.timestamp_regexes:
        for m in rx.finditer(line):
            s, e = m.start(), m.end()
            if _is_inside_quotes(line, s):
                continue
            prev = s - 1
            # Must be start-of-line or not immediately after a comma
            if prev >= 0 and line[prev] == ',':
                continue
            # Should be followed by a comma (after the second number ends)
            if e < len(line) and line[e] != ',':
                continue
            indices.append(s)

    # Sort and unique
    indices = sorted(set(indices))
    return indices
[content] crowd-pilot/insert_missing_csv_newlines.py (python)
    if len(starts) >= 2:
        return True
    if len(starts) == 1:
        # Split when a header-like prefix precedes the first timestamp
        prefix = line[: starts[0]]
        if prefix.strip(" ,;|	
") != "":
            return True
    return False
[content] crowd-pilot/serialize_dataset.py (python)
# We will group by session if present; otherwise treat whole split as one meta-session with per-file grouping.
    # FIXME (f.srambical): this is not the case when loading a local CSV. Check whether this is the case when loading from huggingface
    session_key = "SessionId" if "SessionId" in ds.column_names else None
[content] crowd-pilot/serialize_dataset.py (python)
        # Treat the entire shard as a single session
        session_df = pd.DataFrame(pdf.copy())
        docs = _session_to_documents(
            session_df,
            target_chars=cfg.target_chars,
            overlap_chars=cfg.overlap_chars,
            min_doc_chars=cfg.min_doc_chars,
        )
        for d in docs:
            h = _hash(d)
            if h in seen:
                continue
            seen.add(h)
            buffer.append(d)
            if cfg.max_docs and len(seen) >= cfg.max_docs:
                break
            if len(buffer) >= cfg.shard_size:
                flush_buffer(shard_idx)
                shard_idx += 1
        if cfg.max_docs and len(seen) >= cfg.max_docs:
            break
[content] crowd-pilot/serialize_dataset.py (python)
Single pseudo-session to drive grouping by file
        pdf = pdf.copy()
        pdf["__PseudoSession__"] = 0
        groups_iter = pdf.groupby("__PseudoSession__")

        for _, session_df_obj in tqdm(groups_iter, desc="