[content] crowd-pilot/insert_missing_csv_newlines.py (python)
def tokenize_csv_line_with_indices(line: str) -> List[Tuple[int, int, str]]:
    """
    Split a CSV line into tokens while tracking start/end indices in the original string.
    - Handles quotes ("") and escaped quotes within quoted fields.
    - Does not strip whitespace or quotes; caller can normalize token text as needed.
    Returns a list of (start_index, end_index, token_text) tuples.
    """
    tokens: List[Tuple[int, int, str]] = []
    in_quotes = False
    token_start = 0
    i = 0
    n = len(line)
    while i < n:
        ch = line[i]
        if in_quotes:
            if ch == '"':
                # Handle escaped quote inside quoted field
                nxt = line[i + 1] if i + 1 < n else ''
                if nxt == '"':
                    i += 2
                    continue
                # Closing quote
                in_quotes = False
                i += 1
                continue
            i += 1
            continue
        else:
            if ch == '"':
                in_quotes = True
                i += 1
                continue
            if ch == ',':
                tokens.append((token_start, i, line[token_start:i]))
                i += 1
                token_start = i
                continue
            i += 1
    # Last token (may include trailing newline) 
    tokens.append((token_start, n, line[token_start:n]))
    return tokens


def normalize_csv_token(token_text: str) -> str:
    s = token_text.strip().rstrip("
")
    if len(s) >= 2 and s[0] == '"' and s[-1] == '"':
        # Unquote and unescape double quotes
        inner = s[1:-1]
        return inner.replace('""', '"')
    return s


def is_timestamp_token(token_text: str, regexes: Sequence[Pattern[str]]) -> bool:
    value = normalize_csv_token(token_text)
    for rx in regexes:
        if rx.fullmatch(value) is not None:
            return True
    return False


def compute_quoted_ranges(line: str) -> List[Tuple[int, int]]:
    """Return half-open index ranges [start, end) that are inside quotes."""
    ranges: List[Tuple[int, int]] = []
    in_quotes = False
    start = -1
    i = 0
    n = len(line)
    while i < n:
        ch = line[i]
        if in_quotes:
            if ch == '"':
                nxt = line[i + 1] if i + 1 < n else ''
                if nxt == '"':
                    i += 2
                    continue
                # closing quote
                ranges.append((start, i + 1))
                in_quotes = False
                i += 1
                continue
            i += 1
            continue
        else:
            if ch == '"':
                in_quotes = True
                start = i
                i += 1
                continue
            i += 1
    return ranges


def is_inside_any(pos: int, ranges: Sequence[Tuple[int, int]]) -> bool:
    for a, b in ranges:
        if a <= pos < b:
            return True
    return False


def timestamp_spans_outside_quotes(line: str, regexes: Sequence[Pattern[str]]) -> List[Tuple[int, int]]:
    spans = find_timestamp_spans(line, regexes)
    quoted = compute_quoted_ranges(line)
    return [(s, e) for s, e in spans if not is_inside_any(s, quoted)]


def needs_split(line: str, cfg: HeuristicConfig, expected_columns: Optional[int]) -> bool:
    """
    Decide whether a line needs splitting based on presence of two or more timestamp
    tokens, with the constraint that the FIRST token in the row is a timestamp.
    We allow subsequent timestamps to appear mid-token (e.g., newline missing).
    """
    tokens = tokenize_csv_line_with_indices(line)
    if not tokens:
        return False
    # Require first field to be a timestamp to avoid false positives
    if not is_timestamp_token(tokens[0][2], cfg.timestamp_regexes):
        return False
    spans = timestamp_spans_outside_quotes(line, cfg.timestamp_regexes)