p-doom/crowd-code-dataset-0.2 huggingface dataset:
[terminal_command] TERMINAL ()
python3 -m MaxText.train src/MaxText/configs/base.yml   run_name=$YOUR_JOB_NAME   base_output_directory=test_output dataset_type=grain grain_file_type=parquet \ grain_train_files=/home/franz.srambical/crowd-pilot/test_output_dir/shard_00000.parquet
[content] crowd-pilot/serialize_dataset.py (python)
        full_df = pd.DataFrame(pdf.copy())
        session_dfs = _split_dataframe_into_sessions(full_df)
        for session_df in session_dfs:
            transcript = _session_to_transcript(
                session_df,
                long_pause_threshold_ms=cfg.long_pause_threshold_ms,
            )
            buffer.append(transcript)
            docs_written += 1
            if len(buffer) >= cfg.shard_size:
                flush_buffer(shard_idx)
                shard_idx += 1
            if cfg.max_docs and docs_written >= cfg.max_docs:
                break
[content] crowd-pilot/serialize_dataset.py (python)
def _split_dataframe_into_sessions(df: pd.DataFrame) -> List[pd.DataFrame]:
    """Split a concatenated events DataFrame into per-session chunks.

    Priority:
    1) If a likely session identifier column exists and changes over rows,
       split on changes of that column while preserving order.
    2) Otherwise, if a numeric 'Sequence' column resets (decreases), split on resets.
    3) Fallback: treat entire DataFrame as a single session.
    """
    candidate_cols = [
        "Session",
        "SessionId",
        "SessionID",
        "session",
        "session_id",
        "csv_file",
        "csv_path",
        "source",
        "Source",
    ]

    for col in candidate_cols:
        if col in df.columns:
            # Split when the value changes (stable runs of same id)
            values = df[col].tolist()
            if not values:
                break
            boundaries: List[int] = [0]
            for i in range(1, len(values)):
                if values[i] != values[i - 1]:
                    boundaries.append(i)
            if boundaries == [0]:
                # Single run; still one session
                return [df.reset_index(drop=True)]
            boundaries.append(len(df))
            sessions: List[pd.DataFrame] = []
            for s, e in zip(boundaries[:-1], boundaries[1:]):
                sessions.append(df.iloc[s:e].reset_index(drop=True))
            return sessions

    # Fallback: split by resets in 'Sequence'
    if "Sequence" in df.columns:
        seq_series = pd.to_numeric(df["Sequence"], errors="coerce")
        boundaries = [0]
        prev = None
        for i, v in enumerate(seq_series):
            if prev is None:
                prev = v
                continue
            try:
                if pd.notna(v) and pd.notna(prev) and cast(float, v) < cast(float, prev):
                    boundaries.append(i)
            finally:
                prev = v
        if len(boundaries) > 1:
            boundaries.append(len(df))
            sessions = [df.iloc[s:e].reset_index(drop=True) for s, e in zip(boundaries[:-1], boundaries[1:])]
            return sessions

    return [df.reset_index(drop=True)]
[content] crowd-pilot/serialize_dataset.py (python)
        session_df = pd.DataFrame(pdf.copy())
        transcript = _session_to_transcript(
            session_df,
            long_pause_threshold_ms=cfg.long_pause_threshold_ms,
        )
        buffer.append(transcript)
        docs_written += 1
        if len(buffer) >= cfg.shard_size:
            flush_buffer(shard_idx)
            shard_idx += 1
[content] crowd-pilot/serialize_dataset.py (python)
def _split_dataframe_into_sessions(df: pd.DataFrame) -> List[pd.DataFrame]:
    """Split a concatenated events DataFrame into per-session chunks.

    Priority:
    1) If a likely session identifier column exists and changes over rows,
       split on changes of that column while preserving order.
    2) Otherwise, if a numeric 'Sequence' column resets (decreases), split on resets.
    3) Fallback: treat entire DataFrame as a single session.
    """
    candidate_cols = [
        "Session",
        "SessionId",
        "SessionID",
        "session",
        "session_id",
        "csv_file",
        "csv_path",
        "source",
        "Source",
    ]

    for col in candidate_cols:
        if col in df.columns:
            # Split when the value changes (stable runs of same id)
            values = df[col].tolist()
            if not values:
                break
            boundaries: List[int] = [0]
            for i in range(1, len(values)):
                if values[i] != values[i - 1]:
                    boundaries.append(i)
            if boundaries == [0]:
                # Single run; still one session
                return [df.reset_index(drop=True)]
            boundaries.append(len(df))
            sessions: List[pd.DataFrame] = []
            for s, e in zip(boundaries[:-1], boundaries[1:]):
                sessions.append(df.iloc[s:e].reset_index(drop=True))
            return sessions

    # Fallback: split by resets in 'Sequence'
    if "Sequence" in df.columns:
        seq_values = pd.to_numeric(df["Sequence"], errors="coerce").tolist()
        boundaries = [0]
        prev = None
        for i, v in enumerate(seq_values):
            if prev is None:
                prev = v
                continue
            try:
                if pd.notna(v) and pd.notna(prev) and cast(float, v) < cast(float, prev):
                    boundaries.append(i)
            finally:
                prev = v
        if len(boundaries) > 1:
            boundaries.append(len(df))
            sessions = [df.iloc[s:e].reset_index(drop=True) for s, e in zip(boundaries[:-1], boundaries[1:])]
            return sessions

    return [df.reset_index(drop=True)]
[content] crowd-pilot/serialize_dataset.py (python)
def _split_dataframe_into_sessions(df: pd.DataFrame) -> List[pd.DataFrame]:
    """Split a concatenated events DataFrame into per-session chunks.

    Priority:
    1) If a likely session identifier column exists and changes over rows,
       split on changes of that column while preserving order.
    2) Otherwise, if a numeric 'Sequence' column resets (decreases), split on resets.
    3) Fallback: treat entire DataFrame as a single session.
    """
    candidate_cols = [
        "Session",
        "SessionId",
        "SessionID",
        "session",
        "session_id",
        "csv_file",
        "csv_path",
        "source",
        "Source",
    ]

    for col in candidate_cols:
        if col in df.columns:
            # Split when the value changes (stable runs of same id)
            values = df[col].tolist()
            if not values:
                break
            boundaries: List[int] = [0]
            for i in range(1, len(values)):
                if values[i] != values[i - 1]:
                    boundaries.append(i)
            if boundaries == [0]:
                # Single run; still one session
                return [df.reset_index(drop=True)]
            boundaries.append(len(df))
            sessions: List[pd.DataFrame] = []
            for s, e in zip(boundaries[:-1], boundaries[1:]):
                sessions.append(df.iloc[s:e].reset_index(drop=True))
            return sessions

    # Fallback: split by resets in 'Sequence'
    if "Sequence" in df.columns:
        seq_series = pd.Series(df["Sequence"])  # ensure Series for type checkers
        numeric = pd.to_numeric(seq_series, errors="coerce")
        numeric_series = cast(pd.Series, numeric)
        seq_values = cast(List[float], numeric_series.astype(float).tolist())
        boundaries = [0]
        prev = None
        for i, v in enumerate(seq_values):
            if prev is None:
                prev = v
                continue
            try:
                if pd.notna(v) and pd.notna(prev) and cast(float, v) < cast(float, prev):
                    boundaries.append(i)
            finally:
                prev = v
        if len(boundaries) > 1:
            boundaries.append(len(df))
            sessions = [df.iloc[s:e].reset_index(drop=True) for s, e in zip(boundaries[:-1], boundaries[1:])]
            return sessions

    return [df.reset_index(drop=True)]
[content] crowd-pilot/serialize_dataset.py (python)
    # Build a list of per-session DataFrames
    session_dataframes: List[pd.DataFrame] = []
    if use_local:
        root = Path(cast(str, cfg.csv_root)).expanduser().resolve()
        csv_files = _discover_local_sessions(root)
        assert csv_files, f"No 'source.csv' files found under {root}"
        for csv_file in csv_files:
            df = _load_local_csv(csv_file)
            missing_local = [c for c in required_cols if c not in df.columns]
            assert not missing_local, f"Missing required CSV columns in {csv_file}: {missing_local}"
            session_dataframes.append(df)
    else:
        ds = load_hf_csv(cfg.hf_path, cfg.split)
        missing_hf = [c for c in required_cols if c not in ds.column_names]
        assert not missing_hf, f"Missing required CSV columns: {missing_hf}"
        session_dataframes = [cast(pd.DataFrame, ds.to_pandas())]
[content] crowd-pilot/serialize_dataset.py (python)
def _load_local_csv(csv_path: Path) -> pd.DataFrame:
    df = pd.read_csv(csv_path)
    return df


def _discover_local_sessions(root: Path) -> List[Path]:
    # Recursively find per-session CSVs named 'source.csv'
    paths: List[Path] = []
    for p in root.rglob("source.csv"):
        if p.is_file():
            paths.append(p)
    paths.sort()
    return paths