p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/generate_jsonl_nemo_dataset_4k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

# rough estimate of characters per token
TARGET_CHARS_PER_CONVERSATION=$((4096*3))

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --target_chars_per_conversation=$TARGET_CHARS_PER_CONVERSATION
[content] crowd-pilot/crowd_pilot/serialize_dataset_nemo_json.py (python)
        # Aggregate per-session turns for filtering
        total_turns = sum(per_chunk_turns)

        if total_turns < cfg.min_session_turns:
            print(f"[warning] Session {session_path} is too short ({total_turns} turns)")