p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/hf_part/generate_jsonl_qwen_4k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

MAX_TOKENS_PER_CONVERSATION=4096
TOKENIZER_MODEL="Qwen/Qwen3-Coder-30B-A3B-Instruct"

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION --tokenizer_model=$TOKENIZER_MODEL