p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/generate_jsonl_nemo_dataset_4k copy.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

# rough estimate of characters per token
TARGET_CHARS_PER_CONVERSATION=$((4096*3))

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --target_chars_per_conversation=$TARGET_CHARS_PER_CONVERSATION
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/generate_jsonl_nemo_dataset_full.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

# rough estimate of characters per token
TARGET_CHARS_PER_CONVERSATION=$((4096*3))

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --target_chars_per_conversation=$TARGET_CHARS_PER_CONVERSATION