p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] /home/franz.srambical/slurm/jobs/franz/berlin/crowd-pilot/start_sglang_server_glm4_5_air.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --job-name=crowd_pilot_sglang
#SBATCH --mem=400GB
#SBATCH --qos=normal

source /home/franz.srambical/crowd-pilot-serializer-legacy/.venv/bin/activate
module load CUDA/12.8

model_path="Qwen/Qwen3-8b"
python3 -m sglang.launch_server --model-path $model_path --host 0.0.0.0 --log-requests
[content] /home/franz.srambical/slurm/jobs/franz/berlin/crowd-pilot/start_sglang_server_glm4_5_air.sh (shellscript)
  --tp-size 8 \
  --tool-call-parser glm47  \
  --reasoning-parser glm45 \
  --speculative-algorithm EAGLE \
  --speculative-num-steps 3 \
  --speculative-eagle-topk 1 \
  --speculative-num-draft-tokens 4 \
  --mem-fraction-static 0.8 \
[tab] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_4k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--tokenizer="Qwen/Qwen3-8B"
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_8k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--tokenizer="Qwen/Qwen3-8B"
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_8k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 4096\
--tokenizer="Qwen/Qwen3-8B"
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_16k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 8192 \
--tokenizer="Qwen/Qwen3-8B"
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_32k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 16384 \
--tokenizer="Qwen/Qwen3-8B"
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_128k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 32768 \
--tokenizer="Qwen/Qwen3-8B"
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_4k_glm.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 4096 \
--tokenizer="Qwen/Qwen3-8B"
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_16k_glm.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 16384 \
--tokenizer="zai-org/GLM-4.5-Air"
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_32k_glm.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 32768 \
--tokenizer="zai-org/GLM-4.5-Air"
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_128k_glm.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 131072 \
--tokenizer="zai-org/GLM-4.5-Air"