p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] /home/mihir.mahajan/Projects/jasmine/slurm/jobs/mihir/horeka/batchsize_scaling/tokenizer/sqrt_lr/train_tokenizer_32_nodes.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=32
#SBATCH --ntasks-per-node=4
#SBATCH --time=05:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_tokenizer_batch_size_scaling_32_node

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords/10fps_160x90

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python jasmine/train_tokenizer.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1536 \
    --min_lr=0 \
    --max_lr=3e-4 \
    --log_image_interval=1000 \
    --log_checkpoint_interval=1000 \
    --log \
    --name=tokenizer-batch-size-scaling-32-node-sqrt-lr-$slurm_job_id \
    --tags tokenizer batch-size-scaling 32-node sqrt-lr-scaling \
    --entity instant-uv \
    --project jafar \
    --data_dir $array_records_dir
[content] data/eval/handcrafted/eval_file_switch.md (markdown)

# Assistant
```bash
cat -n src/input_pipeline/validation_to_testcases.py
```

# User
<stdout>
     1  import tyro
     2  import os
     3  from dataclasses import dataclass
     4
     5
     6  @dataclass
     7  class Args:
     8      input_file: str = "data/eval/val/validation.jsonl"
     9      output_file: str = "data/eval/val/validation_testcases.jsonl"
    10      task_name: str = "validation_set"
    11
    12
    13  def convert_to_incremental_jsonl(line, output_file, task_name):
    14      data = json.loads(line)
    15
    16      system_prompt = data.get("system_prompt", "")
    17      conversations = data.get("conversations", [])
    18
    19      output_lines = []
    20      context = []
    21
    22      for i, conv in enumerate(conversations):
    23          # Convert "Assistant"/"User" to "assistant"/"user"
    24          role = conv["from"].lower()
    25          content = conv["value"]
    26
    27          # Add current message to context
    28          context.append({"role": role, "content": content})
    29
    30          if i > 0 and role == "assistant":
    31              task_entry = {
    32                  "system_prompt": system_prompt,
    33                  "task_id": f"{task_name}/{i-1}",
    34                  "context": context[:-1].copy(),
    35                  "expected_final_response": content,
    36              }
    37              output_lines.append(task_entry)
    38
    39      with open(output_file, "a") as f:
    40          for line in output_lines:
    41              f.write(json.dumps(line) + "
")
    42
    43      print(f"Converted {len(output_lines)} entries to {output_file}")
    44
    45
    46  if __name__ == "__main__":
    47      args = tyro.cli(Args)
    48
    49      if os.path.exists(args.output_file):
    50          os.remove(args.output_file)
    51
    52      # iterate over jsonl file and convert to incremental jsonl
    53      conversation_id = 0
    54      with open(args.input_file, "r") as f:
    55          for line in f:
    56              task_name = f"conversation_{conversation_id}/{args.task_name}"
    57              convert_to_incremental_jsonl(line, args.output_file, task_name)
    58              conversation_id += 1
</stdout>

# Assistant
```bash
sed -i '1i\import json' src/input_pipeline/validation_to_testcases.py
```

# User
<stdout>
</stdout>
[content] data/eval/handcrafted/eval_file_switch.md (markdown)
     1  #!/usr/bin/env bash
     2
     3  #SBATCH --nodes=1
     4  #SBATCH --ntasks-per-node=4
     5  #SBATCH --time=05:00:00
     6  #SBATCH --partition=accelerated
     7  #SBATCH --cpus-per-task=5
     8  #SBATCH --gres=gpu:4
     9  #SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
    10  #SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
    11  #SBATCH --job-name=train_tokenizer_batch_size_scaling_1_node
    12
    13  # Log the sbatch script
    14  cat $0
    15
    16  module unload mpi/openmpi/5.0
    17  module unload devel/cuda/12.4
    18  source .venv/bin/activate
    19
    20  array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_new/open_ai_minecraft_arrayrecords/10fps_160x90
    21
    22  job_name=$SLURM_JOB_NAME
    23  slurm_job_id=$SLURM_JOB_ID
    24
    25  CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/checkpoints/$job_name/$slurm_job_id
    26  mkdir -p $CHECKPOINT_DIR
    27
    28  env | grep SLURM
    29
    30  srun python jasmine/train_tokenizer.py \
    31      --ckpt_dir $CHECKPOINT_DIR \
    32      --batch_size=48 \
    33      --min_lr=0 \
    34      --max_lr=1e-4 \
    35      --log_image_interval=1000 \
    36      --log_checkpoint_interval=1000 \
    37      --log \
    38      --name=tokenizer-batch-size-scaling-1-node-sqrt-lr-$slurm_job_id \
    39      --tags tokenizer batch-size-scaling 1-node sqrt-lr \
    40      --entity instant-uv \
    41      --project jafar \
    42      --data_dir $array_records_dir