p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_lora_finetune_32k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_32k_tokens_batch_size_128"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=32768,
        global_batch_size=32,
        micro_batch_size=1,
        num_workers=0,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 2
    recipe.trainer.strategy.expert_model_parallel_size = 2

    # Enable activation checkpointing to fit 30B model on single H100
    recipe.model.config.recompute_granularity = "full"
    recipe.model.config.recompute_method = "uniform"
    recipe.model.config.recompute_num_layers = 1

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_lora_finetune_16k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_16k_tokens_batch_size_128"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_16k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=16384,
        global_batch_size=32,
        micro_batch_size=1,
        num_workers=0,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 2
    recipe.trainer.strategy.expert_model_parallel_size = 2

    # Enable activation checkpointing to fit 30B model on single H100
    recipe.model.config.recompute_granularity = "full"
    recipe.model.config.recompute_method = "uniform"
    recipe.model.config.recompute_num_layers = 1

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[content] crowd-pilot/crowd_pilot/serialization_utils.py (python)
onversation: List[Dict[str, str]] = field(default_factory=list)
    current_chars: int = 0
    files_opened_in_conversation: set[str] = field(default_factory=set)

    def start_new_conversation(self) -> None:
        if self.current_conversation
[content] crowd-pilot/crowd_pilot/serialization_utils.py (python)
onversation) >= self.min_conversation_turns:
                self.conversations.append(self.current_conversation)
        self.current_conversation = []
        self.current_chars = 0
        self.files_opened_in_conversation
[content] crowd-pilot/crowd_pilot/serialization_utils.py (python)
onversation  # only start a new conversation on non-empty current conversation
            and self.current_chars + turn_len > self.max_chars_per_conversation
        ):
            self.start_new_conversation()
        self.current_conversation
[content] crowd-pilot/crowd_pilot/serialization_utils.py (python)
s.

    - Conversations are created by approximately limiting the total characters
      across all `value` fields to `max_chars_per_conversation`.
    - When a new conversation starts (after the first), the first time a file is
      referenced in that conversation
[content] crowd-pilot/crowd_pilot/serialization_utils.py (python)
onversation_state.append_turn({
                        "from": "Assistant",
                        "value": _fenced_block("bash", _clean_text(cmd)),
                    })
                    output = _line_numbered_output(content)
                    conversation
[tab] crowd-pilot/reproduce_issue.py (python)

import sys
import os
from unittest.mock import MagicMock

# Mock pandas and datasets
sys.modules["pandas"] = MagicMock()
sys.modules["datasets"] = MagicMock()

# Add the current directory to sys.path to import serialization_utils
sys.path.append(os.getcwd())

try:
    import crowd_pilot.serialization_utils as su
    print("Imported serialization_utils successfully.")
    print("Attributes:", dir(su))
    from crowd_pilot.serialization_utils import ChunkState
except ImportError as e:
    print(f"ImportError: {e}")
    # Fallback: try to import directly from file if package import fails
    # This is hacky but might work for testing
    import importlib.util
    spec = importlib.util.spec_from_file_location("serialization_utils", "crowd_pilot/serialization_utils.py")
    su = importlib.util.module_from_spec(spec)
    sys.modules["serialization_utils"] = su
    spec.loader.exec_module(su)
    ChunkState = su.ChunkState

def test_min_turns_interpretation():
    print("Testing min_conversation_turns interpretation...")
    
    # Scenario: User wants min 2 "turns" (pairs).
    # They set min_conversation_turns = 2 (assuming it means pairs).
    # We have a chunk with 2 messages (1 pair).
    
    min_turns = 2
    chunk_state = ChunkState(
        chunks=[],
        max_chars_per_conversation=1000,
        max_chars_per_turn=1000,
        min_conversation_turns=min_turns
    )
    
    # Add 2 messages
    chunk_state.append_turn({"from": "User", "value": "Hello"})
    chunk_state.append_turn({"from": "Assistant", "value": "Hi"})
    
    # Force a chunk finish (by simulating end of session)
    if chunk_state.current_chunk:
        if len(chunk_state.current_chunk) >= chunk_state.min_conversation_turns:
            chunk_state.chunks.append(chunk_state.current_chunk)
            
    if len(chunk_state.chunks) == 1:
        print(f"Result: Kept the chunk with {len(chunk_state.chunks[0])} messages.")
        print("Interpretation: min_conversation_turns counts MESSAGES, not PAIRS.")
        print("If the user expected PAIRS, they would be surprised that a 1-pair conversation is kept.")
    else:
        print("Result: Dropped the chunk.")
        print("Interpretation: min_conversation_turns might mean PAIRS (or logic is different).")

if __name__ == "__main__":
    test_min_turns_interpretation()
[tab] crowd-pilot/reproduce_issue.py (python)

import sys
import os
from unittest.mock import MagicMock

# Mock pandas and datasets
sys.modules["pandas"] = MagicMock()
sys.modules["datasets"] = MagicMock()

# Insert the current directory at the BEGINNING of sys.path
sys.path.insert(0, os.getcwd())

try:
    import crowd_pilot.serialization_utils as su
    print("Imported serialization_utils successfully.")
    # print("Attributes:", dir(su))
    from crowd_pilot.serialization_utils import ChunkState
except ImportError as e:
    print(f"ImportError: {e}")
    sys.exit(1)

def test_min_turns_interpretation():
    print("Testing min_conversation_turns interpretation...")
    
    # Scenario: User wants min 2 "turns" (pairs).
    # They set min_conversation_turns = 2 (assuming it means pairs).
    # We have a chunk with 2 messages (1 pair).
    
    min_turns = 2
    chunk_state = ChunkState(
        chunks=[],
        max_chars_per_conversation=1000,
        max_chars_per_turn=1000,
        min_conversation_turns=min_turns
    )
    
    # Add 2 messages
    chunk_state.append_turn({"from": "User", "value": "Hello"})
    chunk_state.append_turn({"from": "Assistant", "value": "Hi"})
    
    # Force a chunk finish (by simulating end of session)
    if chunk_state.current_chunk:
        if len(chunk_state.current_chunk) >= chunk_state.min_conversation_turns:
            chunk_state.chunks.append(chunk_state.current_chunk)
            
    if len(chunk_state.chunks) == 1:
        print(f"Result: Kept the chunk with {len(chunk_state.chunks[0])} messages.")
        print("Interpretation: min_conversation_turns counts MESSAGES, not PAIRS.")
        print("If the user expected PAIRS, they would be surprised that a 1-pair conversation is kept.")
    else:
        print("Result: Dropped the chunk.")
        print("Interpretation: min_conversation_turns might mean PAIRS (or logic is different).")

if __name__ == "__main__":
    test_min_turns_interpretation()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_nemo_dataset_8k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_8k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

# rough estimate of characters per token
MAX_CHARS_PER_CONVERSATION=$((8192*3))

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_chars_per_conversation=$MAX_CHARS_PER_CONVERSATION
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_nemo_dataset_16k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_16k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

# rough estimate of characters per token
MAX_CHARS_PER_CONVERSATION=$((16384*3))

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_chars_per_conversation=$MAX_CHARS_PER_CONVERSATION
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_nemo_dataset_32k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

# rough estimate of characters per token
MAX_CHARS_PER_CONVERSATION=$((32768*3))

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_chars_per_conversation=$MAX_CHARS_PER_CONVERSATION
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_nemo_dataset_full.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_full/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

# rough estimate of characters per token
MAX_CHARS_PER_CONVERSATION=99999999999999999

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_chars_per_conversation=$MAX_CHARS_PER_CONVERSATION
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_4k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

MAX_TOKENS_PER_CONVERSATION=4096
TOKENIZER_MODEL=""

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION --tokenizer_model=$TOKENIZER_MODEL
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_32k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

MAX_TOKENS_PER_CONVERSATION=32768

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_128k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_128k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

MAX_TOKENS_PER_CONVERSATION=128*1024

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_dataset_full.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_full/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

MAX_TOKENS_PER_CONVERSATION=99999999999999999

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_8k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_8k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

MAX_TOKENS_PER_CONVERSATION=8192

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_8k.sh (shellscript)
TOKENIZER_MODEL="Qwen/Qwen3-Coder-30B-A3B-Instruct"

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION --tokenizer_model=$TOKENIZER_MODEL
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_16k.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_16k_tokens/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/"

MAX_TOKENS_PER_CONVERSATION=16384

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_16k.sh (shellscript)
TOKENIZER_MODEL="Qwen/Qwen3-Coder-30B-A3B-Instruct"

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION --tokenizer_model=$TOKENIZER_MODEL
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_32k.sh (shellscript)
TOKENIZER_MODEL="Qwen/Qwen3-Coder-30B-A3B-Instruct"

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION --tokenizer_model=$TOKENIZER_MODEL
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_128k.sh (shellscript)
TOKENIZER_MODEL="Qwen/Qwen3-Coder-30B-A3B-Instruct"

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION --tokenizer_model=$TOKENIZER_MODEL
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/dataset/generate_jsonl_qwen_dataset_full.sh (shellscript)
TOKENIZER_MODEL="Qwen/Qwen3-Coder-30B-A3B-Instruct"

uv run crowd_pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR --max_tokens_per_conversation=$MAX_TOKENS_PER_CONVERSATION --tokenizer_model=$TOKENIZER_MODEL
[tab] /fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/metadata.json (json)
{
  "config": {
    "csv_root": "/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/",
    "output_dir": "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/",
    "min_conversation_messages": 5,
    "val_ratio": 0.1,
    "max_tokens_per_conversation": 4096,
    "max_tokens_per_message": 2048,
    "tokenizer_model": "Qwen/Qwen3-Coder-30B-A3B-Instruct"
  },
  "counts": {
    "total_sessions": 349,
    "train_conversations": 10394,
    "val_conversations": 768,
    "conversations_written": 11162
  },
  "turn_stats": {
    "count": 11162,
    "total": 130974,
    "median": 10.0,
    "avg": 11.733918652571223,
    "min": 5,
    "max": 106
  },
  "char_stats": {
    "count": 11162,
    "total": 117777543,
    "median": 10817.0,
    "avg": 10551.652302454757,
    "min": 382,
    "max": 40048
  },
  "files": {
    "train_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/training.jsonl",
    "val_path": "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/validation.jsonl"
  }
}
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/finetune.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .nemo_venv/bin/activate

python3 "$1"
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_full_finetune_4k copy.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_128"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=128,
        micro_batch_size=1,
    )
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_full_finetune_4k_bs_128.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_128"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=128,
        micro_batch_size=1,
    )
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_full_finetune_4k_bs_1.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_128"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=128,
        micro_batch_size=1,
    )
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_lora_finetune_4k copy.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_4k_tokens_batch_size_128"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=128,
        micro_batch_size=1,
    )

    # Enable activation checkpointing to fit 30B model on two H100s
    recipe.trainer.strategy.tensor_model_parallel_size = 2
    recipe.trainer.strategy.expert_model_parallel_size = 2
    recipe.model.config.recompute_granularity = "full"
    recipe.model.config.recompute_method = "uniform"
    recipe.model.config.recompute_num_layers = 1
    recipe.trainer.strategy.ckpt_async_save = False 

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_lora_finetune_4k_bs_128.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_4k_tokens_batch_size_128"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=128,
        micro_batch_size=1,
    )

    # Enable activation checkpointing to fit 30B model on two H100s
    recipe.trainer.strategy.tensor_model_parallel_size = 2
    recipe.trainer.strategy.expert_model_parallel_size = 2
    recipe.model.config.recompute_granularity = "full"
    recipe.model.config.recompute_method = "uniform"
    recipe.model.config.recompute_num_layers = 1
    recipe.trainer.strategy.ckpt_async_save = False 

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_lora_finetune_4k_bs_1.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_4k_tokens_batch_size_128"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=128,
        micro_batch_size=1,
    )

    # Enable activation checkpointing to fit 30B model on two H100s
    recipe.trainer.strategy.tensor_model_parallel_size = 2
    recipe.trainer.strategy.expert_model_parallel_size = 2
    recipe.model.config.recompute_granularity = "full"
    recipe.model.config.recompute_method = "uniform"
    recipe.model.config.recompute_num_layers = 1
    recipe.trainer.strategy.ckpt_async_save = False 

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[terminal_command] TERMINAL ()
sbatch /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/finetune.sbatch /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_full_finetune_4k_bs_128.py
[terminal_command] TERMINAL ()
sbatch /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/finetune.sbatch /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_lora_finetune_4k_bs_128.py
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_lora_finetune_32k_bs_32.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_32k_tokens_batch_size_32"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=32768,
        global_batch_size=32,
        micro_batch_size=1,
        num_workers=0,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1

    # Enable activation checkpointing to fit 0.6B model on single H100
    #recipe.model.config.recompute_granularity = "full"
    #recipe.model.config.recompute_method = "uniform"
    #recipe.model.config.recompute_num_layers = 1

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[terminal_command] TERMINAL ()
sbatch /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/finetune.sbatch /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_lora_finetune_16k_bs_32.py
[terminal_command] TERMINAL ()
sbatch /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/finetune.sbatch /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/test_lora_finetune_32k_bs_32.py