[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/nemo/qwen3_600m_finetune_requeue_34949.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=00:30:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune_requeue
#SBATCH --mem=400GB
#SBATCH --qos=low
#SBATCH --requeue
#SBATCH --signal=b:usr1@60 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

srun python slurm/dev/mihir/berlin/crowd_pilot/nemo/iterating_on_stuff/test_full_finetune_4k_requeue.py &

child_pid=$!

wait $child_pid
/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
─ Entering Experiment qwen3_600m_full_4k_tokens_34940 with id: qwen3_600m_ful… ─
[16:22:00] INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_full_4k_tokens_34940/                       
                    qwen3_600m_full_4k_tokens_34940_17642                       
                    56919/qwen3_600m_full_4k_tokens_34940                       
[16:22:00] Launching job qwen3_600m_full_4k_tokens_34940 for   experiment.py:744
           experiment qwen3_600m_full_4k_tokens_34940                           
           INFO     Log directory is:                     local_scheduler.py:777
                    /home/mihir.mahajan/.nemo_run/experim                       
                    ents/qwen3_600m_full_4k_tokens_34940/                       
                    qwen3_600m_full_4k_tokens_34940_17642                       
                    56919/qwen3_600m_full_4k_tokens_34940                       
           INFO     Launched app:                                launcher.py:111
                    local_persistent://nemo_run/qwen3_600m_full_                
                    4k_tokens_34940-vpw9h2rcsd2l9                               
─ Waiting for Experiment qwen3_600m_full_4k_tokens_34940_1764256919 to finish ──

Experiment Status for qwen3_600m_full_4k_tokens_34940_1764256919

Task 0: qwen3_600m_full_4k_tokens_34940
- Status: RUNNING
- Executor: LocalExecutor
- Job id: qwen3_600m_full_4k_tokens_34940-vpw9h2rcsd2l9
- Local Directory: /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_full_4k_tokens_34940/qwen3_600m_full_4k_tokens_34940_1764256919/qwen3_600m_full_4k_tokens_34940

           INFO     Waiting for job                              launcher.py:130
                    qwen3_600m_full_4k_tokens_34940-vpw9h2rcsd2l                
                    9 to finish [log=True]...                                   
kens_34940/0 /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
kens_34940/0   import pynvml  # type: ignore[import]
kens_34940/0 I1127 16:22:02.356000 2424402 torch/distributed/run.py:657] Using nproc_per_node=1.
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199] Starting elastic_operator with launch configs:
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   entrypoint         : nemo_run.core.runners.fdl_runner
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   min_nodes          : 1
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   max_nodes          : 1
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   nproc_per_node     : 1
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   run_id             : 5789
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   rdzv_backend       : c10d
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   rdzv_endpoint      : localhost:0
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   rdzv_configs       : {'timeout': 900}
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   max_restarts       : 0
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   monitor_interval   : 0.1
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   log_dir            : /home/mihir.mahajan/.nemo_run/experiments/qwen3_600m_full_4k_tokens_34940/qwen3_600m_full_4k_tokens_34940_1764256919/qwen3_600m_full_4k_tokens_34940/nemo_run/qwen3_600m_full_4k_tokens_34940-vpw9h2rcsd2l9/torchelastic/qwen3_600m_full_4k_tokens_34940
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   metrics_cfg        : {}
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199]   event_log_handler  : null
kens_34940/0 I1127 16:22:02.360000 2424402 torch/distributed/launcher/api.py:199] 
kens_34940/0 I1127 16:22:02.365000 2424402 torch/distributed/elastic/agent/server/api.py:869] [default] starting workers for entrypoint: python3
kens_34940/0 I1127 16:22:02.366000 2424402 torch/distributed/elastic/agent/server/api.py:677] [default] Rendezvous'ing worker group
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523] [default] Rendezvous complete for workers. Result:
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   restart_count=0
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   master_addr=hai003.haicore.berlin
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   master_port=33151
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   group_rank=0
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   group_world_size=1
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   local_ranks=[0]
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   role_ranks=[0]
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   global_ranks=[0]
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   role_world_sizes=[1]
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   global_world_sizes=[1]
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523]   event_log_handler=null
kens_34940/0 I1127 16:22:02.667000 2424402 torch/distributed/elastic/agent/server/api.py:523] 
kens_34940/0 I1127 16:22:02.668000 2424402 torch/distributed/elastic/agent/server/api.py:685] [default] Starting worker group
kens_34940/0 I1127 16:22:02.668000 2424402 torch/distributed/elastic/agent/server/local_elastic_agent.py:296] use_agent_store: True
kens_34940/0 I1127 16:22:02.669000 2424402 torch/distributed/elastic/agent/server/local_elastic_agent.py:192] Environment variable 'TORCHELASTIC_ENABLE_FILE_TIMER' not found. Do not start FileTimerServer.
kens_34940/0 I1127 16:22:02.669000 2424402 torch/distributed/elastic/agent/server/local_elastic_agent.py:236] Environment variable 'TORCHELASTIC_HEALTH_CHECK_PORT' not found. Do not start health check.
kens_34940/0 [default0]:/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
kens_34940/0 [default0]:  import pynvml  # type: ignore[import]
kens_34940/0 [default0]:Traceback (most recent call last):
kens_34940/0 [default0]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main
kens_34940/0 [default0]:    return _run_code(code, main_globals, None,
kens_34940/0 [default0]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code
kens_34940/0 [default0]:    exec(code, run_globals)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 72, in <module>
kens_34940/0 [default0]:    fdl_runner_app()
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 327, in __call__
kens_34940/0 [default0]:    raise e
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 310, in __call__
kens_34940/0 [default0]:    [default0]:return get_command(self)(*args, **kwargs)[default0]:
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1485, in __call__
kens_34940/0 [default0]:    return self.main(*args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 719, in main
kens_34940/0 [default0]:    return _main(
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 192, in _main
kens_34940/0 [default0]:    rv = self.invoke(ctx)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1269, in invoke
kens_34940/0 [default0]:    return ctx.invoke(self.callback, **ctx.params)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 824, in invoke
kens_34940/0 [default0]:    return callback(*args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 691, in wrapper
kens_34940/0 [default0]:    return callback(**use_params)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 67, in fdl_direct_run
kens_34940/0 [default0]:    fdl_fn = fdl.build(fdl_buildable)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/building.py", line 185, in build
kens_34940/0 [default0]:    result = daglish.MemoizedTraversal.run(_build, buildable)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 477, in run
kens_34940/0 [default0]:    return fn(root_obj, state)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/building.py", line 176, in _build
kens_34940/0 [default0]:    sub_traversal = state.flattened_map_children(value)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 631, in flattened_map_children
kens_34940/0 [default0]:    return self._flattened_map_children(value, node_traverser)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 579, in _flattened_map_children
kens_34940/0 [default0]:    new_subvalues = [
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 580, in <listcomp>
kens_34940/0 [default0]:    self.call(subvalue, path_element)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 694, in call
kens_34940/0 [default0]:    return self.traversal.apply(value, new_state)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 786, in apply
kens_34940/0 [default0]:    result = self.traversal_fn(value, state)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/building.py", line 176, in _build
kens_34940/0 [default0]:    sub_traversal = state.flattened_map_children(value)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 631, in flattened_map_children
kens_34940/0 [default0]:    return self._flattened_map_children(value, node_traverser)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 579, in _flattened_map_children
kens_34940/0 [default0]:    new_subvalues = [
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 580, in <listcomp>
kens_34940/0 [default0]:    self.call(subvalue, path_element)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 694, in call
kens_34940/0 [default0]:    return self.traversal.apply(value, new_state)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/daglish.py", line 786, in apply
kens_34940/0 [default0]:    result = self.traversal_fn(value, state)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/building.py", line 180, in _build
kens_34940/0 [default0]:    return call_buildable(value, arguments, current_path=state.current_path)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/building.py", line 119, in call_buildable
kens_34940/0 [default0]:    with reraised_exception.try_with_lazy_message(make_message):
kens_34940/0 [default0]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/contextlib.py", line 153, in __exit__
kens_34940/0 [default0]:    self.gen.throw(typ, value, traceback)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/reraised_exception.py", line 82, in try_with_lazy_message
kens_34940/0 [default0]:    raise decorate_exception(exc, message) from None
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/reraised_exception.py", line 74, in try_with_lazy_message
kens_34940/0 [default0]:    yield
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/building.py", line 120, in call_buildable
kens_34940/0 [default0]:    return buildable.__build__(*args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/fiddle/_src/config.py", line 783, in __build__
kens_34940/0 [default0]:    return self.__fn_or_cls__(*args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/io/mixin.py", line 590, in wrapped_init
kens_34940/0 [default0]:    original_init(self, *args, **kwargs)
kens_34940/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py", line 87, in __init__
kens_34940/0 [default0]:    raise ValueError(
kens_34940/0 [default0]:ValueError: Unable to instantiate HuggingFace AUTOTOKENIZER for /fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/. Exception: expected str, bytes or os.PathLike object, not NoneType
kens_34940/0 [default0]:
kens_34940/0 [default0]:Fiddle context: failed to construct or call AutoTokenizer at <root>.model.tokenizer with positional arguments: (), keyword arguments: (pretrained_model_name='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/', use_fast=True, trust_remote_code=True).
kens_34940/0 E1127 16:22:26.452000 2424402 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 0 (pid: 2424417) of binary: /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/bin/python3
kens_34940/0 I1127 16:22:26.457000 2424402 torch/distributed/elastic/multiprocessing/errors/__init__.py:370] ('local_rank %s FAILED with no error file. Decorate your entrypoint fn with @record for traceback info. See: https://pytorch.org/docs/stable/elastic/errors.html', 0)
kens_34940/0 Traceback (most recent call last):
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/bin/torchrun", line 10, in <module>
kens_34940/0     sys.exit(main())
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper
kens_34940/0     return f(*args, **kwargs)
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main
kens_34940/0     run(args)
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run
kens_34940/0     elastic_launch(
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call__
kens_34940/0     return launch_agent(self._config, self._entrypoint, list(args))
kens_34940/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent
kens_34940/0     raise ChildFailedError(
kens_34940/0 torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
kens_34940/0 ============================================================
kens_34940/0 nemo_run.core.runners.fdl_runner FAILED
kens_34940/0 ------------------------------------------------------------
kens_34940/0 Failures:
kens_34940/0   <NO_OTHER_FAILURES>
kens_34940/0 ------------------------------------------------------------
kens_34940/0 Root Cause (first observed failure):
kens_34940/0 [0]:
kens_34940/0   time      : 2025-11-27_16:22:26
kens_34940/0   host      : hai003.haicore.berlin
kens_34940/0   rank      : 0 (local_rank: 0)
kens_34940/0   exitcode  : 1 (pid: 2424417)
kens_34940/0   error_file: <N/A>
kens_34940/0   traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
kens_34940/0 ============================================================
[16:22:28] INFO     Job                                          launcher.py:160
                    qwen3_600m_full_4k_tokens_34940-vpw9h2rcsd2l                
                    9 finished: FAILED                                          
                                                                                
# The experiment was run with the following tasks: ['qwen3_600m_full_4k_tokens_3
# You can inspect and reconstruct this experiment at a later point in time using
experiment = run.Experiment.from_id("qwen3_600m_full_4k_tokens_34940_1764256919"
experiment.status() # Gets the overall status                                   
experiment.logs("qwen3_600m_full_4k_tokens_34940") # Gets the log for the provid
experiment.cancel("qwen3_600m_full_4k_tokens_34940") # Cancels the provided task
                                                                                
                                                                                
# You can inspect this experiment at a later point in time using the CLI as well
nemo experiment status qwen3_600m_full_4k_tokens_34940_1764256919               
nemo experiment logs qwen3_600m_full_4k_tokens_34940_1764256919 0               
nemo experiment cancel qwen3_600m_full_4k_tokens_34940_1764256919 0