[terminal_command] TERMINAL ()
_4k_tokens/0 [default1]:    modelopt_state_path = ckpt_to_weights_subdir(path, is_saving=False) / "modelopt_state"_4k_tokens/0 [default1]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/io/pl.py", line 123, in ckpt_to_weights_subdir_4k_tokens/0 [default1]:    if maybe_base_dir.is_dir() or is_saving:_4k_tokens/0 [default1]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/pathlib.py", line 1305, in is_dir_4k_tokens/0 [default1]:    return S_ISDIR(self.stat().st_mode)_4k_tokens/0 [default1]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/pathlib.py", line 1097, in stat_4k_tokens/0 [default1]:    return self._accessor.stat(self, follow_symlinks=follow_symlinks)_4k_tokens/0 [default1]:PermissionError: [Errno 13] Permission denied: '/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-coder-30b-a3b-instruct/weights'_4k_tokens/0 [default0]:GPU available: True (cuda), used: True_4k_tokens/0 [default0]:TPU available: False, using: 0 TPU cores_4k_tokens/0 [default0]:HPU available: False, using: 0 HPUs_4k_tokens/0 [default0]:Traceback (most recent call last):_4k_tokens/0 [default0]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 196, in _run_module_as_main_4k_tokens/0 [default0]:    return _run_code(code, main_globals, None,_4k_tokens/0 [default0]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/runpy.py", line 86, in _run_code_4k_tokens/0 [default0]:    exec(code, run_globals)_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 72, in <module>_4k_tokens/0 [default0]:    fdl_runner_app()_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 327, in __call___4k_tokens/0 [default0]:    raise e[default0]:_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 310, in __call___4k_tokens/0 [default0]:    [default0]:return get_command(self)(*args, **kwargs)[default0]:_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1485, in __call___4k_tokens/0 [default0]:    [default0]:return self.main(*args, **kwargs)[default0]:_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 719, in main_4k_tokens/0 [default0]:    [default0]:return _main(_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/core.py", line 192, in _main_4k_tokens/0 [default0]:    [default0]:rv = self.invoke(ctx)[default0]:_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 1269, in invoke_4k_tokens/0 [default0]:    [default0]:return ctx.invoke(self.callback, **ctx.params)_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/click/core.py", line 824, in invoke_4k_tokens/0 [default0]:    return callback(*args, **kwargs)_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/typer/main.py", line 691, in wrapper_4k_tokens/0 [default0]:    return callback(**use_params)_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/core/runners/fdl_runner.py", line 68, in fdl_direct_run_4k_tokens/0 [default0]:    fdl_fn()_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 237, in finetune_4k_tokens/0 [default0]:    return train(_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/api.py", line 121, in train_4k_tokens/0 [default0]:    set_modelopt_spec_if_exists_in_ckpt(model, resume.restore_config.path)_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/collections/llm/modelopt/model_utils.py", line 84, in set_modelopt_spec_if_exists_in_ckpt_4k_tokens/0 [default0]:    modelopt_state_path = ckpt_to_weights_subdir(path, is_saving=False) / "modelopt_state"_4k_tokens/0 [default0]:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo/lightning/io/pl.py", line 123, in ckpt_to_weights_subdir_4k_tokens/0 [default0]:    if maybe_base_dir.is_dir() or is_saving:_4k_tokens/0 [default0]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/pathlib.py", line 1305, in is_dir_4k_tokens/0 [default0]:    return S_ISDIR(self.stat().st_mode)_4k_tokens/0 [default0]:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/pathlib.py", line 1097, in stat_4k_tokens/0 [default0]:    return self._accessor.stat(self, follow_symlinks=follow_symlinks)_4k_tokens/0 [default0]:PermissionError: [Errno 13] Permission denied: '/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-coder-30b-a3b-instruct/weights'^C_4k_tokens/0 E1123 13:05:58.568000 3880733 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 0 (pid: 3880744) of binary: /fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/bin/python3_4k_tokens/0 I1123 13:05:58.575000 3880733 torch/distributed/elastic/multiprocessing/errors/__init__.py:370] ('local_rank %s FAILED with no error file. Decorate your entrypoint fn with @record for traceback info. See: https://pytorch.org/docs/stable/elastic/errors.html', 0)_4k_tokens/0 Traceback (most recent call last):_4k_tokens/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/bin/torchrun", line 10, in <module>_4k_tokens/0     sys.exit(main())_4k_tokens/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper_4k_tokens/0     return f(*args, **kwargs)_4k_tokens/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/run.py", line 901, in main_4k_tokens/0     run(args)_4k_tokens/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/run.py", line 892, in run_4k_tokens/0     elastic_launch(_4k_tokens/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 143, in __call___4k_tokens/0     return launch_agent(self._config, self._entrypoint, list(args))_4k_tokens/0   File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 277, in launch_agent_4k_tokens/0     raise ChildFailedError(_4k_tokens/0 torch.distributed.elastic.multiprocessing.errors.ChildFailedError: _4k_tokens/0 ============================================================_4k_tokens/0 nemo_run.core.runners.fdl_runner FAILED_4k_tokens/0 ------------------------------------------------------------_4k_tokens/0 Failures:_4k_tokens/0 [1]:_4k_tokens/0   time      : 2025-11-23_13:05:58_4k_tokens/0   host      : hai003.haicore.berlin_4k_tokens/0   rank      : 1 (local_rank: 1)_4k_tokens/0   exitcode  : 1 (pid: 3880745)_4k_tokens/0   error_file: <N/A>_4k_tokens/0   traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html_4k_tokens/0 ------------------------------------------------------------_4k_tokens/0 Root Cause (first observed failure):_4k_tokens/0 [0]:_4k_tokens/0   time      : 2025-11-23_13:05:58_4k_tokens/0   host      : hai003.haicore.berlin_4k_tokens/0   rank      : 0 (local_rank: 0)_4k_tokens/0   exitcode  : 1 (pid: 3880744)_4k_tokens/0   error_file: <N/A>_4k_tokens/0   traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html_4k_tokens/0 ============================================================^C                                                                                                                                                                          # The experiment was run with the following tasks: ['qwen3_30b_a3b_lora_finetune_crowd_code_dataset_4k_tokens']                                                           # You can inspect and reconstruct this experiment at a later point in time using:                                                                                         experiment = run.Experiment.from_id("qwen3_30b_a3b_lora_finetune_crowd_code_dataset_4k_tokens_1763899521")                                                                experiment.status() # Gets the overall status                                                                                                                             experiment.logs("qwen3_30b_a3b_lora_finetune_crowd_code_dataset_4k_tokens") # Gets the log for the provided task                                                          experiment.cancel("qwen3_30b_a3b_lora_finetune_crowd_code_dataset_4k_tokens") # Cancels the provided task if still running                                                                                                                                                                                                                                                                                                                                                                                                    # You can inspect this experiment at a later point in time using the CLI as well:                                                                                         nemo experiment status qwen3_30b_a3b_lora_finetune_crowd_code_dataset_4k_tokens_1763899521                                                                                nemo experiment logs qwen3_30b_a3b_lora_finetune_crowd_code_dataset_4k_tokens_1763899521 0                                                                                nemo experiment cancel qwen3_30b_a3b_lora_finetune_crowd_code_dataset_4k_tokens_1763899521 0                                                                                                                                                                                                                                                        Traceback (most recent call last):  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/run/experiment.py", line 801, in _wait_for_jobs    for future in as_completed(futures.keys()):  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/concurrent/futures/_base.py", line 245, in as_completed    waiter.event.wait(wait_timeout)  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/threading.py", line 607, in wait    signaled = self._cond.wait(timeout)  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/threading.py", line 320, in wait    waiter.acquire()KeyboardInterruptDuring handling of the above exception, another exception occurred:Traceback (most recent call last):  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/slurm/dev/franz/berlin/crowd-pilot/nemo/test_lora_finetune_4k.py", line 58, in <module>    run_finetuning()  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/slurm/dev/franz/berlin/crowd-pilot/nemo/test_lora_finetune_4k.py", line 54, in run_finetuning    run.run(recipe, executor=executor, name=NAME)  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/run/api.py", line 79, in run    with Experiment(title=name, executor=executor, log_level=log_level) as exp:  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/run/experiment.py", line 1138, in __exit__    self._wait_for_jobs(jobs=self.jobs)  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/nemo_run/run/experiment.py", line 779, in _wait_for_jobs    with ThreadPoolExecutor(initializer=set_context, initargs=(context,)) as executor:  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/concurrent/futures/_base.py", line 649, in __exit__    self.shutdown(wait=True)  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/concurrent/futures/thread.py", line 235, in shutdown    t.join()  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/threading.py", line 1096, in join    self._wait_for_tstate_lock()  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/threading.py", line 1116, in _wait_for_tstate_lock    if lock.acquire(block, timeout):KeyboardInterrupt^C^CException ignored in atexit callback: <function shutdown_compile_workers at 0x7f6efca3f490>Traceback (most recent call last):  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_inductor/async_compile.py", line 142, in shutdown_compile_workers    pool.shutdown()  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/_inductor/compile_worker/subproc_pool.py", line 243, in shutdown    self.process.wait(300)  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/subprocess.py", line 1222, in wait    self._wait(timeout=sigint_timeout)  File "/home/mihir.mahajan/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/lib/python3.10/subprocess.py", line 1953, in _wait    time.sleep(delay)KeyboardInterrupt: ^C^C(.uv_venv) [mihir.mahajan@hai003.haicore.berlin:~/Projects/crowd-pilot] $ ^C(.uv_venv) [mihir.mahajan@hai003.haicore.berlin:~/Projects/crowd-pilot] $ exitsrun: error: hai003: task 0: Exited with exit code 130salloc: Relinquishing job allocation 34557salloc: Job allocation 34557 has been revoked.