[terminal_command] TERMINAL ()
(jafar) [franz.srambical@hai006.haicore.berlin:~/jafar] $ salloc --gpus=1 --ntasks-per-node=1 --cpus-per-task=1 --mem=100Gsalloc: Granted job allocation 13982salloc: Nodes hai002 are ready for jobRunning inside SLURM, Job ID 13982.[franz.srambical@hai002.haicore.berlin:~/jafar] $ srun: error: hai006: task 0: Killed(jafar) [franz.srambical@hai-login1.haicore.berlin:~/jafar] $ bash experiments/sample.sh 2025-07-26 21:17:21.066503: E external/xla/xla/stream_executor/cuda/cuda_platform.cc:51] failed call to cuInit: INTERNAL: CUDA error: Failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detectedTraceback (most recent call last):  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 742, in backends    backend = _init_backend(platform)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 828, in _init_backend    backend = registration.factory()  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/jax/_src/xla_bridge.py", line 528, in factory[franz.srambical@hai001.haicore.berlin:~/jafar] $ bash experiments/sample.sh Traceback (most recent call last):  File "/fast/home/franz.srambical/jafar/sample.py", line 74, in <module>    jasmine = Jasmine(  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 143, in __call__    return _graph_node_meta_call(cls, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 152, in _graph_node_meta_call    cls._object_meta_construct(node, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 146, in _object_meta_construct    self.__init__(*args, **kwargs)  File "/fast/home/franz.srambical/jafar/jasmine.py", line 80, in __init__    self.tokenizer = TokenizerVQVAE(  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 143, in __call__    return _graph_node_meta_call(cls, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 152, in _graph_node_meta_call    cls._object_meta_construct(node, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 146, in _object_meta_construct    self.__init__(*args, **kwargs)  File "/fast/home/franz.srambical/jafar/models/tokenizer.py", line 45, in __init__    self.encoder = STTransformer(  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 143, in __call__    return _graph_node_meta_call(cls, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 152, in _graph_node_meta_call    cls._object_meta_construct(node, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 146, in _object_meta_construct    self.__init__(*args, **kwargs)[franz.srambical@hai004.haicore.berlin:~/jafar] $ bash experiments/sample.sh Traceback (most recent call last):  File "/fast/home/franz.srambical/jafar/sample.py", line 74, in <module>    jasmine = Jasmine(  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 143, in __call__    return _graph_node_meta_call(cls, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 152, in _graph_node_meta_call    cls._object_meta_construct(node, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 146, in _object_meta_construct    self.__init__(*args, **kwargs)  File "/fast/home/franz.srambical/jafar/jasmine.py", line 80, in __init__    self.tokenizer = TokenizerVQVAE(  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 143, in __call__    return _graph_node_meta_call(cls, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 152, in _graph_node_meta_call    cls._object_meta_construct(node, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 146, in _object_meta_construct    self.__init__(*args, **kwargs)  File "/fast/home/franz.srambical/jafar/models/tokenizer.py", line 45, in __init__    self.encoder = STTransformer(  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 143, in __call__    return _graph_node_meta_call(cls, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 152, in _graph_node_meta_call    cls._object_meta_construct(node, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 146, in _object_meta_construct    self.__init__(*args, **kwargs)  File "/fast/home/franz.srambical/jafar/utils/nn.py", line 202, in __init__    STBlock(  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 143, in __call__    return _graph_node_meta_call(cls, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 152, in _graph_node_meta_call    cls._object_meta_construct(node, *args, **kwargs)  File "/fast/home/franz.srambical/jafar/.venv/lib/python3.10/site-packages/flax/nnx/object.py", line 146, in _object_meta_construct    self.__init__(*args, **kwargs)  File "/fast/home/franz.srambical/jafar/utils/nn.py", line 69, in __init__    attention_fn=_create_flash_attention_fn(TypeError: _create_flash_attention_fn() got an unexpected keyword argument 'decode'srun: error: hai004: task 0: Exited with exit code 1[franz.srambical@hai004.haicore.berlin:~/jafar] $ srun: error: hai001: task 0: Killed