vllm loading error
ERROR 12-03 00:52:08 engine.py:366] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/utils.py", line 101, in
ERROR 12-03 00:52:08 engine.py:366] for weight_name, weight_data in weights)
ERROR 12-03 00:52:08 engine.py:366] ^^^^^^^
ERROR 12-03 00:52:08 engine.py:366] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/models/llama.py", line 588, in
ERROR 12-03 00:52:08 engine.py:366] for name, loaded_weight in weights)
ERROR 12-03 00:52:08 engine.py:366] ^^^^^^^
ERROR 12-03 00:52:08 engine.py:366] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/loader.py", line 877, in _quantized_4bit_generator
ERROR 12-03 00:52:08 engine.py:366] quant_state = _parse_quant_state(weight_name, temp_state_dict)
ERROR 12-03 00:52:08 engine.py:366] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 12-03 00:52:08 engine.py:366] File "/usr/local/lib/python3.12/dist-packages/vllm/model_executor/model_loader/loader.py", line 863, in _parse_quant_state
ERROR 12-03 00:52:08 engine.py:366] return QuantState.from_dict(quant_state, device="cuda")
ERROR 12-03 00:52:08 engine.py:366] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 12-03 00:52:08 engine.py:366] File "/usr/local/lib/python3.12/dist-packages/bitsandbytes/functional.py", line 728, in from_dict
ERROR 12-03 00:52:08 engine.py:366] qs_dict.update(unpack_tensor_to_dict(qs_dict.pop(first_qs_key)))
ERROR 12-03 00:52:08 engine.py:366] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 12-03 00:52:08 engine.py:366] File "/usr/local/lib/python3.12/dist-packages/bitsandbytes/utils.py", line 198, in unpack_tensor_to_dict
ERROR 12-03 00:52:08 engine.py:366] unpacked_dict = json.loads(json_str)
ERROR 12-03 00:52:08 engine.py:366] ^^^^^^^^^^^^^^^^^^^^
ERROR 12-03 00:52:08 engine.py:366] File "/usr/lib/python3.12/json/init.py", line 346, in loads
ERROR 12-03 00:52:08 engine.py:366] return _default_decoder.decode(s)
ERROR 12-03 00:52:08 engine.py:366] ^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 12-03 00:52:08 engine.py:366] File "/usr/lib/python3.12/json/decoder.py", line 337, in decode
ERROR 12-03 00:52:08 engine.py:366] obj, end = self.raw_decode(s, idx=_w(s, 0).end())
ERROR 12-03 00:52:08 engine.py:366] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
ERROR 12-03 00:52:08 engine.py:366] File "/usr/lib/python3.12/json/decoder.py", line 355, in raw_decode
ERROR 12-03 00:52:08 engine.py:366] raise JSONDecodeError("Expecting value", s, err.value) from None
ERROR 12-03 00:52:08 engine.py:366] json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
File "/usr/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
self.run()