decode this error for me please ,     570 )
    572 return engine

File ~/lalb/vllm/vllm/engine/llm_engine.py:325, in LLMEngine.__init__(self, model_config, cache_config, parallel_config, scheduler_config, device_config, load_config, lora_config, speculative_config, decoding_config, observability_config, prompt_adapter_config, executor_class, log_stats, usage_context, stat_loggers, input_registry, use_cached_outputs)
    321 self.input_registry = input_registry
    322 self.input_processor = input_registry.create_input_processor(
    323     model_config)
--> 325 self.model_executor = executor_class(
    326     model_config=model_config,
    327     cache_config=cache_config,
    328     parallel_config=parallel_config,
    329     scheduler_config=scheduler_config,
    330     device_config=device_config,
    331     lora_config=lora_config,
    332     speculative_config=speculative_config,
    333     load_config=load_config,
    334     prompt_adapter_config=prompt_adapter_config,
    335     observability_config=self.observability_config,
    336 )
    338 if not self.model_config.embedding_mode:
    339     self._initialize_kv_caches()

File ~/lalb/vllm/vllm/executor/ray_tpu_executor.py:39, in RayTPUExecutor.__init__(self, *args, **kwargs)
     35 # Updated by implementations that require additional args to be passed
     36 # to the _run_workers execute_model call
     37 self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
---> 39 super().__init__(*args, **kwargs)

File ~/lalb/vllm/vllm/executor/executor_base.py:47, in ExecutorBase.__init__(self, model_config, cache_config, parallel_config, scheduler_config, device_config, load_config, lora_config, speculative_config, prompt_adapter_config, observability_config)
     45 self.prompt_adapter_config = prompt_adapter_config
     46 self.observability_config = observability_config
---> 47 self._init_executor()

File ~/lalb/vllm/vllm/executor/ray_tpu_executor.py:51, in RayTPUExecutor._init_executor(self)
     48     os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
     50 # Create the parallel TPU workers.
---> 51 self._init_workers_ray(placement_group)

File ~/lalb/vllm/vllm/executor/ray_tpu_executor.py:197, in RayTPUExecutor._init_workers_ray(self, placement_group, **ray_remote_kwargs)
    188 init_worker_all_kwargs = [
    189     self._get_worker_kwargs(
    190         local_rank=node_workers[node_id].index(rank),
   (...)
    193     ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
    194 ]
    195 self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
--> 197 self._run_workers("init_device")
    198 self._run_workers("load_model",
    199                   max_concurrent_workers=self.parallel_config.
    200                   max_parallel_loading_workers)

File ~/lalb/vllm/vllm/executor/ray_tpu_executor.py:264, in RayTPUExecutor._run_workers(self, method, async_run_remote_workers_only, all_args, all_kwargs, use_dummy_driver, max_concurrent_workers, use_ray_compiled_dag, *args, **kwargs)
    262 # Start the driver worker after all the ray workers.
    263 if not use_dummy_driver:
--> 264     driver_worker_output = self.driver_worker.execute_method(
    265         method, *driver_args, **driver_kwargs)
    266 else:
    267     assert self.driver_dummy_worker is not None

File ~/lalb/vllm/vllm/worker/worker_base.py:465, in WorkerWrapperBase.execute_method(self, method, *args, **kwargs)
    462 msg = (f"Error executing method {method}. "
    463        "This might cause deadlock in distributed execution.")
    464 logger.exception(msg)
--> 465 raise e

File ~/lalb/vllm/vllm/worker/worker_base.py:456, in WorkerWrapperBase.execute_method(self, method, *args, **kwargs)
    454     target = self if self.worker is None else self.worker
    455     executor = getattr(target, method)
--> 456     return executor(*args, **kwargs)
    457 except Exception as e:
    458     # if the driver worker also execute methods,
    459     # exceptions in the rest worker may cause deadlock in rpc like ray
    460     # see https://github.com/vllm-project/vllm/issues/3455
    461     # print the error and inform the user to solve the error
    462     msg = (f"Error executing method {method}. "
    463            "This might cause deadlock in distributed execution.")

File ~/lalb/vllm/vllm/worker/tpu_worker.py:83, in TPUWorker.init_device(self)
     72 # NOTE(woosuk): This is just to initialize the TP group and broadcast
     73 # the input objects on CPU. The all-reduce and all-gather ops on TPU
     74 # are invoked by `xm.all_reduce` and `xm.all_gather` which use their
     75 # own context.
     76 init_distributed_environment(
     77     world_size=self.parallel_config.world_size,
     78     rank=self.rank,
   (...)
     81     backend="gloo",
     82 )
---> 83 ensure_model_parallel_initialized(
     84     self.parallel_config.tensor_parallel_size,
     85     self.parallel_config.pipeline_parallel_size)
     87 # Device initialization should happen after initializing the distributed
     88 # runtime.
     89 self.device = xm.xla_device()

File ~/lalb/vllm/vllm/distributed/parallel_state.py:1059, in ensure_model_parallel_initialized(tensor_model_parallel_size, pipeline_model_parallel_size, backend)
   1056 backend = backend or torch.distributed.get_backend(
   1057     get_world_group().device_group)
   1058 if not model_parallel_is_initialized():
-> 1059     initialize_model_parallel(tensor_model_parallel_size,
   1060                               pipeline_model_parallel_size, backend)
   1061     return
   1063 assert (
   1064     get_tensor_model_parallel_world_size() == tensor_model_parallel_size
   1065 ), ("tensor parallel group already initialized, but of unexpected size: "
   1066     f"{get_tensor_model_parallel_world_size()=} vs. "
   1067     f"{tensor_model_parallel_size=}")

File ~/lalb/vllm/vllm/distributed/parallel_state.py:1023, in initialize_model_parallel(tensor_model_parallel_size, pipeline_model_parallel_size, backend)
   1020     group_ranks.append(ranks)
   1022 # message queue broadcaster is only used in tensor model parallel group
-> 1023 _TP = init_model_parallel_group(group_ranks,
   1024                                 get_world_group().local_rank,
   1025                                 backend,
   1026                                 use_message_queue_broadcaster=True,
   1027                                 group_name="tp")
   1029 # Build the pipeline model-parallel groups.
   1030 num_pipeline_model_parallel_groups: int = (world_size //
   1031                                            pipeline_model_parallel_size)

File ~/lalb/vllm/vllm/distributed/parallel_state.py:864, in init_model_parallel_group(group_ranks, local_rank, backend, use_custom_allreduce, use_message_queue_broadcaster, group_name)
    862 if use_custom_allreduce is None:
    863     use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
--> 864 return GroupCoordinator(
    865     group_ranks=group_ranks,
    866     local_rank=local_rank,
    867     torch_distributed_backend=backend,
    868     use_pynccl=True,
    869     use_custom_allreduce=use_custom_allreduce,
    870     use_tpu_communicator=True,
    871     use_message_queue_broadcaster=use_message_queue_broadcaster,
    872     group_name=group_name,
    873 )

File ~/lalb/vllm/vllm/distributed/parallel_state.py:231, in GroupCoordinator.__init__(self, group_ranks, local_rank, torch_distributed_backend, use_pynccl, use_custom_allreduce, use_tpu_communicator, use_message_queue_broadcaster, group_name)
    229 self.tpu_communicator: Optional[TpuCommunicator] = None
    230 if use_tpu_communicator and self.world_size > 1:
--> 231     self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
    233 from vllm.distributed.device_communicators.shm_broadcast import (
    234     MessageQueue)
    235 self.mq_broadcaster: Optional[MessageQueue] = None

File ~/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py:53, in TpuCommunicator.__init__(self, group)
     50 os.environ["CLOUD_TPU_TASK_ID"] = str(global_rank)
     51 os.environ["TPU_VISIBLE_CHIPS"] = str(local_rank)
---> 53 pjrt.initialize_multiprocess(local_rank, local_world_size)
     54 xr._init_world_size_ordinal()

File ~/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py:121, in initialize_multiprocess(local_rank, local_world_size)
    118 elif runtime.device_type() == 'NEURON':
    119   neuron.initialize_env(local_rank, local_world_size)
--> 121 devices = xm.get_xla_supported_devices()
    122 xm.set_replication(xm.xla_device(), devices)

File ~/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py:93, in get_xla_supported_devices(devkind, max_devices)
     90 # TODO(wcromar): Remove `devkind` after 2.3 release cut. We no longer support
     91 # multiple device types.
     92 if not devkind:
---> 93   devices = torch_xla._XLAC._xla_get_devices()
     94   return [
     95       f'xla:{i}'
     96       for i, _ in enumerate(devices[:max_devices] if max_devices else devices)
     97   ]
     98 else:

RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,543	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102895, ip=10.130.0.29, actor_id=7f8628802a0597f532e47c4d05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7efdc6b54370>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,545	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102242, ip=10.130.0.28, actor_id=f9be6c3c131a5388e36c675f05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f8cca7d4340>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,545	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=107836, ip=10.130.0.31, actor_id=36bfc0b00d2f6b8f6e679d4605000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7fc7eadb43a0>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,546	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=107954, ip=10.130.0.31, actor_id=981d5cf72d8e3e73ae805ad805000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f19ddfe0460>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,547	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=108195, ip=10.130.0.31, actor_id=d95938ac69d962262223027105000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f47518c8460>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,547	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=358493, ip=10.130.0.30, actor_id=cdceaff2c60b22f8a44b6f3a05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f106e20cc70>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,548	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=358251, ip=10.130.0.30, actor_id=bf0d12344c31f6814e6ba14905000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7facc7f0cb50>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,549	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102774, ip=10.130.0.29, actor_id=b0b2dfe4b5573a904c7c9afa05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f6a2cd9c370>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,549	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102642, ip=10.130.0.29, actor_id=aff0082536fdfa98501960d105000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7ef3fdf5c3d0>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,550	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=358372, ip=10.130.0.30, actor_id=5bc8014451c573642bead5b805000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f5a7f56cc40>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,551	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102124, ip=10.130.0.28, actor_id=79b5099eb5a9274e1b9b75f405000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f2851dc83d0>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,551	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=108075, ip=10.130.0.31, actor_id=4cb98bf15cd4d28c339203ff05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7ef72e2f0430>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,552	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102480, ip=10.130.0.28, actor_id=5a63b3a07dfd9368f3d8145305000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f1fe20e0370>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,553	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=103014, ip=10.130.0.29, actor_id=2229b24714284766de33cfbc05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7f995c5ec400>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
2024-09-24 04:16:22,554	ERROR worker.py:409 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=102361, ip=10.130.0.28, actor_id=e7639c923ffb04993a07e4cc05000000, repr=<vllm.executor.ray_utils.RayWorkerWrapper object at 0x7faf4b3203d0>)
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 465, in execute_method
    raise e
  File "/home/spars/lalb/vllm/vllm/worker/worker_base.py", line 456, in execute_method
    return executor(*args, **kwargs)
  File "/home/spars/lalb/vllm/vllm/worker/tpu_worker.py", line 83, in init_device
    ensure_model_parallel_initialized(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1059, in ensure_model_parallel_initialized
    initialize_model_parallel(tensor_model_parallel_size,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 1023, in initialize_model_parallel
    _TP = init_model_parallel_group(group_ranks,
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 864, in init_model_parallel_group
    return GroupCoordinator(
  File "/home/spars/lalb/vllm/vllm/distributed/parallel_state.py", line 231, in __init__
    self.tpu_communicator = TpuCommunicator(group=self.cpu_group)
  File "/home/spars/lalb/vllm/vllm/distributed/device_communicators/tpu_communicator.py", line 53, in __init__
    pjrt.initialize_multiprocess(local_rank, local_world_size)
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/_internal/pjrt.py", line 121, in initialize_multiprocess
    devices = xm.get_xla_supported_devices()
  File "/home/spars/.local/lib/python3.10/site-packages/torch_xla/core/xla_model.py", line 93, in get_xla_supported_devices
    devices = torch_xla._XLAC._xla_get_devices()
RuntimeError: Bad StatusOr access: UNKNOWN: TPU initialization failed: Invalid --2a886c8_slice_builder_worker_addresses specified. Expected 16 worker addresses, got 4.
tt = llm.get_tokenizer()
 I am running it on TPU V4-32 , with ray setup as this ======== Autoscaler status: 2024-09-24 04:19:17.599164 ========
Node status
---------------------------------------------------------------
Active:
 1 node_a719383e498cb023fd7e8139291e66861cb40bd13862a676f52ac5e9
 1 node_8415f9cdd51ddbb01cab862238db4f51e7c47fee37cc7eddb477e3d9
 1 node_00327d1caf66f2ac5ed52dd91d724dc49662d7284afb6d90befa4e77
 1 node_d6e3b308eb1f88faa69cb0c2bdf9749bacd3c51c79b37487e08d0a46
Pending:
 (no pending nodes)
Recent failures:
 (no failures)

Resources
---------------------------------------------------------------
Usage:
 0.0/960.0 CPU
 16.0/16.0 TPU (16.0 used of 16.0 reserved in placement groups)
 0.0/1.0 TPU-v4-32-head
 0.0/4.0 fspar
 0B/1.08TiB memory
 0B/476.45GiB object_store_memory

Demands:
 (no resource demands)