[content] /fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/run_qwen3-0.6b-sft-torch_36268.log (log)
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Waiting for Ray Dashboard...
Traceback (most recent call last):
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/connection.py", line 204, in _new_conn
    sock = connection.create_connection(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/util/connection.py", line 85, in create_connection
    raise err
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/util/connection.py", line 73, in create_connection
    sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/connectionpool.py", line 787, in urlopen
    response = self._make_request(
               ^^^^^^^^^^^^^^^^^^^
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/connectionpool.py", line 493, in _make_request
    conn.request(
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/connection.py", line 500, in request
    self.endheaders()
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/http/client.py", line 1333, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/http/client.py", line 1093, in _send_output
    self.send(msg)
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/http/client.py", line 1037, in send
    self.connect()
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/connection.py", line 331, in connect
    self.sock = self._new_conn()
                ^^^^^^^^^^^^^^^^
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/connection.py", line 219, in _new_conn
    raise NewConnectionError(
urllib3.exceptions.NewConnectionError: HTTPConnection(host='10.86.2.80', port=8265): Failed to establish a new connection: [Errno 111] Connection refused

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/requests/adapters.py", line 644, in send
    resp = conn.urlopen(
           ^^^^^^^^^^^^^
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/connectionpool.py", line 841, in urlopen
    retries = retries.increment(
              ^^^^^^^^^^^^^^^^^^
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/urllib3/util/retry.py", line 519, in increment
    raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='10.86.2.80', port=8265): Max retries exceeded with url: /api/version (Caused by NewConnectionError("HTTPConnection(host='10.86.2.80', port=8265): Failed to establish a new connection: [Errno 111] Connection refused"))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/dashboard/modules/dashboard_sdk.py", line 269, in _check_connection_and_version_with_url
    r = self._do_request("GET", url)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/dashboard/modules/dashboard_sdk.py", line 311, in _do_request
    response = requests.request(
               ^^^^^^^^^^^^^^^^^
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/requests/api.py", line 59, in request
    return session.request(method=method, url=url, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/requests/sessions.py", line 589, in request
    resp = self.send(prep, **send_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/requests/sessions.py", line 703, in send
    r = adapter.send(request, **kwargs)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/project/HFMI_SynergyUnit/mihir/micromamba/envs/miles/lib/python3.12/site-packages/requests/adapters.py", line 677, in send
    raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='10.86.2.80', port=8265): Max retries exceeded with url: /api/version (Caused by NewConnectionError("HTTPConnection(host='10.86.2.80', port=8265): Failed to establish a new connection: [Errno 111] Connection refused"))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/scripts/scripts.py", line 2847, in <module>
    main()
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/scripts/scripts.py", line 2843, in main
    return cli()
           ^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/click/core.py", line 1442, in __call__
    return self.main(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/click/core.py", line 1363, in main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/click/core.py", line 1830, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/click/core.py", line 1830, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/click/core.py", line 1226, in invoke
    return ctx.invoke(self.callback, **ctx.params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/click/core.py", line 794, in invoke
    return callback(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/dashboard/modules/job/cli_utils.py", line 54, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/autoscaler/_private/cli_logger.py", line 823, in wrapper
    return f(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/dashboard/modules/job/cli.py", line 269, in submit
    client = _get_sdk_client(
             ^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/dashboard/modules/job/cli.py", line 34, in _get_sdk_client
    client = JobSubmissionClient(
             ^^^^^^^^^^^^^^^^^^^^
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/dashboard/modules/job/sdk.py", line 106, in __init__
    self._check_connection_and_version(
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/dashboard/modules/dashboard_sdk.py", line 255, in _check_connection_and_version
    self._check_connection_and_version_with_url(min_version, version_error_message)
  File "/home/mihir.mahajan/.local/lib/python3.12/site-packages/ray/dashboard/modules/dashboard_sdk.py", line 285, in _check_connection_and_version_with_url
    raise ConnectionError(
ConnectionError: Failed to connect to Ray at address: http://10.86.2.80:8265.
srun: error: hai008: task 0: Exited with exit code 1