Bug: Skip `TorchDistributorLocalUnitTests.test_end_to_end_run_locally` with Python 3.12
Status: Resolved | Priority: Minor | Resolution: Fixed
Created: 29/Nov/23 01:30
Affects Version/s: 4.0.0

{code}
======================================================================
ERROR [12.635s]: test_end_to_end_run_locally (pyspark.ml.tests.connect.test_parity_torch_distributor.TorchDistributorLocalUnitTestsIIOnConnect.test_end_to_end_run_locally)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/__w/spark/spark/python/pyspark/ml/torch/tests/test_distributor.py", line 403, in test_end_to_end_run_locally
    output = TorchDistributor(num_processes=2, local_mode=True, use_gpu=False).run(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 969, in run
    return self._run(
           ^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 985, in _run
    output = self._run_local_training(
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 593, in _run_local_training
    output = TorchDistributor._get_output_from_framework_wrapper(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 567, in _get_output_from_framework_wrapper
    return framework_wrapper(
           ^^^^^^^^^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 908, in _run_training_on_pytorch_function
    raise RuntimeError(
RuntimeError: TorchDistributor failed during training.View stdout logs for detailed error message.

======================================================================
ERROR [14.850s]: test_end_to_end_run_locally (pyspark.ml.tests.connect.test_parity_torch_distributor.TorchDistributorLocalUnitTestsOnConnect.test_end_to_end_run_locally)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/__w/spark/spark/python/pyspark/ml/torch/tests/test_distributor.py", line 403, in test_end_to_end_run_locally
    output = TorchDistributor(num_processes=2, local_mode=True, use_gpu=False).run(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 969, in run
    return self._run(
           ^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 985, in _run
    output = self._run_local_training(
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 593, in _run_local_training
    output = TorchDistributor._get_output_from_framework_wrapper(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 567, in _get_output_from_framework_wrapper
    return framework_wrapper(
           ^^^^^^^^^^^^^^^^^^
  File "/__w/spark/spark/python/pyspark/ml/torch/distributor.py", line 908, in _run_training_on_pytorch_function
    raise RuntimeError(
RuntimeError: TorchDistributor failed during training.View stdout logs for detailed error message.

----------------------------------------------------------------------
{code}

https://github.com/apache/spark/actions/runs/7020654429/job/19100964890