Bug: Enable pyspark.sql.connect.column.Column.withField doctest
Status: Resolved | Priority: Major | Resolution: Fixed
Created: 29/Dec/22 12:37
Affects Version/s: 3.4.0

It fails as below:

{code}
File "/.../spark/python/pyspark/sql/connect/column.py", line 391, in pyspark.sql.connect.column.Column.withField
Failed example:
    df.withColumn('a', df['a'].withField('b', lit(3))).select('a.b').show()
Exception raised:
    Traceback (most recent call last):
      File "/.../miniconda3/envs/python3.9/lib/python3.9/doctest.py", line 1336, in __run
        exec(compile(example.source, filename, "single",
      File "<doctest pyspark.sql.connect.column.Column.withField[3]>", line 1, in <module>
        df.withColumn('a', df['a'].withField('b', lit(3))).select('a.b').show()
      File "/.../spark/python/pyspark/sql/connect/dataframe.py", line 538, in show
        print(self._show_string(n, truncate, vertical))
      File "/.../spark/python/pyspark/sql/connect/dataframe.py", line 424, in _show_string
        pdf = DataFrame.withPlan(
      File "/.../python/pyspark/sql/connect/dataframe.py", line 910, in toPandas
        return self._session.client.to_pandas(query)
      File "/.../python/pyspark/sql/connect/client.py", line 413, in to_pandas
        return self._execute_and_fetch(req)
      File "/.../spark/python/pyspark/sql/connect/client.py", line 573, in _execute_and_fetch
        self._handle_error(rpc_error)
      File "/.../spark/python/pyspark/sql/connect/client.py", line 627, in _handle_error
        raise SparkConnectException(str(rpc_error)) from None
    pyspark.sql.connect.client.SparkConnectException: <_MultiThreadedRendezvous of RPC that terminated with:
    	status = StatusCode.UNKNOWN
    	details = "Expression with ID: 0 is not supported"
    	debug_error_string = "UNKNOWN:Error received from peer ipv6:%5B::1%5D:15002 {grpc_message:"Expression with ID: 0 is not supported", grpc_status:2, created_time:"2022-12-29T21:25:46.707558+09:00"}"
    >
**********************************************************************
File "/Users/hyukjin.kwon/workspace/forked/spark/python/pyspark/sql/connect/column.py", line 397, in pyspark.sql.connect.column.Column.withField
Failed example:
    df.withColumn('a', df['a'].withField('d', lit(4))).select('a.d').show()
Exception raised:
    Traceback (most recent call last):
      File "/.../miniconda3/envs/python3.9/lib/python3.9/doctest.py", line 1336, in __run
        exec(compile(example.source, filename, "single",
      File "<doctest pyspark.sql.connect.column.Column.withField[4]>", line 1, in <module>
        df.withColumn('a', df['a'].withField('d', lit(4))).select('a.d').show()
      File "/.../spark/python/pyspark/sql/connect/dataframe.py", line 538, in show
        print(self._show_string(n, truncate, vertical))
      File "/.../spark/python/pyspark/sql/connect/dataframe.py", line 424, in _show_string
        pdf = DataFrame.withPlan(
      File "/.../spark/python/pyspark/sql/connect/dataframe.py", line 910, in toPandas
        return self._session.client.to_pandas(query)
      File "/.../spark/python/pyspark/sql/connect/client.py", line 413, in to_pandas
        return self._execute_and_fetch(req)
      File "/.../spark/python/pyspark/sql/connect/client.py", line 573, in _execute_and_fetch
        self._handle_error(rpc_error)
      File "/.../spark/python/pyspark/sql/connect/client.py", line 627, in _handle_error
        raise SparkConnectException(str(rpc_error)) from None
    pyspark.sql.connect.client.SparkConnectException: <_MultiThreadedRendezvous of RPC that terminated with:
    	status = StatusCode.UNKNOWN
    	details = "Expression with ID: 0 is not supported"
    	debug_error_string = "UNKNOWN:Error received from peer ipv6:%5B::1%5D:15002 {created_time:"2022-12-29T21:25:46.71644+09:00", grpc_status:2, grpc_message:"Expression with ID: 0 is not supported"}"
{code}