Bug: AttributeError: 'datetime.time' object has no attribute 'timetuple'
Status: Open | Priority: Major | Resolution: 
Created: 24/May/22 14:32
Affects Version/s: 3.2.1

 
{code:java}
import pandas as pd
import datetime
pdf = pd.DataFrame({'naive': [datetime.time(11, 30, 33, 0)]})
print(pdf)
print(pdf.info())

from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
sp_df2 = spark.createDataFrame(pdf)
sp_df2.show(10)
{code}
 

throws this error:
 
{code:java}
      naive
0  11:30:33
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1 entries, 0 to 0
Data columns (total 1 columns):
 #   Column  Non-Null Count  Dtype 
---  ------  --------------  ----- 
 0   naive   1 non-null      object
dtypes: object(1)
memory usage: 136.0+ bytes
None
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
/usr/local/share/jupyter/kernels/python39/scripts/launch_ipykernel.py in <module>
     10 spark = SparkSession.builder.getOrCreate()
     11 
---> 12 sp_df2 = spark.createDataFrame(pdf)
     13 sp_df2.show(10)

/opt/ibm/spark/python/lib/pyspark.zip/pyspark/sql/session.py in createDataFrame(self, data, schema, samplingRatio, verifySchema)
    671         if has_pandas and isinstance(data, pandas.DataFrame):
    672             # Create a DataFrame from pandas DataFrame.
--> 673             return super(SparkSession, self).createDataFrame(
    674                 data, schema, samplingRatio, verifySchema)
    675         return self._create_dataframe(data, schema, samplingRatio, verifySchema)

/opt/ibm/spark/python/lib/pyspark.zip/pyspark/sql/pandas/conversion.py in createDataFrame(self, data, schema, samplingRatio, verifySchema)
    338                     raise
    339         data = self._convert_from_pandas(data, schema, timezone)
--> 340         return self._create_dataframe(data, schema, samplingRatio, verifySchema)
    341 
    342     def _convert_from_pandas(self, pdf, schema, timezone):

/opt/ibm/spark/python/lib/pyspark.zip/pyspark/sql/session.py in _create_dataframe(self, data, schema, samplingRatio, verifySchema)
    698             rdd, schema = self._createFromRDD(data.map(prepare), schema, samplingRatio)
    699         else:
--> 700             rdd, schema = self._createFromLocal(map(prepare, data), schema)
    701         jrdd = self._jvm.SerDeUtil.toJavaArray(rdd._to_java_object_rdd())
    702         jdf = self._jsparkSession.applySchemaToPythonRDD(jrdd.rdd(), schema.json())

/opt/ibm/spark/python/lib/pyspark.zip/pyspark/sql/session.py in _createFromLocal(self, data, schema)
    523 
    524         # convert python objects to sql data
--> 525         data = [schema.toInternal(row) for row in data]
    526         return self._sc.parallelize(data), schema
    527 

/opt/ibm/spark/python/lib/pyspark.zip/pyspark/sql/session.py in <listcomp>(.0)
    523 
    524         # convert python objects to sql data
--> 525         data = [schema.toInternal(row) for row in data]
    526         return self._sc.parallelize(data), schema
    527 

/opt/ibm/spark/python/lib/pyspark.zip/pyspark/sql/types.py in toInternal(self, obj)
    624                              for n, f, c in zip(self.names, self.fields, self._needConversion))
    625             elif isinstance(obj, (tuple, list)):
--> 626                 return tuple(f.toInternal(v) if c else v
    627                              for f, v, c in zip(self.fields, obj, self._needConversion))
    628             elif hasattr(obj, "__dict__"):

/opt/ibm/spark/python/lib/pyspark.zip/pyspark/sql/types.py in <genexpr>(.0)
    624                              for n, f, c in zip(self.names, self.fields, self._needConversion))
    625             elif isinstance(obj, (tuple, list)):
--> 626                 return tuple(f.toInternal(v) if c else v
    627                              for f, v, c in zip(self.fields, obj, self._needConversion))
    628             elif hasattr(obj, "__dict__"):

/opt/ibm/spark/python/lib/pyspark.zip/pyspark/sql/types.py in toInternal(self, obj)
    449 
    450     def toInternal(self, obj):
--> 451         return self.dataType.toInternal(obj)
    452 
    453     def fromInternal(self, obj):

/opt/ibm/spark/python/lib/pyspark.zip/pyspark/sql/types.py in toInternal(self, dt)
    180         if dt is not None:
    181             seconds = (calendar.timegm(dt.utctimetuple()) if dt.tzinfo
--> 182                        else time.mktime(dt.timetuple()))
    183             return int(seconds) * 1000000 + dt.microsecond
    184 

AttributeError: 'datetime.time' object has no attribute 'timetuple'{code}