Repository: ibis-project/ibis

Issue:
bug: Ibis doesn't work with pySpark 3.2
### What happened?

This probably doesn't strictly qualify as a bug, but Ibis doesn't seem to work with pySpark version 3.2, and I don't see a reason why. I checked Ibis dependencies and I see that Ibis assumes version >3, but nothing more specific.

My problem is that at work our Cloudera cluster runs Spark version 3.2, and pySpark version must match it. When I run `pip install ibis-framework[pyspark]` Ibis installs pySpark version 3.4.

```
import ibis
from pyspark.sql import SparkSession
session = SparkSession.builder.getOrCreate()
```

This throws Error 1 (below).

If I manually downgrade pySpark to 3.2.0 I get Error 2, i.e. a Spark session can now be created, but Ibis complains about some missing attribute.


### What version of ibis are you using?

6.2.0

### What backend(s) are you using, if any?

pyspark

### Relevant log output

Error 1:

```python
Py4JError                                 Traceback (most recent call last)
Cell In[1], line 1
----> 1 session = SparkSession.builder.getOrCreate()

File ~/.local/lib/python3.10/site-packages/pyspark/sql/session.py:480, in SparkSession.Builder.getOrCreate(self)
    477     sc = SparkContext.getOrCreate(sparkConf)
    478     # Do not update `SparkConf` for existing `SparkContext`, as it's shared
    479     # by all sessions.
--> 480     session = SparkSession(sc, options=self._options)
    481 else:
    482     getattr(
    483         getattr(session._jvm, "SparkSession$"), "MODULE$"
    484     ).applyModifiableSettings(session._jsparkSession, self._options)

File ~/.local/lib/python3.10/site-packages/pyspark/sql/session.py:525, in SparkSession.__init__(self, sparkContext, jsparkSession, options)
    521         getattr(getattr(self._jvm, "SparkSession$"), "MODULE$").applyModifiableSettings(
    522             jsparkSession, options
    523         )
    524     else:
--> 525         jsparkSession = self._jvm.SparkSession(self._jsc.sc(), options)
    526 else:
    527     getattr(getattr(self._jvm, "SparkSession$"), "MODULE$").applyModifiableSettings(
    528         jsparkSession, options
    529     )

File ~/.local/lib/python3.10/site-packages/py4j/java_gateway.py:1587, in JavaClass.__call__(self, *args)
   1581 command = proto.CONSTRUCTOR_COMMAND_NAME +\
   1582     self._command_header +\
   1583     args_command +\
   1584     proto.END_COMMAND_PART
   1586 answer = self._gateway_client.send_command(command)
-> 1587 return_value = get_return_value(
   1588     answer, self._gateway_client, None, self._fqn)
   1590 for temp_arg in temp_args:
   1591     if hasattr(temp_arg, "_detach"):

File ~/.local/lib/python3.10/site-packages/py4j/protocol.py:330, in get_return_value(answer, gateway_client, target_id, name)
    326         raise Py4JJavaError(
    327             "An error occurred while calling {0}{1}{2}.\n".
    328             format(target_id, ".", name), value)
    329     else:
--> 330         raise Py4JError(
    331             "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n".
    332             format(target_id, ".", name, value))
    333 else:
    334     raise Py4JError(
    335         "An error occurred while calling {0}{1}{2}".
    336         format(target_id, ".", name))

Py4JError: An error occurred while calling None.org.apache.spark.sql.SparkSession. Trace:
py4j.Py4JException: Constructor org.apache.spark.sql.SparkSession([class org.apache.spark.SparkContext, class java.util.HashMap]) does not exist
	at py4j.reflection.ReflectionEngine.getConstructor(ReflectionEngine.java:179)
	at py4j.reflection.ReflectionEngine.getConstructor(ReflectionEngine.java:196)
	at py4j.Gateway.invoke(Gateway.java:237)
	at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)
	at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)
	at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
	at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
	at java.lang.Thread.run(Thread.java:750)
```



Error 2:
```python
import ibis
from pyspark.sql import SparkSession
session = SparkSession.builder.getOrCreate()
Setting spark.hadoop.yarn.resourcemanager.principal to xxxxxx
Setting spark.hadoop.yarn.resourcemanager.principal to xxxxxx
con = ibis.pyspark.connect(session=session)
AttributeError: module 'pyspark.sql.types' has no attribute 'DayTimeIntervalType'
AttributeError                            Traceback (most recent call last)
Cell In[1], line 1
----> 1 con = ibis.pyspark.connect(session=session)

File ~/.local/lib/python3.10/site-packages/ibis/__init__.py:76, in __getattr__(name)
     74 (entry_point,) = entry_points
     75 try:
---> 76     module = entry_point.load()
     77 except ImportError as exc:
     78     raise ImportError(
     79         f"Failed to import the {name} backend due to missing dependencies.\n\n"
     80         f"You can pip or conda install the {name} backend as follows:\n\n"
     81         f'  python -m pip install -U "ibis-framework[{name}]"  # pip install\n'
     82         f"  conda install -c conda-forge ibis-{name}           # or conda install"
     83     ) from exc

File /usr/local/lib/python3.10/importlib/metadata/__init__.py:171, in EntryPoint.load(self)
    166 """Load the entry point from its definition. If only a module
    167 is indicated by the value, return that module. Otherwise,
    168 return the named object.
    169 """
    170 match = self.pattern.match(self.value)
--> 171 module = import_module(match.group('module'))
    172 attrs = filter(None, (match.group('attr') or '').split('.'))
    173 return functools.reduce(getattr, attrs, module)

File /usr/local/lib/python3.10/importlib/__init__.py:126, in import_module(name, package)
    124             break
    125         level += 1
--> 126 return _bootstrap._gcd_import(name[level:], package, level)

File <frozen importlib._bootstrap>:1050, in _gcd_import(name, package, level)

File <frozen importlib._bootstrap>:1027, in _find_and_load(name, import_)

File <frozen importlib._bootstrap>:1006, in _find_and_load_unlocked(name, import_)

File <frozen importlib._bootstrap>:688, in _load_unlocked(spec)

File <frozen importlib._bootstrap_external>:883, in exec_module(self, module)

File <frozen importlib._bootstrap>:241, in _call_with_frames_removed(f, *args, **kwds)

File ~/.local/lib/python3.10/site-packages/ibis/backends/pyspark/__init__.py:30
     23 from ibis.backends.base.sql.compiler import Compiler, TableSetFormatter
     24 from ibis.backends.base.sql.ddl import (
     25     CreateDatabase,
     26     DropTable,
     27     TruncateTable,
     28     is_fully_qualified,
     29 )
---> 30 from ibis.backends.pyspark import ddl
     31 from ibis.backends.pyspark.client import PySparkTable
     32 from ibis.backends.pyspark.compiler import PySparkExprTranslator

File ~/.local/lib/python3.10/site-packages/ibis/backends/pyspark/ddl.py:13
      3 from ibis.backends.base.sql.ddl import (
      4     CTAS,
      5     AlterTable,
   (...)
     10     RenameTable,
     11 )
     12 from ibis.backends.base.sql.registry import quote_identifier
---> 13 from ibis.backends.pyspark.datatypes import type_to_sql_string
     15 _format_aliases = {'TEXTFILE': 'TEXT'}
     18 def _sanitize_format(format):

File ~/.local/lib/python3.10/site-packages/ibis/backends/pyspark/datatypes.py:42
     38 _to_pyspark_dtypes = {v: k for k, v in _from_pyspark_dtypes.items()}
     39 _to_pyspark_dtypes[dt.JSON] = pt.StringType
     41 _pyspark_interval_units = {
---> 42     pt.DayTimeIntervalType.SECOND: 's',
     43     pt.DayTimeIntervalType.MINUTE: 'm',
     44     pt.DayTimeIntervalType.HOUR: 'h',
     45     pt.DayTimeIntervalType.DAY: 'D',
     46 }
     49 class PySparkType(TypeMapper):
     50     @classmethod
     51     def to_ibis(cls, typ, nullable=True):

AttributeError: module 'pyspark.sql.types' has no attribute 'DayTimeIntervalType'
```


### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
Hey @contang0 -- thanks for raising this!

Regarding error 1:
`pip` is going to try to grab the newest version of any required dependencies that aren't already installed (or aren't constrained in some other way).

You can constrain that by either:
1. Already installing a version of `pyspark` that is >= the minimum version Ibis requires, or
2. Specify it alongside the ibis install command, e.g.

```sh
pip install 'ibis-framework[pyspark]' pyspark==3.2
```


Regarding error 2:
It looks like we might not support Spark 3.2 anymore -- the `DayTimeIntervalType` is definitely part of Spark 3.2 but I can't find it in the Pyspark API -- it looks like it might've been missed out from there.
I'll check with the other maintainers to see if there's any way we can work around that.  Sorry for the trouble!
Thanks a lot @gforsyth ! Error 1 doesn't matter that much, because I cannot change the version that runs on the cluster. We're a big organisation and these kind of changes happen very slowly. I can only play with Ibis and pySpark versions on my end, so some degree of backwards compatibility with earlier pySpark versions would be very useful.

Resolving patch:
diff --git a/ibis/backends/pyspark/compiler.py b/ibis/backends/pyspark/compiler.py
--- a/ibis/backends/pyspark/compiler.py
+++ b/ibis/backends/pyspark/compiler.py
@@ -9,6 +9,7 @@
 import pyspark
 import pyspark.sql.functions as F
 import pyspark.sql.types as pt
+from packaging.version import parse as vparse
 from pyspark.sql import Window
 from pyspark.sql.functions import PandasUDFType, pandas_udf
 
@@ -1557,14 +1558,20 @@ def compile_date_sub(t, op, **kwargs):
     )
 
 
-@compiles(ops.DateDiff)
-def compile_date_diff(t, op, **kwargs):
-    left = t.translate(op.left, **kwargs)
-    right = t.translate(op.right, **kwargs)
+if vparse(pyspark.__version__) >= vparse("3.3"):
 
-    return F.concat(F.lit("INTERVAL '"), F.datediff(left, right), F.lit("' DAY")).cast(
-        pt.DayTimeIntervalType(pt.DayTimeIntervalType.DAY, pt.DayTimeIntervalType.DAY)
-    )
+    @compiles(ops.DateDiff)
+    def compile_date_diff(t, op, **kwargs):
+        left = t.translate(op.left, **kwargs)
+        right = t.translate(op.right, **kwargs)
+
+        return F.concat(
+            F.lit("INTERVAL '"), F.datediff(left, right), F.lit("' DAY")
+        ).cast(
+            pt.DayTimeIntervalType(
+                pt.DayTimeIntervalType.DAY, pt.DayTimeIntervalType.DAY
+            )
+        )
 
 
 @compiles(ops.TimestampAdd)


Test patch:
diff --git a/ibis/backends/tests/test_temporal.py b/ibis/backends/tests/test_temporal.py
--- a/ibis/backends/tests/test_temporal.py
+++ b/ibis/backends/tests/test_temporal.py
@@ -949,6 +949,11 @@ def convert_to_offset(x):
             ),
             id="date-subtract-date",
             marks=[
+                pytest.mark.xfail_version(
+                    pyspark=["pyspark<3.3"],
+                    raises=AttributeError,
+                    reason="DayTimeIntervalType added in pyspark 3.3",
+                ),
                 pytest.mark.notimpl(["bigquery"], raises=com.OperationNotDefinedError),
                 pytest.mark.notimpl(
                     ["druid"],