Repository: ibis-project/ibis

Issue:
test: fix pyspark ddl tests inherited from the old spark backend
The PySpark backend used to subclass the legacy Spark backend. Including the DDL functionality. When the old Spark backend was deleted in #2869, the functionality that PySpark used to inherit, was moved to the PySpark test. In the case of DDL, when the tests were moved from the Spark backend to the PySpark one, looks like some of the functionality inherited by the PySpark backend was not working. Since the DDL functionality was tested only for the Spark backend, it wasn't known until tests were moved that some things were broken in the PySpark backend.

The tests that are failing are (all them in `ibis/backends/pyspark/tests/test_ddl.py`):
- `test_truncate_table`
- `test_truncate_table_expression`
- `test_insert_table`
- `test_create_table_reserved_identifier`
- `test_create_table_or_temp_view_from_csv`

All them have been xfailed for now, but should be eventually fixed (or the functionality should be removed if it doesn't make sense to have it in the PySpark backend).


Discussion:

Resolving patch:
diff --git a/ibis/backends/pyspark/compiler.py b/ibis/backends/pyspark/compiler.py
--- a/ibis/backends/pyspark/compiler.py
+++ b/ibis/backends/pyspark/compiler.py
@@ -448,6 +448,10 @@ def compile_endswith(t, expr, scope, timecontext, **kwargs):
     return col.startswith(end)
 
 
+def _is_table(table):
+    return isinstance(table.op().arg, ir.TableExpr)
+
+
 def compile_aggregator(
     t, expr, scope, timecontext, *, fn, context=None, **kwargs
 ):
@@ -467,6 +471,8 @@ def compile_aggregator(
         # Here we get the root table df of that column and compile
         # the expr to:
         # df.select(max(some_col))
+        if _is_table(expr):
+            return src_col.select(col)
         return t.translate(
             expr.op().arg.op().table, scope, timecontext
         ).select(col)
@@ -548,10 +554,18 @@ def fn(col):
         )
 
 
+def _count_star(_):
+    return F.count(F.lit(1))
+
+
 @compiles(ops.Count)
 def compile_count(t, expr, scope, timecontext, context=None, **kwargs):
+    if _is_table(expr):
+        fn = _count_star
+    else:
+        fn = F.count
     return compile_aggregator(
-        t, expr, scope, timecontext, fn=F.count, context=context, **kwargs
+        t, expr, scope, timecontext, fn=fn, context=context, **kwargs
     )
 
 


Test patch:
diff --git a/ibis/backends/pyspark/tests/conftest.py b/ibis/backends/pyspark/tests/conftest.py
--- a/ibis/backends/pyspark/tests/conftest.py
+++ b/ibis/backends/pyspark/tests/conftest.py
@@ -280,9 +280,9 @@ def _random_identifier(suffix):
 
 @pytest.fixture(scope='session', autouse=True)
 def test_data_db(client):
+    name = os.environ.get('IBIS_TEST_DATA_DB', 'ibis_testing')
+    client.create_database(name)
     try:
-        name = os.environ.get('IBIS_TEST_DATA_DB', 'ibis_testing')
-        client.create_database(name)
         client.set_database(name)
         yield name
     finally:
diff --git a/ibis/backends/pyspark/tests/test_ddl.py b/ibis/backends/pyspark/tests/test_ddl.py
--- a/ibis/backends/pyspark/tests/test_ddl.py
+++ b/ibis/backends/pyspark/tests/test_ddl.py
@@ -55,10 +55,6 @@ def test_drop_table_not_exist(client):
     client.drop_table(non_existent_table, force=True)
 
 
-@pytest.mark.xfail(
-    reason='Moved from the legacy spark backend, '
-    'but not working in the pyspark one #2879'
-)
 def test_truncate_table(client, alltypes, temp_table):
     expr = alltypes.limit(1)
 
@@ -72,10 +68,6 @@ def test_truncate_table(client, alltypes, temp_table):
     assert not nrows
 
 
-@pytest.mark.xfail(
-    reason='Moved from the legacy spark backend, '
-    'but not working in the pyspark one #2879'
-)
 def test_truncate_table_expression(client, alltypes, temp_table):
     expr = alltypes.limit(1)
 
@@ -113,10 +105,6 @@ def test_create_empty_table(client, temp_table):
     assert client.table(table_name).execute().empty
 
 
-@pytest.mark.xfail(
-    reason='Moved from the legacy spark backend, '
-    'but not working in the pyspark one #2879'
-)
 def test_insert_table(client, alltypes, temp_table, test_data_db):
     expr = alltypes
     table_name = temp_table
@@ -240,10 +228,6 @@ def test_change_properties(client, table):
         assert value == props[key]
 
 
-@pytest.mark.xfail(
-    reason='Moved from the legacy spark backend, '
-    'but not working in the pyspark one #2879'
-)
 def test_create_table_reserved_identifier(client, alltypes):
     table_name = 'distinct'
     expr = alltypes
@@ -281,10 +265,6 @@ def test_schema_from_csv(client, awards_players_filename):
     assert schema.equals(awards_players_schema)
 
 
-@pytest.mark.xfail(
-    reason='Moved from the legacy spark backend, '
-    'but not working in the pyspark one #2879'
-)
 def test_create_table_or_temp_view_from_csv(client, awards_players_filename):
     client._create_table_or_temp_view_from_csv(
         'awards', awards_players_filename