Repository: ibis-project/ibis

Issue:
Add CI steps to check that formatting/linting is correct
We have the pre-commit hook, but we should go further and add a step to the CI pipeline that fails if black would try to make formatting changes to a proposed PR.


Discussion:
it needs to use the same black version/rev used by pre-commit hook
Do you know the exact reason we pin to such a specific version of black in the pre-commit hook? We'll need to change the requirements.txt to match that specific version if we want to implement this.
Because black was incorrectly adding a trailing comma to `**kwargs` in function signatures, which isn't valid syntax in Python 3.5. It is however valid syntax in 3.5 to add a trailing comma to `**kwargs` at the call site. 
ah, right, i remember now
i think the best place to put this is in linux.yml around here ```- bash: docker-compose run ibis flake8
      displayName: 'Lint'```

I don't quite understand how the `docker-compose run` thing works though

Resolving patch:
diff --git a/ibis/pyspark/compiler.py b/ibis/pyspark/compiler.py
--- a/ibis/pyspark/compiler.py
+++ b/ibis/pyspark/compiler.py
@@ -4,8 +4,8 @@
 import operator
 
 import pyspark.sql.functions as F
-from pyspark.sql.functions import pandas_udf, PandasUDFType
 from pyspark.sql import Window
+from pyspark.sql.functions import PandasUDFType, pandas_udf
 
 import ibis.common.exceptions as com
 import ibis.expr.datatypes as dtypes
@@ -16,7 +16,7 @@
 from ibis.spark.compiler import SparkContext, SparkDialect
 from ibis.spark.datatypes import (
     ibis_array_dtype_to_spark_dtype,
-    ibis_dtype_to_spark_dtype
+    ibis_dtype_to_spark_dtype,
 )
 
 
@@ -1118,7 +1118,7 @@ def compile_cumulative_max(t, expr, scope, *, window, **kwargs):
     'D': 'day',
     'h': 'hour',
     'm': 'minute',
-    's': 'second'
+    's': 'second',
 }
 
 
@@ -1292,9 +1292,9 @@ def _get_interval_col(t, interval_ibis_expr, scope, allowed_units=None):
             'Interval unit "{}" is not allowed. Allowed units are: '
             '{}'.format(dtype.unit, allowed_units)
         )
-    return F.expr('INTERVAL {} {}'.format(
-        op.value, _time_unit_mapping[dtype.unit]
-    ))
+    return F.expr(
+        'INTERVAL {} {}'.format(op.value, _time_unit_mapping[dtype.unit])
+    )
 
 
 def _compile_datetime_binop(t, expr, scope, fn, allowed_units, **kwargs):
@@ -1310,8 +1310,12 @@ def _compile_datetime_binop(t, expr, scope, fn, allowed_units, **kwargs):
 def compile_date_add(t, expr, scope, **kwargs):
     allowed_units = ['Y', 'W', 'M', 'D']
     return _compile_datetime_binop(
-        t, expr, scope, (lambda l, r: (l + r).cast('timestamp')),
-        allowed_units, **kwargs
+        t,
+        expr,
+        scope,
+        (lambda l, r: (l + r).cast('timestamp')),
+        allowed_units,
+        **kwargs,
     )
 
 
@@ -1319,8 +1323,12 @@ def compile_date_add(t, expr, scope, **kwargs):
 def compile_date_sub(t, expr, scope, **kwargs):
     allowed_units = ['Y', 'W', 'M', 'D']
     return _compile_datetime_binop(
-        t, expr, scope, (lambda l, r: (l - r).cast('timestamp')),
-        allowed_units, **kwargs
+        t,
+        expr,
+        scope,
+        (lambda l, r: (l - r).cast('timestamp')),
+        allowed_units,
+        **kwargs,
     )
 
 
@@ -1336,8 +1344,12 @@ def compile_date_diff(t, expr, scope, **kwargs):
 def compile_timestamp_add(t, expr, scope, **kwargs):
     allowed_units = ['Y', 'W', 'M', 'D', 'h', 'm', 's']
     return _compile_datetime_binop(
-        t, expr, scope, (lambda l, r: (l + r).cast('timestamp')),
-        allowed_units, **kwargs
+        t,
+        expr,
+        scope,
+        (lambda l, r: (l + r).cast('timestamp')),
+        allowed_units,
+        **kwargs,
     )
 
 
@@ -1345,8 +1357,12 @@ def compile_timestamp_add(t, expr, scope, **kwargs):
 def compile_timestamp_sub(t, expr, scope, **kwargs):
     allowed_units = ['Y', 'W', 'M', 'D', 'h', 'm', 's']
     return _compile_datetime_binop(
-        t, expr, scope, (lambda l, r: (l - r).cast('timestamp')),
-        allowed_units, **kwargs
+        t,
+        expr,
+        scope,
+        (lambda l, r: (l - r).cast('timestamp')),
+        allowed_units,
+        **kwargs,
     )
 
 


Test patch:
diff --git a/ibis/pyspark/tests/conftest.py b/ibis/pyspark/tests/conftest.py
--- a/ibis/pyspark/tests/conftest.py
+++ b/ibis/pyspark/tests/conftest.py
@@ -16,8 +16,7 @@ def client():
     df.createTempView('basic_table')
 
     df_dates = client._session.createDataFrame(
-        [['2018-01-02'], ['2018-01-03'], ['2018-01-04']],
-        ['date_str']
+        [['2018-01-02'], ['2018-01-03'], ['2018-01-04']], ['date_str']
     )
     df_dates.createTempView('date_table')
 
@@ -27,9 +26,9 @@ def client():
             ['k2', [4, 5], ['test1', 'test2', 'test3']],
             ['k3', [6], ['w', 'x', 'y', 'z']],
             ['k1', [], ['cat', 'dog']],
-            ['k1', [7, 8], []]
+            ['k1', [7, 8], []],
         ],
-        ['key', 'array_int', 'array_str']
+        ['key', 'array_int', 'array_str'],
     )
     df_arrays.createTempView('array_table')
     return client
diff --git a/ibis/pyspark/tests/test_array.py b/ibis/pyspark/tests/test_array.py
--- a/ibis/pyspark/tests/test_array.py
+++ b/ibis/pyspark/tests/test_array.py
@@ -15,8 +15,9 @@ def test_array_length(client):
     result = table.mutate(length=table.array_int.length()).compile()
 
     expected = table.compile().toPandas()
-    expected['length'] = expected['array_int'].map(
-        lambda a: len(a)).astype('int32')
+    expected['length'] = (
+        expected['array_int'].map(lambda a: len(a)).astype('int32')
+    )
     tm.assert_frame_equal(result.toPandas(), expected)
 
 
@@ -153,8 +154,7 @@ def test_array_concat(client, op):
 
     df = table.compile().toPandas()
     expected = op(
-        df.array_int.apply(lambda x: list(map(str, x))),
-        df.array_str,
+        df.array_int.apply(lambda x: list(map(str, x))), df.array_str
     ).rename('tmp')
     tm.assert_series_equal(result, expected)
 
@@ -204,8 +204,8 @@ def test_array_collect(client):
     df = table.compile().toPandas()
     expected = (
         df.groupby('key')
-          .array_int.apply(list)
-          .reset_index()
-          .rename(columns={'array_int': 'collected'})
+        .array_int.apply(list)
+        .reset_index()
+        .rename(columns={'array_int': 'collected'})
     )
     tm.assert_frame_equal(result, expected)
diff --git a/ibis/pyspark/tests/test_basic.py b/ibis/pyspark/tests/test_basic.py
--- a/ibis/pyspark/tests/test_basic.py
+++ b/ibis/pyspark/tests/test_basic.py
@@ -178,19 +178,19 @@ def test_cast(client):
     'fn',
     [
         param(lambda t: t.date_str.to_timestamp('yyyy-MM-dd')),
-        param(lambda t: t.date_str.to_timestamp('yyyy-MM-dd', timezone='UTC'))
+        param(lambda t: t.date_str.to_timestamp('yyyy-MM-dd', timezone='UTC')),
     ],
 )
 def test_string_to_timestamp(client, fn):
     import pyspark.sql.functions as F
+
     table = client.table('date_table')
 
     result = table.mutate(date=fn(table)).compile()
 
     df = table.compile()
     expected = df.withColumn(
-        'date',
-        F.to_date(df.date_str, 'yyyy-MM-dd').alias('date')
+        'date', F.to_date(df.date_str, 'yyyy-MM-dd').alias('date')
     )
     expected_pdf = expected.toPandas()
     expected_pdf['date'] = pd.to_datetime(expected_pdf['date'])
diff --git a/ibis/tests/all/test_temporal.py b/ibis/tests/all/test_temporal.py
--- a/ibis/tests/all/test_temporal.py
+++ b/ibis/tests/all/test_temporal.py
@@ -204,12 +204,10 @@ def test_integer_to_interval_date_failure(backend, con, alltypes, df, unit):
             id='timestamp-add-interval',
         ),
         param(
-            lambda t, be: t.timestamp_col + (
-                ibis.interval(days=4) - ibis.interval(days=2)
-            ),
-            lambda t, be: t.timestamp_col + (
-                pd.Timedelta(days=4) - pd.Timedelta(days=2)
-            ),
+            lambda t, be: t.timestamp_col
+            + (ibis.interval(days=4) - ibis.interval(days=2)),
+            lambda t, be: t.timestamp_col
+            + (pd.Timedelta(days=4) - pd.Timedelta(days=2)),
             id='timestamp-add-interval-binop',
         ),
         param(