Repository: ibis-project/ibis

Issue:
BUG: Black git pre-commit hook doesn't reformat the code
Using this flag https://github.com/ibis-project/ibis/blob/master/.pre-commit-config.yaml#L22 , black is not reformatting files in pre-commit hook

If I removed this flag, it works again.


Discussion:

Resolving patch:
diff --git a/ibis/common/geospatial.py b/ibis/common/geospatial.py
--- a/ibis/common/geospatial.py
+++ b/ibis/common/geospatial.py
@@ -1,5 +1,5 @@
 import sys
-from typing import Iterable
+from typing import Iterable, TypeVar
 
 import ibis.expr.types as ir
 from ibis.common import exceptions as ex
@@ -13,42 +13,75 @@
 except ImportError:
     ...
 
+NumberType = TypeVar('NumberType', int, float)
+# Geometry primitives (2D)
+PointType = Iterable[NumberType]
+LineStringType = Iterable[PointType]
+PolygonType = Iterable[LineStringType]
+# Multipart geometries (2D)
+MultiPointType = Iterable[PointType]
+MultiLineStringType = Iterable[LineStringType]
+MultiPolygonType = Iterable[PolygonType]
 
-def _format_point_value(value: Iterable) -> str:
+
+def _format_point_value(value: PointType) -> str:
     """Convert a iterable with a point to text."""
     return ' '.join(str(v) for v in value)
 
 
-def _format_linestring_value(value: Iterable) -> str:
+def _format_linestring_value(value: LineStringType, nested=False) -> str:
     """Convert a iterable with a linestring to text."""
-    return ', '.join(_format_point_value(point) for point in value)
+    template = '({})' if nested else '{}'
+    if not isinstance(value[0], (tuple, list)):
+        msg = '{} structure expected: LineStringType'.format(
+            'Data' if not nested else 'Inner data'
+        )
+        raise ex.IbisInputError(msg)
+    return template.format(
+        ', '.join(_format_point_value(point) for point in value)
+    )
 
 
-def _format_polygon_value(value: Iterable) -> str:
+def _format_polygon_value(value: PolygonType, nested=False) -> str:
     """Convert a iterable with a polygon to text."""
-    return ', '.join(
-        '({})'.format(_format_linestring_value(line)) for line in value
+    template = '({})' if nested else '{}'
+    if not isinstance(value[0][0], (tuple, list)):
+        msg = '{} data structure expected: PolygonType'.format(
+            'Data' if not nested else 'Inner data'
+        )
+        raise ex.IbisInputError(msg)
+
+    return template.format(
+        ', '.join(
+            _format_linestring_value(line, nested=True) for line in value
+        )
     )
 
 
-def _format_multilinestring_value(value: Iterable) -> str:
-    """Convert a iterable with a multilinestring to text."""
+def _format_multipoint_value(value: MultiPointType) -> str:
+    """Convert a iterable with a multipoint to text."""
+    if not isinstance(value[0], (tuple, list)):
+        raise ex.IbisInputError('Data structure expected: MultiPointType')
     return ', '.join(
-        '({})'.format(_format_linestring_value(line)) for line in value
+        '({})'.format(_format_point_value(point)) for point in value
     )
 
 
-def _format_multipoint_value(value: Iterable) -> str:
-    """Convert a iterable with a multipoint to text."""
+def _format_multilinestring_value(value: MultiLineStringType) -> str:
+    """Convert a iterable with a multilinestring to text."""
+    if not isinstance(value[0][0], (tuple, list)):
+        raise ex.IbisInputError('Data structure expected: MultiLineStringType')
     return ', '.join(
-        '({})'.format(_format_point_value(point)) for point in value
+        '({})'.format(_format_linestring_value(line)) for line in value
     )
 
 
-def _format_multipolygon_value(value: Iterable) -> str:
+def _format_multipolygon_value(value: MultiPolygonType) -> str:
     """Convert a iterable with a multipolygon to text."""
+    if not isinstance(value[0][0], (tuple, list)):
+        raise ex.IbisInputError('Data structure expected: MultiPolygonType')
     return ', '.join(
-        '({})'.format(_format_polygon_value(polygon)) for polygon in value
+        _format_polygon_value(polygon, nested=True) for polygon in value
     )
 
 
@@ -120,9 +153,9 @@ def translate_literal(expr, inline_metadata: bool = False) -> str:
         result = translate_linestring(value)
     elif isinstance(expr, ir.PolygonScalar):
         result = translate_polygon(value)
-    elif isinstance(expr, ir.MultiLineString):
+    elif isinstance(expr, ir.MultiLineStringScalar):
         result = translate_multilinestring(value)
-    elif isinstance(expr, ir.MultiPoint):
+    elif isinstance(expr, ir.MultiPointScalar):
         result = translate_multipoint(value)
     elif isinstance(expr, ir.MultiPolygonScalar):
         result = translate_multipolygon(value)
diff --git a/ibis/sql/alchemy.py b/ibis/sql/alchemy.py
--- a/ibis/sql/alchemy.py
+++ b/ibis/sql/alchemy.py
@@ -85,9 +85,7 @@ def _to_sqla_type(itype, type_map=None):
         elif itype.geotype == 'geography':
             return ga.Geography
         else:
-            raise TypeError(
-                'Unexpected geospatial geotype {}'.format(itype.geotype)
-            )
+            return ga.types._GISType
     else:
         return type_map[type(itype)]
 
@@ -140,7 +138,7 @@ def sa_double(_, satype, nullable=True):
 
 if geospatial_supported:
 
-    @dt.dtype.register(SQLAlchemyDialect, ga.Geometry)
+    @dt.dtype.register(SQLAlchemyDialect, (ga.Geometry, ga.types._GISType))
     def ga_geometry(_, gatype, nullable=True):
         t = gatype.geometry_type
         if t == 'POINT':
@@ -155,6 +153,8 @@ def ga_geometry(_, gatype, nullable=True):
             return dt.MultiPoint(nullable=nullable)
         if t == 'MULTIPOLYGON':
             return dt.MultiPolygon(nullable=nullable)
+        if t == 'GEOMETRY':
+            return dt.Geometry(nullable=nullable)
         else:
             raise ValueError("Unrecognized geometry type: {}".format(t))
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -15,13 +15,13 @@
 
 VERSION = sys.version_info.major, sys.version_info.minor
 
-impala_requires = ['hdfs>=2.0.16', 'sqlalchemy>=1.1', 'requests']
+impala_requires = ['hdfs>=2.0.16', 'sqlalchemy>=1.1,<1.3.7', 'requests']
 if VERSION == (3, 5):
     impala_requires.append('impyla<0.14.2')
 else:
     impala_requires.append('impyla>=0.15.0')
 
-sqlite_requires = ['sqlalchemy>=1.1']
+sqlite_requires = ['sqlalchemy>=1.1,<1.3.7']
 postgres_requires = sqlite_requires + ['psycopg2']
 mysql_requires = sqlite_requires + ['pymysql']
 
@@ -99,7 +99,7 @@
         'bigquery': bigquery_requires,
         'hdf5': hdf5_requires,
         'parquet': parquet_requires,
-        'spark' : spark_requires,
+        'spark': spark_requires,
         'geospatial': geospatial_requires,
     },
     description="Productivity-centric Python Big Data Framework",


Test patch:
diff --git a/ibis/expr/tests/conftest.py b/ibis/expr/tests/conftest.py
--- a/ibis/expr/tests/conftest.py
+++ b/ibis/expr/tests/conftest.py
@@ -17,7 +17,8 @@
 import pytest
 
 import ibis
-from ibis.expr.tests.mocks import GeoMockConnection, MockConnection
+import ibis.common.exceptions as com
+from ibis.expr.tests.mocks import MockConnection
 
 
 @pytest.fixture
@@ -97,11 +98,22 @@ def lineitem(con):
     return con.table('tpch_lineitem')
 
 
-@pytest.fixture
-def geo_con():
-    return GeoMockConnection()
-
-
-@pytest.fixture
-def geo_table(geo_con):
-    return geo_con.table('geo')
+@pytest.hookimpl(hookwrapper=True)
+def pytest_pyfunc_call(pyfuncitem):
+    """Dynamically add an xfail marker for specific backends."""
+    outcome = yield
+    try:
+        outcome.get_result()
+    except (
+        com.OperationNotDefinedError,
+        com.UnsupportedOperationError,
+        com.UnsupportedBackendType,
+        NotImplementedError,
+    ) as e:
+        markers = list(pyfuncitem.iter_markers(name="xfail_unsupported"))
+        assert (
+            len(markers) == 1
+        ), "More than one xfail_unsupported marker found on test {}".format(
+            pyfuncitem
+        )
+        pytest.xfail(reason=repr(e))
diff --git a/ibis/expr/tests/mocks.py b/ibis/expr/tests/mocks.py
--- a/ibis/expr/tests/mocks.py
+++ b/ibis/expr/tests/mocks.py
@@ -420,42 +420,51 @@ def _build_ast(self, expr, context):
         return build_ast(expr, context)
 
 
-class GeoMockConnection(SQLClient):
+GEO_TABLE = {
+    'geo': [
+        ('id', 'int32'),
+        ('geo_point', 'point'),
+        ('geo_linestring', 'linestring'),
+        ('geo_polygon', 'polygon'),
+        ('geo_multipolygon', 'multipolygon'),
+    ]
+}
+
+
+class GeoMockConnectionPostGIS(MockAlchemyConnection):
+    _tables = GEO_TABLE
+
+    def __init__(self):
+        super().__init__()
+        self.executed_queries = []
+
+    def _get_table_schema(self, name):
+        return Schema.from_tuples(self._tables[name])
+
     @property
     def dialect(self):
-        from ibis.omniscidb.compiler import OmniSciDBDialect
+        from ibis.sql.postgres.compiler import PostgreSQLDialect
 
-        return OmniSciDBDialect
+        return PostgreSQLDialect
 
-    _tables = {
-        'geo': [
-            ('id', 'int32'),
-            ('geo_point', 'point'),
-            ('geo_linestring', 'linestring'),
-            ('geo_polygon', 'polygon'),
-            ('geo_multipolygon', 'multipolygon'),
-        ]
-    }
+
+class GeoMockConnectionOmniSciDB(SQLClient):
+    _tables = GEO_TABLE
 
     def __init__(self):
+        super().__init__()
         self.executed_queries = []
 
     def _get_table_schema(self, name):
-        # name = name.replace('`', '')
         return Schema.from_tuples(self._tables[name])
 
+    @property
+    def dialect(self):
+        from ibis.omniscidb.compiler import OmniSciDBDialect
+
+        return OmniSciDBDialect
+
     def _build_ast(self, expr, context):
         from ibis.omniscidb.compiler import build_ast
 
         return build_ast(expr, context)
-
-    def execute(self, expr, limit=None, params=None):
-        ast = self._build_ast_ensure_limit(expr, limit, params=params)
-        for query in ast.queries:
-            self.executed_queries.append(query.compile())
-        return None
-
-    def compile(self, expr, limit=None, params=None):
-        ast = self._build_ast_ensure_limit(expr, limit, params=params)
-        queries = [q.compile() for q in ast.queries]
-        return queries[0] if len(queries) == 1 else queries
diff --git a/ibis/expr/tests/test_geospatial.py b/ibis/expr/tests/test_geospatial.py
--- a/ibis/expr/tests/test_geospatial.py
+++ b/ibis/expr/tests/test_geospatial.py
@@ -1,78 +1,183 @@
 """ Tests for geo spatial data types"""
+import sys
+
 import pytest
 
 import ibis
+from ibis.expr.tests.mocks import (
+    GeoMockConnectionOmniSciDB,
+    GeoMockConnectionPostGIS,
+)
 
-
-@pytest.mark.parametrize(
-    'modifier', ['', ';4326', ';4326:geometry', ';4326:geography']
+pytestmark = pytest.mark.skipif(
+    sys.version_info < (3, 6),
+    reason='Geo Spatial support available just for Python >= 3.6',
 )
-def test_geo_literals_smoke(modifier):
-    """Smoke tests for geo spatial literals"""
-    point = (0, 1)
-    ibis.literal(point, type='point{}'.format(modifier))
 
-    linestring = [(0, 1), (2, 3)]
-    ibis.literal(linestring, type='linestring{}'.format(modifier))
+pytest.importorskip('geoalchemy2')
+pytest.importorskip('shapely')
+pytest.importorskip('geopandas')
 
-    polygon = [
-        ((0, 0), (4, 0), (4, 4), (0, 4), (0, 0)),
-        ((1, 1), (2, 1), (2, 2), (1, 2), (1, 1)),
-    ]
-    ibis.literal(polygon, type='polygon{}'.format(modifier))
+pytest.mark.postgis
+pytest.mark.omniscidb
 
-    point1 = (0, 1)
-    point2 = (2, 3)
-    multipoint = [point1, point2]
-    ibis.literal(multipoint, type='multipoint{}'.format(modifier))
+mock_omniscidb = GeoMockConnectionOmniSciDB()
+mock_postgis = GeoMockConnectionPostGIS()
 
-    linestring1 = [(0, 1), (2, 3)]
-    linestring2 = [(4, 5), (6, 7)]
-    multilinestring = [linestring1, linestring2]
-    ibis.literal(multilinestring, type='multilinestring{}'.format(modifier))
 
-    polygon1 = (
-        (0, 0), (1, 0), (0.5, 1), (0, 0)
+@pytest.mark.parametrize('backend', [mock_postgis])
+@pytest.mark.parametrize(
+    'modifier',
+    [
+        {},
+        {'srid': '4326'},
+        {'srid': '4326', 'geo_type': 'geometry'},
+        {'srid': '4326', 'geo_type': 'geography'},
+    ],
+)
+@pytest.mark.parametrize(
+    'shape,value,expected',
+    [
+        # Geometry primitives (2D)
+        ('point', (30, 10), 'POINT (30 10)'),
+        (
+            'linestring',
+            ((30, 10), (10, 30), (40, 40)),
+            'LINESTRING (30 10, 10 30, 40 40)',
+        ),
+        (
+            'polygon',
+            (
+                ((35, 10), (45, 45), (15, 40), (10, 20), (35, 10)),
+                ((20, 30), (35, 35), (30, 20), (20, 30)),
+            ),
+            (
+                'POLYGON ((35 10, 45 45, 15 40, 10 20, 35 10), '
+                + '(20 30, 35 35, 30 20, 20 30))'
+            ),
+        ),
+        (
+            'polygon',
+            (((30, 10), (40, 40), (20, 40), (10, 20), (30, 10)),),
+            'POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))',
+        ),
+        # Multipart geometries (2D)
+        (
+            'multipoint',
+            ((10, 40), (40, 30), (20, 20), (30, 10)),
+            'MULTIPOINT ((10 40), (40 30), (20 20), (30 10))',
+        ),
+        (
+            'multilinestring',
+            (
+                ((10, 10), (20, 20), (10, 40)),
+                ((40, 40), (30, 30), (40, 20), (30, 10)),
+            ),
+            (
+                'MULTILINESTRING ((10 10, 20 20, 10 40), '
+                + '(40 40, 30 30, 40 20, 30 10))'
+            ),
+        ),
+        (
+            'multipolygon',
+            (
+                (((40, 40), (20, 45), (45, 30), (40, 40)),),
+                (
+                    (
+                        (20, 35),
+                        (10, 30),
+                        (10, 10),
+                        (30, 5),
+                        (45, 20),
+                        (20, 35),
+                    ),
+                    ((30, 20), (20, 15), (20, 25), (30, 20)),
+                ),
+            ),
+            (
+                'MULTIPOLYGON (((40 40, 20 45, 45 30, 40 40)), '
+                + '((20 35, 10 30, 10 10, 30 5, 45 20, 20 35), '
+                + '(30 20, 20 15, 20 25, 30 20)))'
+            ),
+        ),
+    ],
+)
+def test_geo_literals_smoke(backend, shape, value, modifier, expected):
+    """Smoke tests for geo spatial literals"""
+    expr_type = '{}{}{}'.format(
+        shape,
+        ';{}'.format(modifier['srid']) if 'srid' in modifier else '',
+        ':{}'.format(modifier['geo_type']) if 'geo_type' in modifier else '',
     )
-    polygon2 = (
-        ((35, 10), (45, 45), (15, 40), (10, 20), (35, 10)),
-        ((20, 30), (35, 35), (30, 20), (20, 30))
+    expr = ibis.literal(value, type=expr_type).name('tmp')
+    result_expected = "SELECT '{}{}'{}".format(
+        'SRID={};'.format(modifier['srid']) if 'srid' in modifier else '',
+        expected,
+        '::{}'.format(modifier['geo_type']) if 'geo_type' in modifier else '',
     )
-    multipolygon = [polygon1, polygon2]
-    ibis.literal(multipolygon, type='multipolygon{}'.format(modifier))
-
-
-def test_geo_ops_smoke(geo_table):
-    """Smoke tests for geo spatial operations."""
-    t = geo_table
-
-    # alias for fields
-    point = t.geo_point
-    linestring = t.geo_linestring
-    polygon = t.geo_polygon
-    multipolygon = t.geo_multipolygon
-    # TODO: the mock tests don't support multipoint and multilinestring yet,
-    # but once they do, add some more tests here.
 
-    # test ops
-    point.srid()
-    point.set_srid(4326)
-    point.x()
-    point.y()
+    assert str(backend.compile(expr)) == result_expected
 
-    linestring.contains(point)
-    linestring.end_point()
-    linestring.length()
-    linestring.max_distance(point)
-    linestring.point_n(1)
-    linestring.start_point()
-    linestring.x_max()
-    linestring.x_min()
-    linestring.y_max()
-    linestring.y_min()
 
-    polygon.area()
-    polygon.perimeter()
-
-    multipolygon.n_points()
-    multipolygon.n_rings()
+@pytest.mark.parametrize(
+    'fn_expr',
+    [
+        pytest.param(lambda t: t.geo_point.srid(), id='point_srid'),
+        pytest.param(
+            lambda t: t.geo_point.set_srid(4326), id='point_set_srid'
+        ),
+        pytest.param(lambda t: t.geo_point.x(), id='point_x'),
+        pytest.param(lambda t: t.geo_point.y(), id='point_y'),
+        pytest.param(
+            lambda t: t.geo_linestring.contains(t.geo_point),
+            id='linestring_contains',
+        ),
+        pytest.param(
+            lambda t: t.geo_linestring.end_point(), id='linestring_end_point'
+        ),
+        pytest.param(
+            lambda t: t.geo_linestring.length(), id='linestring_length'
+        ),
+        pytest.param(
+            lambda t: t.geo_linestring.max_distance(t.geo_point),
+            id='linestring_max_distance',
+        ),
+        pytest.param(
+            lambda t: t.geo_linestring.point_n(1), id='linestring_point_n'
+        ),
+        pytest.param(
+            lambda t: t.geo_linestring.start_point(),
+            id='linestring_start_point',
+        ),
+        pytest.param(
+            lambda t: t.geo_linestring.x_max(), id='linestring_x_max'
+        ),
+        pytest.param(
+            lambda t: t.geo_linestring.x_min(), id='linestring_x_min'
+        ),
+        pytest.param(
+            lambda t: t.geo_linestring.y_max(), id='linestring_y_max'
+        ),
+        pytest.param(
+            lambda t: t.geo_linestring.y_min(), id='linestring_y_min'
+        ),
+        pytest.param(lambda t: t.geo_polygon.area(), id='polygon_area'),
+        pytest.param(
+            lambda t: t.geo_polygon.perimeter(), id='polygon_perimeter'
+        ),
+        pytest.param(
+            lambda t: t.geo_multipolygon.n_points(), id='multipolygon_n_points'
+        ),
+        pytest.param(
+            lambda t: t.geo_multipolygon.n_rings(), id='multipolygon_n_rings'
+        ),
+        # TODO: the mock tests don't support multipoint and multilinestring
+        #       yet, but once they do, add some more tests here.
+    ],
+)
+@pytest.mark.parametrize('backend', [mock_omniscidb, mock_postgis])
+@pytest.mark.xfail_unsupported
+def test_geo_ops_smoke(backend, fn_expr):
+    """Smoke tests for geo spatial operations."""
+    geo_table = backend.table('geo')
+    assert fn_expr(geo_table).compile() != ''