Repository: altair-viz/altair
Pull request #1664 (2019-08-18 20:33:47)

accept __geo_interface__ attribute as data
I'm not sure if I've understood you correctly here: https://github.com/altair-viz/altair/issues/588#issuecomment-522193822

But in this PR I tried to integrate serialising data objects with a `__geo_interface__` attribute.
And it worked OK for Python packages that deal with geographical data types that support the `__geo_interface__`. I tried the packages `shapely`, `pyshp`, `geojson`, `geopandas`.

It works for:
* `InlineData`
* `json` data transformer and
*  `data_server` data transformer.

See animated gif:
![ezgif com-optimize-2](https://user-images.githubusercontent.com/5186265/63229978-e6d7b780-c206-11e9-906f-a38a8813b489.gif)

After I got this working, I compared it to https://github.com/altair-viz/altair/pull/818/files, but the idea is very much the same. Maybe @iliatimofeev can shed a light on this proof of concept as well.


Patch:
diff --git a/altair/utils/core.py b/altair/utils/core.py
index c1deed47..ffe8c088 100644
--- a/altair/utils/core.py
+++ b/altair/utils/core.py
@@ -3,6 +3,7 @@ Utility routines
 """
 import collections
 from copy import deepcopy
+import json
 import itertools
 import re
 import sys
@@ -108,6 +109,55 @@ def infer_vegalite_type(data):
                       "Defaulting to nominal.".format(typ))
         return 'nominal'
 
+def merge_props_geom(feat):
+    """
+    Merge properties with geometry
+    * Overwrites 'type' and 'geometry' entries if existing    
+    """
+
+    geom = {k: feat[k] for k in ('type', 'geometry')}
+    try:
+        feat['properties'].update(geom)
+        props_geom = feat['properties']
+    except (AttributeError, KeyError):
+        # AttributeError when 'properties' equals None
+        # KeyError when 'properties' is non-existing        
+        props_geom = geom   
+
+    return props_geom   
+
+def sanitize_geo_interface(geo):
+    """Santize a geo_interface to prepare it for serialization.
+    
+    * Make a copy
+    * Convert type array or _Array to list
+    * Convert tuples to lists (using json.loads/dumps)
+    * Merge properties with geometry
+    """
+
+    geo = deepcopy(geo)
+
+    # convert type _Array or array to list
+    for key in geo.keys():        
+        if str(type(geo[key]).__name__).startswith(('_Array','array')):
+            geo[key] = geo[key].tolist()
+    
+    # convert (nested) tuples to lists
+    geo = json.loads(json.dumps(geo))
+
+    # sanitize features
+    if geo['type'] == 'FeatureCollection':
+        geo = geo['features']
+        if len(geo) > 0:          
+            for idx, feat in enumerate(geo):
+                geo[idx] = merge_props_geom(feat)
+    elif geo['type'] == 'Feature':  
+        geo = merge_props_geom(geo)
+    else:
+        geo = {'type': 'Feature', 'geometry': geo}
+
+    return geo
+
 
 def sanitize_dataframe(df):
     """Sanitize a DataFrame to prepare it for serialization.
@@ -166,6 +216,10 @@ def sanitize_dataframe(df):
                              'not supported by Altair. Please convert to '
                              'either a timestamp or a numerical value.'
                              ''.format(col_name=col_name, dtype=dtype))
+        elif str(dtype).startswith('geometry'):
+            # geopandas >=0.6.1 uses the dtype geometry. Continue here
+            # otherwise it will give an error on np.issubdtype(dtype, np.integer)
+            continue                            
         elif np.issubdtype(dtype, np.integer):
             # convert integers to objects; np.int is not JSON serializable
             df[col_name] = df[col_name].astype(object)
diff --git a/altair/utils/data.py b/altair/utils/data.py
index 9dbeb094..c0da1300 100644
--- a/altair/utils/data.py
+++ b/altair/utils/data.py
@@ -7,6 +7,7 @@ from toolz.curried import curry, pipe  # noqa
 from typing import Callable
 
 from .core import sanitize_dataframe
+from .core import sanitize_geo_interface 
 from .plugin_registry import PluginRegistry
 
 
@@ -58,7 +59,12 @@ def limit_rows(data, max_rows=5000):
     If max_rows is None, then do not perform any check.
     """
     check_data_type(data)
-    if isinstance(data, pd.DataFrame):
+    if hasattr(data, '__geo_interface__'):
+        if data.__geo_interface__['type'] == 'FeatureCollection':
+            values = data.__geo_interface__['features']
+        else:
+            values = data.__geo_interface__
+    elif isinstance(data, pd.DataFrame):
         values = data
     elif isinstance(data, dict):
         if 'values' in data:
@@ -125,7 +131,12 @@ def to_csv(data, prefix='altair-data', extension='csv',
 def to_values(data):
     """Replace a DataFrame by a data model with values."""
     check_data_type(data)
-    if isinstance(data, pd.DataFrame):
+    if hasattr(data, '__geo_interface__'):
+        if isinstance(data, pd.DataFrame):
+            data = sanitize_dataframe(data)        
+        data = sanitize_geo_interface(data.__geo_interface__)
+        return {'values': data}
+    elif isinstance(data, pd.DataFrame):
         data = sanitize_dataframe(data)
         return {'values': data.to_dict(orient='records')}
     elif isinstance(data, dict):
@@ -136,8 +147,8 @@ def to_values(data):
 
 def check_data_type(data):
     """Raise if the data is not a dict or DataFrame."""
-    if not isinstance(data, (dict, pd.DataFrame)):
-        raise TypeError('Expected dict or DataFrame, got: {}'.format(type(data)))
+    if not isinstance(data, (dict, pd.DataFrame)) and not hasattr(data, '__geo_interface__'):
+        raise TypeError('Expected dict, DataFrame or a __geo_interface__ attribute, got: {}'.format(type(data)))
 
 
 # ==============================================================================
@@ -151,7 +162,12 @@ def _compute_data_hash(data_str):
 def _data_to_json_string(data):
     """Return a JSON string representation of the input data"""
     check_data_type(data)
-    if isinstance(data, pd.DataFrame):
+    if hasattr(data, '__geo_interface__'):
+        if isinstance(data, pd.DataFrame):
+            data = sanitize_dataframe(data)        
+        data = sanitize_geo_interface(data.__geo_interface__)
+        return json.dumps(data)
+    elif isinstance(data, pd.DataFrame):
         data = sanitize_dataframe(data)
         return data.to_json(orient='records')
     elif isinstance(data, dict):
@@ -166,7 +182,10 @@ def _data_to_json_string(data):
 def _data_to_csv_string(data):
     """return a CSV string representation of the input data"""
     check_data_type(data)
-    if isinstance(data, pd.DataFrame):
+    if hasattr(data, '__geo_interface__'):    
+        raise NotImplementedError("to_csv does not work with data that "
+                                  "contains the __geo_interface__ attribute")
+    elif isinstance(data, pd.DataFrame):
         data = sanitize_dataframe(data)
         return data.to_csv(index=False)
     elif isinstance(data, dict):
diff --git a/altair/vegalite/v3/api.py b/altair/vegalite/v3/api.py
index 44663262..2a87828b 100644
--- a/altair/vegalite/v3/api.py
+++ b/altair/vegalite/v3/api.py
@@ -79,8 +79,8 @@ def _prepare_data(data, context):
     if data is Undefined:
         return data
 
-    # convert dataframes to dict
-    if isinstance(data, pd.DataFrame):
+    # convert dataframes  or objects with __geo_interface__ to dict
+    if isinstance(data, pd.DataFrame) or hasattr(data, '__geo_interface__'):
         data = pipe(data, data_transformers.get())
 
     # convert string input to a URLData
diff --git a/doc/user_guide/data.rst b/doc/user_guide/data.rst
index 33dc1e48..f5285fa3 100644
--- a/doc/user_guide/data.rst
+++ b/doc/user_guide/data.rst
@@ -8,11 +8,12 @@ Specifying Data in Altair
 Each top-level chart object (i.e. :class:`Chart`, :class:`LayerChart`,
 and :class:`VConcatChart`, :class:`HConcatChart`, :class:`RepeatChart`,
 :class:`FacetChart`) accepts a dataset as its first argument.
-The dataset can be specified in one of three ways:
+The dataset can be specified in one of the following ways:
 
 - as a `Pandas DataFrame <http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.html>`_
 - as a :class:`Data` or related object (i.e. :class:`UrlData`, :class:`InlineData`, :class:`NamedData`)
 - as a url string pointing to a ``json`` or ``csv`` formatted text file
+- as an object that supports the `__geo_interface__` (eg. `Geopandas GeoDataFrame <http://geopandas.org/data_structures.html#geodataframe>`_, `Shapely Geometries <https://shapely.readthedocs.io/en/latest/manual.html#geometric-objects>`_, `GeoJSON Objects <https://github.com/jazzband/geojson#geojson-objects>`_)
 
 For example, here we specify data via a DataFrame:
 
@@ -233,3 +234,137 @@ to be folded. The ``as_`` argument is optional, with the default being ``["key",
 .. _Pandas pivot documentation: https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.pivot.html
 .. _Pandas melt documentation: https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.melt.html#pandas.DataFrame.melt
 .. _Reshaping and Pivot Tables: https://pandas.pydata.org/pandas-docs/stable/reshaping.html
+
+.. _data-geo-interface:
+
+Geospatial Data
+~~~~~~~~~~~~~~~
+Working with geographical data in Altair is possible if the object contains a 
+`__geo_interface__` attribute. This attribute represents the geo_interface which is a 
+Python protocol for Geospatial Data. The protocol follows a GeoJSON-like structure to 
+store geo-spatial vector data.
+
+To make working with Geospatial Data as similar as working with long-form structured data 
+the geo_interface is serialized in order to:
+- make it be correctly interpreted by Altair
+- provide users a similar experience as when working with tabular data such as Pandas.
+
+Altair can interpret a spatial bounded entity (a Feature) or a list of Features 
+(FeatureCollection). In order for correct interpretation it is made sure that all records 
+contain a single geometry (one of Point, LineString, Polygon, MultiPoint, 
+MultiLineString, MultiPolygon, and GeometryCollection) and is stored as a Feature entity.
+
+The most basic Feature is an entity that only contains a Geometry object. For example
+a Polygon:
+
+.. code:: python
+
+    {
+        "type": "Feature",
+        "geometry": {
+            "coordinates": [[
+                [0, 0],
+                [0, 2],
+                [2, 2],
+                [2, 0],
+                [0, 0]
+            ]],
+            "type": "Polygon"
+        }
+    }
+
+Often, the Feature contains also additional metadata next to the Geometry object.
+The `__geo_interface__` provides two approaches to store metadata.
+- Metadata stored as a dictionary within the key `properties` (so called properties 
+member). This properties member must exist in a valid Feature entity.
+- Metada may be stored directly as foreign members on the top-level of the Feature. 
+There is no normative processing model for usage of this declaration.
+
+Altair serializes the metadata from the properties in combination with the declared
+geometry as Feature entities. The result of this approach is that the keys `type` 
+and `geometry` in the properties member will be overwriten if used.
+
+So a `__geo_interface__` that is registered as such
+
+.. code:: python
+
+    {
+        "type": "Feature",
+        "id": "f1",
+        "geometry": {...},
+        "properties": {
+            "id": 1,
+            "foo": "xx",
+            "bah": "yy",
+            "type": "zz"
+        },
+        "title": "Example Feature"
+    }
+
+Is serialized as such:
+
+.. code:: python
+
+    {
+        "type": "Feature",
+        "geometry": {...},
+        "foo": "xx",
+        "bah": "yy",
+        "id": 1
+    }
+
+The nested `"type": "zz"` in the properties member is overwriten by `"type":"Feature"`
+and only the metadata stored in the properties member is serialized. Meaning that 
+foreign members and the commonly used identifier are not serialized.
+
+.. _data-geopandas-vs-pandas:
+
+GeoPandas vs Pandas
+~~~~~~~~~~~~~~~~~~~
+A `GeoDataFrame` is a `DataFrame` including a special column with spatial geometries.
+The column-name containing the spatial geometries defaults to `geometry`. To directly
+use a `GeoDataFrame` with Altair means in practice that only the column-name `type` 
+should be avoided.
+
+.. _data-projections:
+
+Projections
+~~~~~~~~~~~
+Altair works best when the Geospatial Data adopts the World Geodetic System 1984 as 
+its geographic coordinate reference system with units in decimal degrees.
+
+Try to avoid putting projected data into Altair, but reproject your spatial data to
+EPSG:4326 first.
+
+If your data comes in a different projection (eg. with units in meters) and you don't 
+have the option to reproject the data, try using the project configuration 
+`(type: 'identity', reflectY': True)`. It draws the geometries in a cartesian grid 
+without applying a projection.
+
+.. _data-winding-order:
+
+Winding order
+~~~~~~~~~~~~~
+LineString, Polygon and MultiPolygon geometries contain coordinates in an order: lines 
+go in a certain direction, and polygon rings do too. The GeoJSON-like structure of the 
+__geo_interface__ recommends the right-hand rule winding order for Polygon and 
+MultiPolygons. Meaning that the exterior rings should be counterclockwise and interior 
+rings are clockwise. While it recommends the right-hand rule winding order, it does not 
+reject geometries that do not use the right-hand rule.
+
+Altair does NOT follow the right-hand rule for geometries, but uses the left-hand rule. 
+Meaning that exterior rings should be clockwise and interior rings should be 
+counterclockwise. 
+
+If you face a problem regarding winding order, try to force the left-hand rule on your
+data before usage in Altair using GeoPandas for example as such:
+
+.. code:: python
+
+    from shapely.ops import orient # version >=1.7a2
+    gdf = gdf.geometry.apply(orient, args=(-1,))
+
+
+.. _Protocol geo_interface: https://gist.github.com/sgillies/2217756
+.. _Packages supporting the geo_interface: https://github.com/mlaloux/Python-geo_interface-applications
+.. _The GeoJSON format: https://tools.ietf.org/html/rfc7946#section-3.1.9 


Test patch:
diff --git a/altair/vegalite/v3/tests/test_geo_interface.py b/altair/vegalite/v3/tests/test_geo_interface.py
new file mode 100644
index 00000000..b21b8cfd
--- /dev/null
+++ b/altair/vegalite/v3/tests/test_geo_interface.py
@@ -0,0 +1,221 @@
+import pytest
+import altair.vegalite.v3 as alt
+
+def geom_obj(geom):
+    class Geom(object):
+        pass
+    geom_obj = Geom()
+    setattr(geom_obj, '__geo_interface__', geom)
+    return geom_obj
+
+# correct translation of Polygon geometry to Feature type
+def test_geo_interface_polygon_feature():
+    geom = {
+        "coordinates": [[
+            (0, 0), 
+            (0, 2), 
+            (2, 2), 
+            (2, 0), 
+            (0, 0)
+        ]],
+        "type": "Polygon"
+    }
+    feat = geom_obj(geom)
+
+    with alt.data_transformers.enable(consolidate_datasets=False):
+        spec = alt.Chart(feat).mark_geoshape().to_dict()
+    assert spec['data']['values']['type'] == 'Feature'
+
+# merge geometry with empty properties dictionary
+def test_geo_interface_removal_empty_properties():
+    geom = {
+        "geometry": {
+            "coordinates": [[
+                [6.90, 53.48],
+                [5.98, 51.85],
+                [6.07, 53.51],
+                [6.90, 53.48]
+            ]], 
+            "type": "Polygon"
+        }, 
+        "id": None, 
+        "properties": {}, 
+        "type": "Feature"
+    }
+    feat = geom_obj(geom)
+
+    with alt.data_transformers.enable(consolidate_datasets=False):
+        spec = alt.Chart(feat).mark_geoshape().to_dict()
+    assert spec['data']['values']['type'] == 'Feature'
+
+# only register metadata in the properties member
+def test_geo_interface_register_foreign_member():
+    geom = {
+        "geometry": {
+            "coordinates": [[
+                [6.90, 53.48],
+                [5.98, 51.85],
+                [6.07, 53.51],
+                [6.90, 53.48]
+            ]], 
+            "type": "Polygon"
+        }, 
+        "id": 2, 
+        "properties": {"foo": "bah"}, 
+        "type": "Feature"
+    }
+    feat = geom_obj(geom)
+
+    with alt.data_transformers.enable(consolidate_datasets=False):
+        spec = alt.Chart(feat).mark_geoshape().to_dict()
+    with pytest.raises(KeyError):
+        spec['data']['values']['id']    
+    assert spec['data']['values']['foo'] == 'bah'
+
+
+# correct serializing of arrays and nested tuples
+def test_geo_interface_serializing_arrays_tuples():
+    import array as arr
+    geom = {
+        "bbox": arr.array('d', [1, 2, 3, 4]),    
+        "geometry": {
+            "coordinates": [tuple((
+                tuple((6.90, 53.48)),
+                tuple((5.98, 51.85)),
+                tuple((6.07, 53.51)),
+                tuple((6.90, 53.48))
+            ))], 
+            "type": "Polygon"
+        }, 
+        "id": 27, 
+        "properties": {}, 
+        "type": "Feature"
+    }
+    feat = geom_obj(geom)
+
+    with alt.data_transformers.enable(consolidate_datasets=False):
+        spec = alt.Chart(feat).mark_geoshape().to_dict()
+    assert spec['data']['values']['geometry']['coordinates'][0][0] == [6.9, 53.48]
+
+# overwrite existing 'type' value in properties with `Feature`
+def test_geo_interface_reserved_members():
+    geom = {
+        "geometry": {
+            "coordinates": [[
+                [6.90, 53.48],
+                [5.98, 51.85],
+                [6.07, 53.51],
+                [6.90, 53.48]
+            ]], 
+            "type": "Polygon"
+        }, 
+        "id": 27, 
+        "properties": {"type": "foo"}, 
+        "type": "Feature"
+    }
+    feat = geom_obj(geom)
+
+    with alt.data_transformers.enable(consolidate_datasets=False):
+        spec = alt.Chart(feat).mark_geoshape().to_dict()
+    assert spec['data']['values']['type'] == 'Feature'
+
+# an empty FeatureCollection is valid
+def test_geo_interface_empty_feature_collection():
+    geom = {
+        "type": "FeatureCollection",
+        "features": []
+    }
+    feat = geom_obj(geom)
+
+    with alt.data_transformers.enable(consolidate_datasets=False):
+        spec = alt.Chart(feat).mark_geoshape().to_dict()
+    assert spec['data']['values'] == []
+
+# Features in a FeatureCollection only keep properties and geometry
+def test_geo_interface_feature_collection():
+    geom = {
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "geometry": {
+                    "coordinates": [[
+                        [6.90, 53.48],
+                        [5.98, 51.85],
+                        [6.07, 53.51],
+                        [6.90, 53.48]
+                    ]], 
+                    "type": "Polygon"
+                }, 
+                "id": 27, 
+                "properties": {
+                    "type": "foo", 
+                    "id": 1, 
+                    "geometry": 1
+                }, 
+                "type": "Feature"
+            },    
+            {
+                "geometry": {
+                    "coordinates": [[
+                        [8.90, 53.48],
+                        [7.98, 51.85],
+                        [8.07, 53.51],
+                        [8.90, 53.48]
+                    ]],
+                    "type": "Polygon"
+                }, 
+                "id": 28, 
+                "properties": {
+                    "type": "foo", 
+                    "id": 2, 
+                    "geometry": 1
+                }, 
+                "type": "Feature"
+            },
+              
+        ]
+    }
+    feat = geom_obj(geom)
+
+    with alt.data_transformers.enable(consolidate_datasets=False):
+        spec = alt.Chart(feat).mark_geoshape().to_dict()
+    assert spec['data']['values'][0]['id'] == 1
+    assert spec['data']['values'][1]['id'] == 2
+    assert 'coordinates' in spec['data']['values'][0]['geometry']
+    assert 'coordinates' in spec['data']['values'][1]['geometry'] 
+    assert spec['data']['values'][0]['type'] == 'Feature'
+    assert spec['data']['values'][1]['type'] == 'Feature' 
+
+# typical output of a __geo_interface__ from geopandas GeoDataFrame
+# notic that the index value is registerd as a commonly used identifier
+# with the name "id" (in this case 49). Similar to serialization of a 
+# pandas DataFrame is the index not included in the output
+def test_geo_interface_feature_collection_gdf():
+    geom = {
+        'bbox': (19.89, -26.82, 29.43, -17.66),
+        'features': [
+            {'bbox': (19.89, -26.82, 29.43, -17.66),
+            'geometry': {
+                'coordinates': [[
+                    [6.90, 53.48],
+                    [5.98, 51.85],
+                    [6.07, 53.51],
+                    [6.90, 53.48]
+                ]], 
+                'type': 'Polygon'},
+            'id': '49',
+            'properties': {
+                'continent': 'Africa','gdp_md_est': 35900.0,
+                'id': 'BWA',
+                'iso_a3': 'BWA',
+                'name': 'Botswana',
+                'pop_est': 2214858},
+            'type': 'Feature'}
+        ],
+        'type': 'FeatureCollection'
+    }
+    feat = geom_obj(geom)
+
+    with alt.data_transformers.enable(consolidate_datasets=False):
+        spec = alt.Chart(feat).mark_geoshape().to_dict()
+    assert spec['data']['values'][0]['id'] == 'BWA'
\ No newline at end of file