Repository: kedro-org/kedro

Issue:
[KED-1312] New DataSet : GeoJSONLocalDataSet 
## Description
Create a GeoJSONLocalDataSet that would be read as a [geopandas.GeoDataFrame](http://geopandas.org/reference/geopandas.GeoDataFrame.html#geopandas.GeoDataFrame)

## Context
GeoDataFrame allow for fast spatial joins in pandas (with the use of Rtree) 

## Possible Implementation
Inherhit from [AbstractDataSet](https://github.com/lblanche/kedro/blob/bfd802db5cb72841730f5e2e1a787255658cd89d/kedro/io/core.py#L86)  and create class in contrib.io

```python
"""GeoJSONLocalDataSet loads and saves data to a local geojson file. The
underlying functionality is supported by geopandas, so it supports all
allowed geopandas (pandas) options for loading and saving geosjon files.
"""
from os.path import isfile
from typing import Any, Union, Dict

import geopandas as gpd

from kedro.io import AbstractDataSet


class GeoJSONLocalDataSet(AbstractDataSet):
    """``GeoJSONLocalDataSet`` loads and saves data to a local geojson file. The
    underlying functionality is supported by geopandas, so it supports all
    allowed geopandas (pandas) options for loading and saving geosjon files.

    Example:
    ::

        >>> import geopandas as gpd
        >>> from shapely.geometry import Point
        >>>
        >>> data = gpd.GeoDataFrame({'col1': [1, 2], 'col2': [4, 5],
        >>>                      'col3': [5, 6]}, geometry=[Point(1,1), Point(2,4)])
        >>> data_set = GeoJSONLocalDataSet(filepath="test.geojson",
        >>>                                save_args=None)
        >>>                                 
        >>> data_set.save(data)
        >>> reloaded = data_set.load()
        >>>
        >>> assert data.equals(reloaded)

    """

    def _describe(self) -> Dict[str, Any]:
        return dict(
            filepath=self._filepath,
            driver=self._driver,
            load_args=self._load_args,
            save_args=self._save_args,
        )

    def __init__(
        self,
        filepath: str,
        driver: str = "GeoJSON",
        load_args: Dict[str, Any] = None,
        save_args: Dict[str, Any] = None,
    ) -> None:
        """Creates a new instance of ``GeoJSONLocalDataSet`` pointing to a concrete
        filepath.

        Args:
            driver: The driver used to write to GeoJSON files. The default
                          engine is 'GeoJSON'.

            filepath: path to a geojson file.

            load_args: GeoPandas options for loading GeoJSON files.
                Here you can find all available arguments:

            save_args: GeoPandas options for saving geojson files.
                Here you can find all available arguments:
                The default_save_arg driver is 'GeoJSON', all others preserved.

        """
        self._filepath = filepath
        default_save_args = {"driver": "GeoJSON"}
        default_load_args = {}

        self._load_args = (
            {**default_load_args, **load_args}
            if load_args is not None
            else default_load_args
        )
        self._save_args = (
            {**default_save_args, **save_args}
            if save_args is not None
            else default_save_args
        )
        self._driver = driver

    def _load(self) -> Union[gpd.GeoDataFrame, Dict[str, gpd.GeoDataFrame]]:
        return gpd.read_file(self._filepath, **self._load_args)

    def _save(self, data: gpd.GeoDataFrame) -> None:
        data.to_file(**self._save_args)

    def _exists(self) -> bool:
        return isfile(self._filepath)

```

Remark : other types of  geo data format could be added later (eg. .shp, .kml etc) 



Discussion:

Resolving patch:
diff --git a/kedro/extras/datasets/geopandas/__init__.py b/kedro/extras/datasets/geopandas/__init__.py
new file mode 100644
--- /dev/null
+++ b/kedro/extras/datasets/geopandas/__init__.py
@@ -0,0 +1,31 @@
+# Copyright 2020 QuantumBlack Visual Analytics Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
+# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS
+# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo
+# (either separately or in combination, "QuantumBlack Trademarks") are
+# trademarks of QuantumBlack. The License does not grant you any right or
+# license to the QuantumBlack Trademarks. You may not use the QuantumBlack
+# Trademarks or any confusingly similar mark as a trademark for your product,
+#     or use the QuantumBlack Trademarks in any other manner that might cause
+# confusion in the marketplace, including but not limited to in advertising,
+# on websites, or on software.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""``GeoJSONLocalDataset`` is an ``AbstractVersionedDataSet`` to save and load GeoJSON files.
+"""
+
+from .geojson_dataset import GeoJSONDataSet  # NOQA
diff --git a/kedro/extras/datasets/geopandas/geojson_dataset.py b/kedro/extras/datasets/geopandas/geojson_dataset.py
new file mode 100644
--- /dev/null
+++ b/kedro/extras/datasets/geopandas/geojson_dataset.py
@@ -0,0 +1,166 @@
+# Copyright 2020 QuantumBlack Visual Analytics Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
+# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS
+# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo
+# (either separately or in combination, "QuantumBlack Trademarks") are
+# trademarks of QuantumBlack. The License does not grant you any right or
+# license to the QuantumBlack Trademarks. You may not use the QuantumBlack
+# Trademarks or any confusingly similar mark as a trademark for your product,
+#     or use the QuantumBlack Trademarks in any other manner that might cause
+# confusion in the marketplace, including but not limited to in advertising,
+# on websites, or on software.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""GeoJSONDataSet loads and saves data to a local geojson file. The
+underlying functionality is supported by geopandas, so it supports all
+allowed geopandas (pandas) options for loading and saving geosjon files.
+"""
+import copy
+from pathlib import PurePosixPath
+from typing import Any, Dict, Union
+
+import fsspec
+import geopandas as gpd
+
+from kedro.io.core import (
+    AbstractVersionedDataSet,
+    DataSetError,
+    Version,
+    get_filepath_str,
+    get_protocol_and_path,
+)
+
+
+class GeoJSONDataSet(AbstractVersionedDataSet):
+    """``GeoJSONDataSet`` loads/saves data to a GeoJSON file using an underlying filesystem
+    (eg: local, S3, GCS).
+    The underlying functionality is supported by geopandas, so it supports all
+    allowed geopandas (pandas) options for loading and saving GeoJSON files.
+
+    Example:
+    ::
+
+        >>> import geopandas as gpd
+        >>> from shapely.geometry import Point
+        >>> from kedro.extras.datasets.geopandas import GeoJSONDataSet
+        >>>
+        >>> data = gpd.GeoDataFrame({'col1': [1, 2], 'col2': [4, 5],
+        >>>                      'col3': [5, 6]}, geometry=[Point(1,1), Point(2,4)])
+        >>> # data_set = GeoJSONDataSet(filepath="gcs://bucket/test.geojson",
+        >>>                                save_args=None)
+        >>> data_set = GeoJSONDataSet(filepath="test.geojson",
+        >>>                                save_args=None)
+        >>> data_set.save(data)
+        >>> reloaded = data_set.load()
+        >>>
+        >>> assert data.equals(reloaded)
+
+    """
+
+    DEFAULT_LOAD_ARGS = {}  # type: Dict[str, Any]
+    DEFAULT_SAVE_ARGS = {"driver": "GeoJSON"}
+
+    # pylint: disable=too-many-arguments
+    def __init__(
+        self,
+        filepath: str,
+        load_args: Dict[str, Any] = None,
+        save_args: Dict[str, Any] = None,
+        version: Version = None,
+        credentials: Dict[str, Any] = None,
+        fs_args: Dict[str, Any] = None,
+    ) -> None:
+        """Creates a new instance of ``GeoJSONDataSet`` pointing to a concrete GeoJSON file
+        on a specific filesystem fsspec.
+
+        Args:
+
+            filepath: Filepath to a GeoJSON file prefixed with a protocol like `s3://`.
+                If prefix is not provided `file` protocol (local filesystem) will be used.
+                The prefix should be any protocol supported by ``fsspec``.
+                Note: `http(s)` doesn't support versioning.
+            load_args: GeoPandas options for loading GeoJSON files.
+                Here you can find all available arguments:
+                https://geopandas.org/reference/geopandas.read_file.html
+            save_args: GeoPandas options for saving geojson files.
+                Here you can find all available arguments:
+                https://geopandas.org/reference.html#geopandas.GeoDataFrame.to_file
+                The default_save_arg driver is 'GeoJSON', all others preserved.
+            version: If specified, should be an instance of
+                ``kedro.io.core.Version``. If its ``load`` attribute is
+                None, the latest version will be loaded. If its ``save``
+            credentials: credentials required to access the underlying filesystem.
+                Eg. for ``GCFileSystem`` it would look like `{'token': None}`.
+            fs_args: Extra args to pass into the underlying filesystem
+                Eg. for ``GCFileSystem`` it would look like `{'project': 'my-project' , ...}`.
+        """
+        _fs_args = copy.deepcopy(fs_args) or {}
+        _credentials = copy.deepcopy(credentials) or {}
+        protocol, path = get_protocol_and_path(filepath, version)
+        self._protocol = protocol
+        self._fs = fsspec.filesystem(self._protocol, **_credentials, **_fs_args)
+
+        super().__init__(
+            filepath=PurePosixPath(path),
+            version=version,
+            exists_function=self._fs.exists,
+            glob_function=self._fs.glob,
+        )
+
+        self._load_args = copy.deepcopy(self.DEFAULT_LOAD_ARGS)
+        if load_args is not None:
+            self._load_args.update(load_args)
+
+        self._save_args = copy.deepcopy(self.DEFAULT_SAVE_ARGS)
+        if save_args is not None:
+            self._save_args.update(save_args)
+
+    def _load(self) -> Union[gpd.GeoDataFrame, Dict[str, gpd.GeoDataFrame]]:
+        load_path = get_filepath_str(self._get_load_path(), self._protocol)
+        with self._fs.open(load_path, mode="rb") as fs_file:
+            return gpd.read_file(fs_file, **self._load_args)
+
+    def _save(self, data: gpd.GeoDataFrame) -> None:
+        save_path = get_filepath_str(self._get_save_path(), self._protocol)
+        with self._fs.open(save_path, mode="wb") as fs_file:
+            data.to_file(fs_file, **self._save_args)
+        self.invalidate_cache()
+
+    def _exists(self) -> bool:
+        try:
+            load_path = get_filepath_str(self._get_load_path(), self._protocol)
+        except DataSetError:
+            return False
+        return self._fs.exists(load_path)
+
+    def _describe(self) -> Dict[str, Any]:
+        return dict(
+            filepath=self._filepath,
+            protocol=self._protocol,
+            load_args=self._load_args,
+            save_args=self._save_args,
+            version=self._version,
+        )
+
+    def _release(self) -> None:
+        self.invalidate_cache()
+
+    def invalidate_cache(self) -> None:
+        """Invalidate underlying filesystem cache."""
+        filepath = get_filepath_str(self._filepath, self._protocol)
+        self._fs.invalidate_cache(filepath)
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -99,6 +99,7 @@
     "matplotlib": ["matplotlib>=3.0.3, <4.0"],
     "networkx": ["networkx>=2.4, <3.0"],
     "memory_profiler": ["memory_profiler>=0.50.0, <1.0"],
+    "geopandas": ["geopandas<=0.6.0, <1.0"],
 }
 
 extras_require["pandas"] = [*extras_require["azure"], *extras_require["gbq"]]


Test patch:
diff --git a/test_requirements.txt b/test_requirements.txt
--- a/test_requirements.txt
+++ b/test_requirements.txt
@@ -8,6 +8,7 @@ black==v19.10.b0; python_version >= '3.6'
 dask[complete]>=2.6.0, <3.0
 flake8>=3.5,<4.0
 gcsfs>=0.3.0, <1.0
+geopandas>=0.6.0, <1.0
 gitdb2==3.0.0 # pin trufflehog dependency to get it working for now https://github.com/dxa4481/truffleHog/issues/200
 hdfs>=2.5.8, <3.0
 import-linter==1.0; python_version >= '3.6'
diff --git a/tests/extras/datasets/geojson/__init__.py b/tests/extras/datasets/geojson/__init__.py
new file mode 100644
diff --git a/tests/extras/datasets/geojson/test_geojson_dataset.py b/tests/extras/datasets/geojson/test_geojson_dataset.py
new file mode 100644
--- /dev/null
+++ b/tests/extras/datasets/geojson/test_geojson_dataset.py
@@ -0,0 +1,231 @@
+# Copyright 2020 QuantumBlack Visual Analytics Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
+# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS
+# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo
+# (either separately or in combination, "QuantumBlack Trademarks") are
+# trademarks of QuantumBlack. The License does not grant you any right or
+# license to the QuantumBlack Trademarks. You may not use the QuantumBlack
+# Trademarks or any confusingly similar mark as a trademark for your product,
+#     or use the QuantumBlack Trademarks in any other manner that might cause
+# confusion in the marketplace, including but not limited to in advertising,
+# on websites, or on software.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pathlib import PurePosixPath
+
+import geopandas as gpd
+import pytest
+from fsspec.implementations.http import HTTPFileSystem
+from fsspec.implementations.local import LocalFileSystem
+from gcsfs import GCSFileSystem
+from pandas.util.testing import assert_frame_equal
+from s3fs import S3FileSystem
+from shapely.geometry import Point
+
+from kedro.extras.datasets.geopandas import GeoJSONDataSet
+from kedro.io import DataSetError
+from kedro.io.core import Version, generate_timestamp
+
+
+@pytest.fixture(params=[None])
+def load_version(request):
+    return request.param
+
+
+@pytest.fixture(params=[None])
+def save_version(request):
+    return request.param or generate_timestamp()
+
+
+@pytest.fixture
+def filepath(tmp_path):
+    return str(tmp_path / "test.geojson")
+
+
+@pytest.fixture(params=[None])
+def load_args(request):
+    return request.param
+
+
+@pytest.fixture(params=[{"driver": "GeoJSON"}])
+def save_args(request):
+    return request.param
+
+
+@pytest.fixture
+def dummy_dataframe():
+    return gpd.GeoDataFrame(
+        {"col1": [1, 2], "col2": [4, 5], "col3": [5, 6]},
+        geometry=[Point(1, 1), Point(2, 2)],
+    )
+
+
+@pytest.fixture
+def geojson_data_set(filepath, load_args, save_args):
+    return GeoJSONDataSet(filepath=filepath, load_args=load_args, save_args=save_args)
+
+
+@pytest.fixture
+def versioned_geojson_data_set(filepath, load_version, save_version):
+    return GeoJSONDataSet(
+        filepath=filepath, version=Version(load_version, save_version),
+    )
+
+
+class TestGeoJSONDataSet:
+    def test_save_and_load(self, geojson_data_set, dummy_dataframe):
+        """Test that saved and reloaded data matches the original one."""
+        geojson_data_set.save(dummy_dataframe)
+        reloaded_df = geojson_data_set.load()
+        assert_frame_equal(reloaded_df, dummy_dataframe)
+
+    @pytest.mark.parametrize("geojson_data_set", [{"index": False}], indirect=True)
+    def test_load_missing_file(self, geojson_data_set):
+        """Check the error while trying to load from missing source."""
+        pattern = r"Failed while loading data from data set GeoJSONDataSet"
+        with pytest.raises(DataSetError, match=pattern):
+            geojson_data_set.load()
+
+    def test_exists(self, geojson_data_set, dummy_dataframe):
+        """Test `exists` method invocation for both cases."""
+        assert not geojson_data_set.exists()
+        geojson_data_set.save(dummy_dataframe)
+        assert geojson_data_set.exists()
+
+    @pytest.mark.parametrize(
+        "load_args", [{"crs": "init:4326"}, {"crs": "init:2154", "driver": "GeoJSON"}]
+    )
+    def test_load_extra_params(self, geojson_data_set, load_args):
+        """Test overriding default save args"""
+        for k, v in load_args.items():
+            assert geojson_data_set._load_args[k] == v
+
+    @pytest.mark.parametrize(
+        "save_args", [{"driver": "ESRI Shapefile"}, {"driver": "GPKG", "layer": "test"}]
+    )
+    def test_save_extra_params(self, geojson_data_set, save_args):
+        """Test overriding default save args"""
+        for k, v in save_args.items():
+            assert geojson_data_set._save_args[k] == v
+
+    @pytest.mark.parametrize(
+        "filepath,instance_type",
+        [
+            ("s3://bucket/file.geojson", S3FileSystem),
+            ("/tmp/test.geojson", LocalFileSystem),
+            ("gcs://bucket/file.geojson", GCSFileSystem),
+            ("file:///tmp/file.geojson", LocalFileSystem),
+            ("https://example.com/file.geojson", HTTPFileSystem),
+        ],
+    )
+    def test_protocol_usage(self, filepath, instance_type):
+        geojson_data_set = GeoJSONDataSet(filepath=filepath)
+        assert isinstance(geojson_data_set._fs, instance_type)
+
+        if geojson_data_set._protocol == "https":
+            path = filepath.split("://")[-1]
+        else:
+            path = geojson_data_set._fs._strip_protocol(filepath)
+
+        assert str(geojson_data_set._filepath) == path
+        assert isinstance(geojson_data_set._filepath, PurePosixPath)
+
+    def test_catalog_release(self, mocker):
+        fs_mock = mocker.patch("fsspec.filesystem").return_value
+        filepath = "test.geojson"
+        geojson_data_set = GeoJSONDataSet(filepath=filepath)
+        geojson_data_set.release()
+        fs_mock.invalidate_cache.assert_called_once_with(filepath)
+
+
+class TestGeoJSONDataSetVersioned:
+    def test_version_str_repr(self, load_version, save_version):
+        """Test that version is in string representation of the class instance
+        when applicable."""
+        filepath = "test.geojson"
+        ds = GeoJSONDataSet(filepath=filepath)
+        ds_versioned = GeoJSONDataSet(
+            filepath=filepath, version=Version(load_version, save_version)
+        )
+        assert filepath in str(ds)
+        assert "version" not in str(ds)
+
+        assert filepath in str(ds_versioned)
+        ver_str = "version=Version(load={}, save='{}')".format(
+            load_version, save_version
+        )
+        assert ver_str in str(ds_versioned)
+        assert "GeoJSONDataSet" in str(ds_versioned)
+        assert "GeoJSONDataSet" in str(ds)
+        assert "protocol" in str(ds_versioned)
+        assert "protocol" in str(ds)
+
+    def test_save_and_load(self, versioned_geojson_data_set, dummy_dataframe):
+        """Test that saved and reloaded data matches the original one for
+        the versioned data set."""
+        versioned_geojson_data_set.save(dummy_dataframe)
+        reloaded_df = versioned_geojson_data_set.load()
+        assert_frame_equal(reloaded_df, dummy_dataframe)
+
+    def test_no_versions(self, versioned_geojson_data_set):
+        """Check the error if no versions are available for load."""
+        pattern = r"Did not find any versions for GeoJSONDataSet\(.+\)"
+        with pytest.raises(DataSetError, match=pattern):
+            versioned_geojson_data_set.load()
+
+    def test_exists(self, versioned_geojson_data_set, dummy_dataframe):
+        """Test `exists` method invocation for versioned data set."""
+        assert not versioned_geojson_data_set.exists()
+        versioned_geojson_data_set.save(dummy_dataframe)
+        assert versioned_geojson_data_set.exists()
+
+    def test_prevent_override(self, versioned_geojson_data_set, dummy_dataframe):
+        """Check the error when attempt to override the same data set
+        version."""
+        versioned_geojson_data_set.save(dummy_dataframe)
+        pattern = (
+            r"Save path \`.+\` for GeoJSONDataSet\(.+\) must not "
+            r"exist if versioning is enabled"
+        )
+        with pytest.raises(DataSetError, match=pattern):
+            versioned_geojson_data_set.save(dummy_dataframe)
+
+    @pytest.mark.parametrize(
+        "load_version", ["2019-01-01T23.59.59.999Z"], indirect=True
+    )
+    @pytest.mark.parametrize(
+        "save_version", ["2019-01-02T00.00.00.000Z"], indirect=True
+    )
+    def test_save_version_warning(
+        self, versioned_geojson_data_set, load_version, save_version, dummy_dataframe
+    ):
+        """Check the warning when saving to the path that differs from
+        the subsequent load path."""
+        pattern = (
+            r"Save version `{0}` did not match load version `{1}` "
+            r"for GeoJSONDataSet\(.+\)".format(save_version, load_version)
+        )
+        with pytest.warns(UserWarning, match=pattern):
+            versioned_geojson_data_set.save(dummy_dataframe)
+
+    def test_http_filesystem_no_versioning(self):
+        pattern = r"HTTP\(s\) DataSet doesn't support versioning\."
+
+        with pytest.raises(DataSetError, match=pattern):
+            GeoJSONDataSet(
+                filepath="https://example/file.geojson", version=Version(None, None)
+            )