Repository: thinkingmachines/geomancer

Issue:
Add SpellBook
Usage ideas:

```python
from geomancer import SpellBook

# When you want to register spells
my_spellbook = SpellBook([
  DistanceToNearest("embassy", within=10000, source_table="tm-geospatial.ph_osm.pois"), # From BQ
  DistanceToNearest("hospital", within=5000, source_table="pois"), # From Spatialite
])

# You can then do multiple casts
my_features = my_spellbook.cast(df, host=[bigquery.Client(), "tests/data/source.sqlite"])

# Saving the Spellbook
my_spellbook.author = "Lj Miranda" # optional 
my_spellbook.description = "Some cool features for other stuff" # optional
my_spellbook.to_json("path/to/my/own/features.json")
```

Some potential challenges:
- It is possible to create a spellbook with spells coming from different warehouses (one feature from BQ, another from SQLite, etc.). However, setting the `source_table` and the `host` is decoupled (one during init, another during `cast()`). 
- Concatenating everything inside a dataframe (similar output column names, etc.). We should do some validation before doing concat?

Some preliminary tasks:
- Write down all possible metadata to include: things that are automatically generated (date? unique ID? etc.) and those that are manually set).

Comments:

Patch:
diff --git a/geomancer/backend/settings.py b/geomancer/backend/settings.py
index c23a36e..6a43bad 100644
--- a/geomancer/backend/settings.py
+++ b/geomancer/backend/settings.py
@@ -45,7 +45,7 @@ class BQConfig(Config):
 
     @property
     def name(self):
-        return "bq"
+        return "bigquery"
 
     DATASET_ID = "geomancer"
     EXPIRY = 3
diff --git a/geomancer/spellbook/__init__.py b/geomancer/spellbook/__init__.py
new file mode 100644
index 0000000..b27808e
--- /dev/null
+++ b/geomancer/spellbook/__init__.py
@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+
+"""A :code:`SpellBook` is a collection of spells that can be sequentially casted and
+merged in a single dataframe
+
+    >>> from geomancer.spells import DistanceOf, NumberOf
+    >>> from geomancer.spellbook import SpellBook
+    >>> spellbook = SpellBook(
+            spells=[
+                DistanceOf(...),
+                NumberOf(...),
+            ],
+        )
+    >>> df = ...
+    >>> df_with_features = spellbook.cast(df)
+
+:code:`SpellBook`s can be distributed by exporting them to JSON files.
+
+    >>> spellbook.author = "My Name"
+    >>> spellbook.description = "My Features"
+    >>> spellbook.to_json("my_features.json")
+
+Now other people can easily reuse your feature extractions in with their own datasets!
+
+    >>> spellbook = SpellBook.read_json("my_features.json")
+    >>> my_df = ...
+    >>> my_df_with_features = spellbook.cast(my_df)
+"""
+
+# Import standard library
+import importlib
+import json
+
+# Import modules
+import pandas as pd
+
+
+class SpellBook(object):
+    def __init__(self, spells, column="WKT", author=None, description=None):
+        """SpellBook constructor
+
+        Parameters
+        ----------
+        spells : list of :class:`geomancer.spells.Spell`
+            List of spell instances.
+        column : str, optional
+            Column to look the geometries from. The default is :code:`WKT`
+        author : str, optional
+            Author of the spell book
+        description : str, optional
+            Description of the spell book
+        """
+        self.column = column
+        self.spells = spells
+        self.author = author
+        self.description = description
+
+    def cast(self, df):
+        """Runs the cast method of each spell in the spell book
+
+        Parameters
+        ----------
+        df : pandas.DataFrame
+            Dataframe containing the points to compare upon. By default, we
+            will look into the :code:`geometry` column. You can specify your
+            own column by passing an argument to the :code:`column` parameter.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Output dataframe with the features from all spells
+        """
+        for spell in self.spells:
+            df = df.join(
+                spell.cast(
+                    df, column=self.column, features_only=True
+                ).set_index("__index_level_0__")
+            )
+        return df
+
+    def to_json(self, filename=None, **kwargs):
+        """Exports spell book as a JSON string
+
+        Parameters
+        ----------
+        filename : str, optional
+            Output filename. If none is given, output is returned
+
+        Returns
+        -------
+        str or None
+            Export of spell book in JSON format
+        """
+        obj = {
+            **self.__dict__,
+            "spells": [
+                {
+                    **s.__dict__,
+                    "module": type(s).__module__,
+                    "type": type(s).__name__,
+                }
+                for s in self.spells
+            ],
+        }
+        if filename:
+            with open(filename, "w") as f:
+                json.dump(obj, f, **kwargs)
+        else:
+            return json.dumps(obj, **kwargs)
+
+    @classmethod
+    def _instantiate_spells(cls, spells):
+        for spell in spells:
+            mod = importlib.import_module(spell.pop("module"))
+            spell_cls = getattr(mod, spell.pop("type"))
+            on = "{}:{}".format(
+                spell.pop("source_column"), spell.pop("source_filter")
+            )
+            yield spell_cls(on, **spell)
+
+    @classmethod
+    def read_json(cls, filename):
+        """Reads a JSON exported spell book
+
+        Parameters
+        ----------
+        filename : str
+            Filename of JSON file to read.
+
+        Returns
+        -------
+        :class:`geomancer.spellbook.SpellBook`
+            :code:`SpellBook` instance parsed from given JSON file.
+        """
+        with open(filename) as f:
+            obj = json.load(f)
+            obj["spells"] = cls._instantiate_spells(obj.pop("spells"))
+            return cls(**obj)
diff --git a/geomancer/spells/base.py b/geomancer/spells/base.py
index 34cd0e3..c82854c 100644
--- a/geomancer/spells/base.py
+++ b/geomancer/spells/base.py
@@ -37,8 +37,8 @@ class Spell(abc.ABC):
         self,
         source_table,
         feature_name,
-        column="WKT",
         source_id="osm_id",
+        dburl=None,
         options=None,
     ):
         """Spell constructor
@@ -49,17 +49,17 @@ class Spell(abc.ABC):
             Table URI to run queries against.
         feature_name : str
             Column name for the output feature.
-        column : str, optional
-            Column to look the geometries from. The default is :code:`WKT`
+        dburl : str, optional
+            Database url used to configure backend connection
         options : geomancer.Config, optional
             Specify configuration for interacting with the database backend.
             Auto-detected if not set.
         """
         self.source_table = source_table
         self.feature_name = feature_name
-        self.options = options
-        self.column = column
         self.source_id = source_id
+        self.dburl = dburl
+        self.options = options
 
     def extract_columns(self, x):
         """Spell constructor
@@ -120,8 +120,22 @@ class Spell(abc.ABC):
         """
         raise NotImplementedError
 
-    @logger.catch
-    def cast(self, df, dburl):
+    def _include_column(self, col, keep_index, features_only):
+        if features_only:
+            return col.key in ("__index_level_0__", self.feature_name)
+        if keep_index:
+            return True
+        return col.key != "__index_level_0__"
+
+    @logger.catch(reraise=True)
+    def cast(
+        self,
+        df,
+        dburl=None,
+        column="WKT",
+        keep_index=False,
+        features_only=False,
+    ):
         """Apply the feature transform to an input pandas.DataFrame
 
         If using BigQuery, a :code:`google.cloud.client.Client`
@@ -133,14 +147,28 @@ class Spell(abc.ABC):
             Dataframe containing the points to compare upon. By default, we
             will look into the :code:`geometry` column. You can specify your
             own column by passing an argument to the :code:`column` parameter.
-        dburl : str
+        dburl : str, optional
             Database url used to configure backend connection
+        column : str, optional
+            Column to look the geometries from. The default is :code:`WKT`
+        keep_index : boolean, optional
+            Include index in output dataframe
+        features_only : boolean, optional
+            Only return features as output dataframe. Automatically sets
+            :code:`keep_index` to :code:`True`.
 
         Returns
         -------
         pandas.DataFrame
             Output dataframe with the features per given point
         """
+        dburl = dburl or self.dburl
+        if not dburl:
+            raise ValueError("dburl was not supplied")
+
+        if features_only:
+            keep_index = True
+
         core = self.get_core(dburl)
 
         # Get engine
@@ -152,11 +180,15 @@ class Spell(abc.ABC):
         )
 
         # Build query
-        query = self.query(source, target, core)
+        query = self.query(source, target, core, column)
 
-        # Remove temporary index column
+        # Filter output columns
         query = select(
-            [col for col in query.columns if col.key != "__index_level_0__"]
+            [
+                col
+                for col in query.columns
+                if self._include_column(col, keep_index, features_only)
+            ]
         ).select_from(query)
 
         # Perform query
diff --git a/geomancer/spells/distance_to_nearest.py b/geomancer/spells/distance_to_nearest.py
index 707310e..df53544 100644
--- a/geomancer/spells/distance_to_nearest.py
+++ b/geomancer/spells/distance_to_nearest.py
@@ -26,15 +26,15 @@ class DistanceToNearest(Spell):
             Column name for the output feature.
         column : str, optional
             Column to look the geometries from. The default is :code:`WKT`
-        options : geomancer.Config
+        options : geomancer.Config, optional
             Specify configuration for interacting with the database backend.
-            Default is a BigQuery Configuration
+            Auto-detected if not set.
         """
         super(DistanceToNearest, self).__init__(**kwargs)
         self.source_column, self.source_filter = self.extract_columns(on)
         self.within = within
 
-    def query(self, source, target, core):
+    def query(self, source, target, core, column):
         # Get all POIs of fclass `on`
         pois = select(
             [source.c[self.source_id], source.c.WKT],
@@ -42,7 +42,7 @@ class DistanceToNearest(Spell):
         ).cte("pois")
         # Compute the distance from `column` to each POI within given distance
         distance = func.ST_Distance(
-            core.ST_GeoFromText(target.c[self.column]),
+            core.ST_GeoFromText(target.c[column]),
             core.ST_GeoFromText(pois.c.WKT),
         )
         pairs = (
diff --git a/geomancer/spells/number_of.py b/geomancer/spells/number_of.py
index 3865177..b42e57b 100644
--- a/geomancer/spells/number_of.py
+++ b/geomancer/spells/number_of.py
@@ -26,15 +26,15 @@ class NumberOf(Spell):
             Column name for the output feature.
         column : str, optional
             Column to look the geometries from. The default is :code:`WKT`
-        options : geomancer.Config
+        options : geomancer.Config, optional
             Specify configuration for interacting with the database backend.
-            Default is a BigQuery Configuration
+            Auto-detected if not set.
         """
         super(NumberOf, self).__init__(**kwargs)
         self.source_column, self.source_filter = self.extract_columns(on)
         self.within = within
 
-    def query(self, source, target, core):
+    def query(self, source, target, core, column):
         # Get all POIs of fclass `on`
         pois = select(
             [source.c[self.source_id], source.c.WKT],
@@ -42,7 +42,7 @@ class NumberOf(Spell):
         ).cte("pois")
         # Compute the distance from `column` to each POI within given distance
         distance = func.ST_Distance(
-            core.ST_GeoFromText(target.c[self.column]),
+            core.ST_GeoFromText(target.c[column]),
             core.ST_GeoFromText(pois.c.WKT),
         )
         pairs = (


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 5953f3f..65193fd 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -10,3 +10,9 @@ def sample_points():
     """Return a set of POINTS in a pandas.DataFrame"""
     df = pd.read_csv("tests/data/sample_points.csv")
     return df
+
+
+@pytest.fixture
+def spellbook_json():
+    with open("tests/data/spellbook.json") as f:
+        return f.read()
diff --git a/tests/data/spellbook.json b/tests/data/spellbook.json
new file mode 100644
index 0000000..80f4339
--- /dev/null
+++ b/tests/data/spellbook.json
@@ -0,0 +1,1 @@
+{"column": "WKT", "spells": [{"source_table": "gis_osm_pois_free_1", "feature_name": "dist_supermarket", "source_id": "osm_id", "dburl": "sqlite:///tests/data/source.sqlite", "options": null, "source_column": "fclass", "source_filter": "supermarket", "within": 10000, "module": "geomancer.spells.distance_to_nearest", "type": "DistanceToNearest"}, {"source_table": "gis_osm_pois_free_1", "feature_name": "num_embassy", "source_id": "osm_id", "dburl": "sqlite:///tests/data/source.sqlite", "options": null, "source_column": "fclass", "source_filter": "embassy", "within": 10000, "module": "geomancer.spells.number_of", "type": "NumberOf"}], "author": null, "description": null}
\ No newline at end of file
diff --git a/tests/spellbook/test_spellbook.py b/tests/spellbook/test_spellbook.py
new file mode 100644
index 0000000..ca65cea
--- /dev/null
+++ b/tests/spellbook/test_spellbook.py
@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+
+# Import modules
+import pandas as pd
+import pytest
+
+# Import from package
+from geomancer.spellbook import SpellBook
+from geomancer.spells import DistanceToNearest, NumberOf
+
+
+@pytest.mark.usefixtures("sample_points")
+def test_spell_dburl(sample_points):
+    with pytest.raises(ValueError, match="dburl was not supplied"):
+        spell = DistanceToNearest(
+            on="embassy",
+            source_table="gis_osm_pois_free_1",
+            feature_name="dist_embassy",
+        )
+        spell.cast(sample_points)
+
+
+@pytest.mark.usefixtures("sample_points")
+def test_spell_keep_index(sample_points):
+    spell = DistanceToNearest(
+        on="embassy",
+        source_table="gis_osm_pois_free_1",
+        feature_name="dist_embassy",
+    )
+    df = spell.cast(
+        sample_points,
+        dburl="sqlite:///tests/data/source.sqlite",
+        keep_index=True,
+    )
+    assert "__index_level_0__" in df.columns
+    df = spell.cast(
+        sample_points,
+        dburl="sqlite:///tests/data/source.sqlite",
+        keep_index=False,
+    )
+    assert "__index_level_0__" not in df.columns
+
+
+@pytest.mark.usefixtures("sample_points")
+def test_spell_features_only(sample_points):
+    spell = DistanceToNearest(
+        on="embassy",
+        source_table="gis_osm_pois_free_1",
+        feature_name="dist_embassy",
+    )
+    df = spell.cast(
+        sample_points,
+        dburl="sqlite:///tests/data/source.sqlite",
+        features_only=True,
+    )
+    assert ["__index_level_0__", "dist_embassy"] == df.columns.tolist()
+
+
+@pytest.fixture
+def spellbook():
+    return SpellBook(
+        [
+            DistanceToNearest(
+                "supermarket",
+                source_table="gis_osm_pois_free_1",
+                feature_name="dist_supermarket",
+                dburl="sqlite:///tests/data/source.sqlite",
+            ),
+            NumberOf(
+                on="embassy",
+                source_table="gis_osm_pois_free_1",
+                feature_name="num_embassy",
+                dburl="sqlite:///tests/data/source.sqlite",
+            ),
+        ]
+    )
+
+
+@pytest.mark.usefixtures("spellbook", "sample_points")
+def test_spellbook_spells(spellbook, sample_points):
+    df = spellbook.cast(sample_points)
+    assert "dist_supermarket" in df.columns
+    assert "num_embassy" in df.columns
+
+
+@pytest.mark.usefixtures("spellbook", "spellbook_json")
+def test_spellbook_to_json(spellbook, spellbook_json):
+    assert spellbook.to_json() == spellbook_json
+
+
+@pytest.mark.usefixtures("spellbook", "spellbook_json")
+def test_spellbook_to_json_file(spellbook, spellbook_json, tmpdir):
+    filename = "spellbook.json"
+    f = tmpdir.mkdir(__name__).join(filename)
+    spellbook.to_json(f.strpath)
+    f.read() == spellbook_json
+
+
+@pytest.mark.usefixtures("spellbook", "spellbook_json")
+def test_spellbook_read_json(spellbook, spellbook_json, tmpdir):
+    filename = "spellbook.json"
+    f = tmpdir.mkdir(__name__).join(filename)
+    f.write(spellbook_json)
+    _spellbook = SpellBook.read_json(f.strpath)
+    assert _spellbook.column == spellbook.column
+    assert _spellbook.author == spellbook.author
+    assert _spellbook.description == spellbook.description
+    for i, spell in enumerate(_spellbook.spells):
+        assert spell.__dict__ == spellbook.spells[i].__dict__
diff --git a/tests/spells/base_test_spell.py b/tests/spells/base_test_spell.py
index c8d0e3c..ec67e14 100644
--- a/tests/spells/base_test_spell.py
+++ b/tests/spells/base_test_spell.py
@@ -38,7 +38,9 @@ class BaseTestSpell:
             engine=engine,
         )
         # Perform the test
-        query = spelldb.spell.query(source=source, target=target, core=core)
+        query = spelldb.spell.query(
+            source=source, target=target, core=core, column="WKT"
+        )
         assert isinstance(query, ClauseElement)
 
     @pytest.mark.usefixtures("spelldb", "sample_points")
diff --git a/tests/spells/test_distance_to_nearest.py b/tests/spells/test_distance_to_nearest.py
index 43106e7..1bde1ed 100644
--- a/tests/spells/test_distance_to_nearest.py
+++ b/tests/spells/test_distance_to_nearest.py
@@ -16,13 +16,16 @@ params = [
         ),
         dburl="sqlite:///tests/data/source.sqlite",
     ),
-    SpellDB(
-        spell=DistanceToNearest(
-            on="primary",
-            source_table="gis_osm_roads_free_1",
-            feature_name="dist_primary",
+    pytest.param(
+        SpellDB(
+            spell=DistanceToNearest(
+                on="primary",
+                source_table="gis_osm_roads_free_1",
+                feature_name="dist_primary",
+            ),
+            dburl="sqlite:///tests/data/source.sqlite",
         ),
-        dburl="sqlite:///tests/data/source.sqlite",
+        marks=pytest.mark.slow,
     ),
     pytest.param(
         SpellDB(
diff --git a/tests/spells/test_number_of.py b/tests/spells/test_number_of.py
index 9680f5e..2371e37 100644
--- a/tests/spells/test_number_of.py
+++ b/tests/spells/test_number_of.py
@@ -17,13 +17,16 @@ params = [
         ),
         dburl="sqlite:///tests/data/source.sqlite",
     ),
-    SpellDB(
-        spell=NumberOf(
-            on="primary",
-            source_table="gis_osm_roads_free_1",
-            feature_name="num_primary",
+    pytest.param(
+        SpellDB(
+            spell=NumberOf(
+                on="primary",
+                source_table="gis_osm_roads_free_1",
+                feature_name="num_primary",
+            ),
+            dburl="sqlite:///tests/data/source.sqlite",
         ),
-        dburl="sqlite:///tests/data/source.sqlite",
+        marks=pytest.mark.slow,
     ),
     pytest.param(
         SpellDB(