Repository: big-o/cherrypicker

Issue:
Delim parameter for flatten is not customizable
The method signature of `CherryPickerMapping.flatten` implies that `delim` can be set to something else, for example very common delimiters like `.` or `/`.

- However, as a `@property`, arguments passed to it are ignored and don't override the default value. If it is intended that the delimiter can be overridden, `flatten` should not be a property.
- Secondly, the inner calls `cls._flatten(…)` miss the `delim` argument so that the overridden value would only be appended after the first key, whereas deeper keys get again the default delimiter.

**Test case:**  
```python
from cherrypicker import CherryPicker
CherryPicker({"outer": {"inner": {"innermost": "value"}}}).flatten(delim=".").get()
```

**Expected result:**  
`{'outer.inner.innermost': 'value'}`

**Actual result:**  
`{'outer_inner_innermost': 'value'}`


Discussion:

Patch:
diff --git a/cherrypicker/__about__.py b/cherrypicker/__about__.py
index 78a9e73..01aaf12 100644
--- a/cherrypicker/__about__.py
+++ b/cherrypicker/__about__.py
@@ -1,6 +1,6 @@
 __author__ = "big-o"
 __author_email__ = "big-o@users.noreply.github.com"
-__version__ = "0.3.1"
+__version__ = "0.4.0"
 __release__ = __version__
 __name__ = "cherrypicker"
 __title__ = "CherryPicker"
diff --git a/cherrypicker/__init__.py b/cherrypicker/__init__.py
index b8bea70..d692939 100644
--- a/cherrypicker/__init__.py
+++ b/cherrypicker/__init__.py
@@ -1,3 +1,3 @@
-from .traversable import *
 from .leaf import *
 from .picker import *
+from .traversable import *
diff --git a/cherrypicker/leaf.py b/cherrypicker/leaf.py
index 13c7895..83aced8 100644
--- a/cherrypicker/leaf.py
+++ b/cherrypicker/leaf.py
@@ -1,9 +1,9 @@
 from __future__ import division
 
-from .exceptions import LeafError
-from .picker import CherryPicker
 from typing import Any
 
+from .exceptions import LeafError
+from .picker import CherryPicker
 
 __all__ = ("CherryPickerLeaf",)
 
diff --git a/cherrypicker/picker.py b/cherrypicker/picker.py
index dceddc1..df49d9b 100644
--- a/cherrypicker/picker.py
+++ b/cherrypicker/picker.py
@@ -1,9 +1,9 @@
 from __future__ import division
 
 from collections.abc import Iterable, Mapping
-from joblib import effective_n_jobs
 from typing import Any, Callable, List, NoReturn, Optional, Tuple, Union
 
+from joblib import effective_n_jobs
 
 __all__ = ("CherryPicker",)
 
diff --git a/cherrypicker/traversable.py b/cherrypicker/traversable.py
index 47ea4b5..138a257 100644
--- a/cherrypicker/traversable.py
+++ b/cherrypicker/traversable.py
@@ -1,15 +1,15 @@
 from __future__ import division
 
-from .picker import CherryPicker
-
+import re
 from collections.abc import Callable
 from fnmatch import fnmatchcase
 from functools import partial
 from itertools import chain
-from joblib import Parallel, delayed
-import re
 from typing import Any, Dict, Generator, NoReturn, Optional, Tuple, Union
 
+from joblib import Parallel, delayed
+
+from .picker import CherryPicker
 
 __all__ = ("CherryPickerIterable", "CherryPickerMapping", "CherryPickerTraversable")
 
@@ -214,6 +214,10 @@ class CherryPickerTraversable(CherryPicker):
         elif how == "all":
             return True
 
+    @property
+    def flat(self):
+        return self.flatten()
+
     def keys(self) -> NoReturn:
         raise NotImplementedError()
 
@@ -289,8 +293,10 @@ class CherryPickerMapping(CherryPickerTraversable):
         if flat is None:
             flat = {}
 
+        dlen = len(delim)
+
         if maxdepth is not None and depth > maxdepth:
-            flat[prefix[:-1]] = obj
+            flat[prefix[:-dlen]] = obj
             return flat
 
         ccls = cls._get_cherry_class(obj)
@@ -300,6 +306,7 @@ class CherryPickerMapping(CherryPickerTraversable):
                     obj[key],
                     flat,
                     prefix="{}{}{}".format(prefix, key, delim),
+                    delim=delim,
                     maxdepth=maxdepth,
                     depth=depth + 1,
                 )
@@ -310,16 +317,16 @@ class CherryPickerMapping(CherryPickerTraversable):
                     val,
                     flat,
                     prefix="{}{}{}".format(prefix, idx, delim),
+                    delim=delim,
                     maxdepth=maxdepth,
                     depth=depth + 1,
                 )
 
         else:
-            flat[prefix[:-1]] = obj
+            flat[prefix[:-dlen]] = obj
 
         return flat
 
-    @property
     def flatten(self, delim="_", maxdepth=100):
         """
         Flatten down the object so that all of its values are leaf nodes.
@@ -407,14 +414,13 @@ class CherryPickerIterable(CherryPickerTraversable):
             ccls = cls._get_cherry_class(item)
             if ccls is CherryPickerMapping:
                 flats.append(
-                    CherryPickerMapping._flatten(item, delim="_", maxdepth=100)
+                    CherryPickerMapping._flatten(item, delim=delim, maxdepth=100)
                 )
             else:
                 flats.append(item)
 
         return flats
 
-    @property
     def flatten(self, delim="_", maxdepth=100):
         with Parallel(self._effective_n_jobs) as parallel:
             flats = parallel(
diff --git a/cherrypicker/util.py b/cherrypicker/util.py
index 3264d95..8f4ac9f 100644
--- a/cherrypicker/util.py
+++ b/cherrypicker/util.py
@@ -1,7 +1,6 @@
 import collections.abc
 from typing import Any, Generator
 
-
 __all__ = ("OrderedSet",)
 
 
diff --git a/docs/source/conf.py b/docs/source/conf.py
index f2a0862..a6750bb 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -18,13 +18,13 @@ import sys
 sys.path.insert(0, os.path.abspath("../../"))
 
 from cherrypicker.__about__ import (
-    __name__,
-    __title__,
-    __version__,
     __author__,
     __author_email__,
-    __release__,
     __description__,
+    __name__,
+    __release__,
+    __title__,
+    __version__,
 )
 
 # -- Project information -----------------------------------------------------
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 4180799..029477f 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -96,9 +96,10 @@ custom functions. Read all about them in the :doc:`filter` documentation.
 Of course, it would be nice if we could extract data in the example above from
 both the base level and the ``info`` sub-level of each item and put them into a
 flat table, ready to load into your favourite data analysis package. We can do
-this in :mod:`cherrypicker` with :meth:`cherrypicker.CherryPickerMapping.flatten`.
-Let's say that each item in our data list has a city name and a list of average
-low/high temperatures for each month of the year:
+this conveniently in :mod:`cherrypicker` with the ``flat`` property of our picker
+which gives us a flattened view of the data. Let's say that each item in our data
+list has a city name and a list of average low/high temperatures for each month of the
+year:
 
 .. code-block:: python
 
@@ -132,18 +133,31 @@ monthly temperatures alongside each other:
 
 .. code-block:: python
 
-    >>> picker.flatten(
+    >>> picker.flat(
     ...     monthlyAvg_0_high=lambda tmp: tmp > 30
     ... )['city', 'monthlyAvg_0_high'].get()
     [['Bangkok', 33], ['Brasilia', 31], ['Ho Chi Minh City', 33], ...]
 
 .. code-block:: python
 
-    >>> picker.flatten(
+    >>> picker.flat(
     ...     monthlyAvg_0_high=lambda tmp: tmp < 0
     ... )['city', 'monthlyAvg_0_high'].get()
     [['Calgary', -1], ['Montreal', -4], ['Moscow', -4], ...]
 
+If you would like to customise the flattening behaviour, use the
+:meth:`cherrypicker.CherryPickerMapping.flatten` method instead:
+
+.. code-block:: python
+
+    >>> picker.flat[0].get()
+    {'id': 1, 'city': 'Amsterdam', 'country': 'Netherlands', 'monthlyAvg_0_high': 7, ...}
+
+.. code-block:: python
+
+    >>> picker.flatten(delim='.')[0].get()
+    {'id': 1, 'city': 'Amsterdam', 'country': 'Netherlands', 'monthlyAvg.0.high': 7, ...}
+
 One final point to note is that :mod:`cherrypicker` understands data by looking
 at its *interfaces* rather than its *types*. This means that it isn't just
 limited to JSON data: as long as it can act like a dict or list, you can start
diff --git a/setup.cfg b/setup.cfg
index 6c29091..90d74d6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,7 +2,7 @@
 addopts = -ra --black --cov=cherrypicker --cov-report=term
 
 [tox:tox]
-envlist = clean,py37,py38
+envlist = clean,py37,py38,py39
 
 [testenv]
 usedevelop = True
diff --git a/setup.py b/setup.py
index fb39ccd..bfa635d 100644
--- a/setup.py
+++ b/setup.py
@@ -1,8 +1,8 @@
 # Always prefer setuptools over distutils
-from setuptools import setup, find_packages
-import os
 import importlib.util
+import os
 
+from setuptools import find_packages, setup
 
 about = None
 for root, dirs, files in os.walk("."):


Test patch:
diff --git a/requirements-test.txt b/requirements-test.txt
index 74b453c..9fbbb71 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -3,3 +3,4 @@ coverage
 pytest
 pytest-black
 pytest-cov
+pytest-sphinx
diff --git a/tests/test_picker.py b/tests/test_picker.py
index 4103e7c..2059918 100644
--- a/tests/test_picker.py
+++ b/tests/test_picker.py
@@ -1,10 +1,12 @@
-from cherrypicker import CherryPicker
-from cherrypicker.exceptions import *
 import json
 import os
-import pytest
 import re
 
+import pytest
+
+from cherrypicker import CherryPicker
+from cherrypicker.exceptions import *
+
 
 def abs_path(path):
     return os.path.join(os.path.dirname(__file__), path)
@@ -67,10 +69,10 @@ def run_flatten_tests(n_jobs=None):
 
     fpicker = CherryPicker(first, n_jobs=n_jobs)
     picker = CherryPicker(data, n_jobs=n_jobs)
-    assert list(fpicker.flatten.keys()) == flat_keys
+    assert list(fpicker.flat.keys()) == flat_keys
     assert list(fpicker.flatten().keys()) == flat_keys
 
-    assert list(picker.flatten[0].keys()) == flat_keys
+    assert list(picker.flat[0].keys()) == flat_keys
     assert list(picker.flatten()[0].keys()) == flat_keys
 
 
diff --git a/tests/test_util.py b/tests/test_util.py
index 25f9e4c..50ee114 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -1,7 +1,7 @@
-from cherrypicker.util import *
-
 import pytest
 
+from cherrypicker.util import *
+
 
 def test_orderedset():
     l = list(range(10))