Repository: niconoe/pyinaturalist
Pull request #117 (2021-03-10 01:46:38)

Some minor updates to response formatting and other utility functions
* Add `response_utils` module for format functions that aren't directly used by API
* Add `simplify_observation()` function
* Add `to_dataframe()` function (works only if pandas is installed)
* Remove `minify` from `get_taxa_autocomplete()`, and instead show a similar example in docstring


Patch:
diff --git a/HISTORY.md b/HISTORY.md
index 577e324..b42b1e7 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -26,11 +26,13 @@
   * `rest_api.get_all_observation_fields()`
 * Removed `node_api.get_all_observation_species_counts()`, since this was only added recently
 * Updated `rest_api.get_observation_fields()` to return a dict with `'results'` for consistency with other endpoints
+* Removed `minify` option to `get_taxa_autocomplete`, and instead show a similar example in docstring
 
 ### Other Changes
 * Added global rate-limiting to stay within the rates suggested in
   [API Recommended Practices](https://www.inaturalist.org/pages/api+recommended+practices)
   (per second, minute, and day)
+* Added new module `pyinaturalist.response_utils` for optional formatting functions
 
 ## 0.12 (2021-02-02)
 [See all Issues & PRs](https://github.com/niconoe/pyinaturalist/milestone/3?closed=1)
diff --git a/examples/observations_to_gpx.py b/examples/observations_to_gpx.py
index c3ffff7..d6b9539 100644
--- a/examples/observations_to_gpx.py
+++ b/examples/observations_to_gpx.py
@@ -9,7 +9,8 @@ from gpxpy.gpx import GPX, GPXTrack, GPXTrackPoint, GPXTrackSegment, GPXWaypoint
 
 from pyinaturalist.constants import JsonResponse, List
 from pyinaturalist.node_api import get_all_observations, get_observations  # noqa
-from pyinaturalist.response_format import convert_observation_timestamps, format_observation
+from pyinaturalist.response_format import convert_observation_timestamps
+from pyinaturalist.response_utils import format_observation_str
 
 logger = getLogger(__name__)
 
@@ -67,7 +68,7 @@ def observation_to_gpx_point(observation: JsonResponse, track: bool = True):
         latitude=lat,
         longitude=long,
         time=convert_observation_timestamps(observation),
-        comment=format_observation(observation),
+        comment=format_observation_str(observation),
     )
     point.description = observation["description"]
     point.link = link
diff --git a/pyinaturalist/api_docs.py b/pyinaturalist/api_docs.py
index 7c3a75a..b978bc2 100644
--- a/pyinaturalist/api_docs.py
+++ b/pyinaturalist/api_docs.py
@@ -493,12 +493,6 @@ def _geojson_properties(properties: List[str] = None):
     """
 
 
-def _minify(minify: str = None):
-    """
-    minify: Condense each match into a single string containing taxon ID, rank, and name
-    """
-
-
 def _name(name: str = None):
     """
     name: Name must match this value
diff --git a/pyinaturalist/forge_utils.py b/pyinaturalist/forge_utils.py
index f592c04..7ee5393 100644
--- a/pyinaturalist/forge_utils.py
+++ b/pyinaturalist/forge_utils.py
@@ -4,7 +4,6 @@ This module makes ``python-forge`` optional; if not installed, these functions w
 without modifying the target functions.
 """
 from inspect import cleandoc
-from itertools import chain
 from logging import getLogger
 from typing import Callable, List
 
@@ -50,7 +49,7 @@ def document_request_params(template_functions: List[TemplateFunction]) -> Calla
         template_functions: Template functions containing docstrings and params to apply to the
             wrapped function
     """
-    template_functions = [*template_functions, _user_agent]
+    template_functions += [_user_agent]
 
     def wrapper(func):
         # Modify docstring
@@ -137,12 +136,10 @@ def _get_combined_revision(template_functions: List[TemplateFunction]):
     """Create a :py:class:`forge.Revision` from the combined parameters of multiple functions"""
     import forge
 
-    # Use forge.copy to create a revision for each template function
-    revisions = [forge.copy(func) for func in template_functions]
-
-    # Combine the parameters of all revisions into a single revision
-    fparams = [list(rev.signature.parameters.values()) for rev in revisions]
-    return forge.sign(*list(chain.from_iterable(fparams)))
+    fparams = {}
+    for func in template_functions:
+        fparams.update(forge.copy(func).signature.parameters)
+    return forge.sign(*fparams.values())
 
 
 # Param template that's added to every function signature by default
diff --git a/pyinaturalist/node_api.py b/pyinaturalist/node_api.py
index c36fc58..30a619f 100644
--- a/pyinaturalist/node_api.py
+++ b/pyinaturalist/node_api.py
@@ -37,9 +37,7 @@ from pyinaturalist.response_format import (
     convert_all_place_coordinates,
     convert_all_timestamps,
     convert_observation_timestamps,
-    flatten_nested_params,
     format_histogram,
-    format_taxon,
 )
 
 logger = getLogger(__name__)
@@ -309,9 +307,9 @@ def get_observations(**params) -> JsonResponse:
 
         Get basic info for observations in response:
 
-        >>> from pyinaturalist.response_format import format_observation
+        >>> from pyinaturalist.response_utils import format_observation_str
         >>> for obs in response['results']:
-        >>>     print(format_observation(obs))
+        >>>     print(format_observation_str(obs))
         '[57754375] Species: Danaus plexippus (Monarch) observed by samroom on 2020-08-27 at Railway Ave, Wilcox, SK'
         '[57707611] Species: Danaus plexippus (Monarch) observed by ingridt3 on 2020-08-26 at Michener Dr, Regina, SK'
 
@@ -391,7 +389,7 @@ def get_geojson_observations(properties: List[str] = None, **params) -> JsonResp
     params['page'] = 'all'
     response = get_observations(**params)
     return as_geojson_feature_collection(
-        (flatten_nested_params(obs) for obs in response['results']),
+        response['results'],
         properties=properties if properties is not None else DEFAULT_OBSERVATION_ATTRS,
     )
 
@@ -762,7 +760,7 @@ def get_taxa_by_id(taxon_id: MultiInt, user_agent: str = None) -> JsonResponse:
     return taxa
 
 
-@document_request_params([docs._taxon_params, docs._minify])
+@document_request_params([docs._taxon_params])
 def get_taxa_autocomplete(**params) -> JsonResponse:
     """Given a query string, returns taxa with names starting with the search term
 
@@ -773,22 +771,28 @@ def get_taxa_autocomplete(**params) -> JsonResponse:
 
     Example:
 
+        Get just the name of the first matching taxon:
+
         >>> response = get_taxa_autocomplete(q='vespi')
-        >>> first_result = response['results'][0]
-        >>> print(first_result['rank'], first_result['name'])
-        'family Vespidae'
+        >>> print(response['results'][0]['name'])
+        'Vespidae'
+
+        Get basic info for taxa in response:
+
+        >>> from pyinaturalist.response_utils import format_taxon_str
+        >>> for taxon in response['results']:
+        >>>     print(format_taxon_str(taxon, align=True))
+        >>> # See example output below
 
         .. admonition:: Example Response
             :class: toggle
 
-            .. literalinclude:: ../sample_data/get_taxa_autocomplete.json
-                :language: JSON
+            .. literalinclude:: ../sample_data/get_taxa_autocomplete.py
 
-        .. admonition:: Example Response (with **minify=True**)
+        .. admonition:: Example Response (formatted)
             :class: toggle
 
-            .. literalinclude:: ../sample_data/get_taxa_autocomplete_minified.json
-                :language: JSON
+            .. literalinclude:: ../sample_data/get_taxa_autocomplete_minified.py
 
     Returns:
         Response dict containing taxon records
@@ -796,8 +800,4 @@ def get_taxa_autocomplete(**params) -> JsonResponse:
     params = translate_rank_range(params)
     r = node_api_get('taxa/autocomplete', params=params)
     r.raise_for_status()
-    json_response = r.json()
-
-    if params.get('minify'):
-        json_response['results'] = [format_taxon(t, align=True) for t in json_response['results']]
-    return json_response
+    return r.json()
diff --git a/pyinaturalist/response_format.py b/pyinaturalist/response_format.py
index 797c02a..b045297 100644
--- a/pyinaturalist/response_format.py
+++ b/pyinaturalist/response_format.py
@@ -39,6 +39,7 @@ def as_geojson_feature_collection(
         results: List of results from API response
         properties: Whitelist of specific properties to include
     """
+    results = [flatten_nested_params(obs) for obs in results]
     return {
         'type': 'FeatureCollection',
         'features': [as_geojson_feature(record, properties) for record in results],
@@ -228,7 +229,7 @@ def try_float(value: Any) -> Optional[float]:
         return None
 
 
-# Formatting functions
+# Formatting Functions
 # --------------------
 
 
@@ -250,7 +251,6 @@ def flatten_nested_params(observation: ResponseObject) -> ResponseObject:
     return observation
 
 
-# TODO: Is any error handling necessary here? Responses seem fairly consistent and predictable.
 def format_histogram(response: JsonResponse) -> HistogramResponse:
     """Format a response containing time series data into a single ``{date: value}`` dict"""
     # The inner result object's key will be the name of the interval requested
@@ -262,33 +262,3 @@ def format_histogram(response: JsonResponse) -> HistogramResponse:
         return {int(k): v for k, v in histogram.items()}
     else:
         return {parse_date(k): v for k, v in histogram.items()}
-
-
-def format_observation(observation: ResponseObject) -> str:
-    """Make a condensed summary from basic observation details: what, who, when, where"""
-    taxon_str = format_taxon(observation.get('taxon') or {})
-    location = observation.get('place_guess') or observation.get('location')
-    return (
-        f"[{observation['id']}] {taxon_str} "
-        f"observed by {observation['user']['login']} "
-        f"on {observation['observed_on']} "
-        f'at {location}'
-    )
-
-
-def format_taxon(taxon: ResponseObject, align: bool = False) -> str:
-    """Format a taxon result into a single string containing taxon ID, rank, and name
-    (including common name, if available).
-    """
-    if not taxon:
-        return 'unknown taxon'
-    common_name = taxon.get('preferred_common_name')
-    name = f"{taxon['name']}" + (f' ({common_name})' if common_name else '')
-    rank = taxon['rank'].title()
-
-    # Visually align taxon IDs (< 7 chars) and ranks (< 11 chars)
-    if align:
-        # return '{:>8}: {:>12} {}'.format(taxon['id'], rank, name)
-        return f"{taxon['id']:>8}: {rank:>12} {name}"
-    else:
-        return f'{rank}: {name}'
diff --git a/pyinaturalist/response_utils.py b/pyinaturalist/response_utils.py
new file mode 100644
index 0000000..d38ecc2
--- /dev/null
+++ b/pyinaturalist/response_utils.py
@@ -0,0 +1,61 @@
+"""Some utility functions that can simplify working with response data, but are not directly used in
+any API functions.
+"""
+from copy import deepcopy
+
+from pyinaturalist.constants import ResponseObject
+
+
+def format_observation_str(observation: ResponseObject) -> str:
+    """Make a condensed summary from basic observation details: what, who, when, where"""
+    taxon_str = format_taxon_str(observation.get('taxon') or {})
+    location = observation.get('place_guess') or observation.get('location')
+    return (
+        f"[{observation['id']}] {taxon_str} "
+        f"observed by {observation['user']['login']} "
+        f"on {observation['observed_on']} "
+        f'at {location}'
+    )
+
+
+def format_taxon_str(taxon: ResponseObject, align: bool = False) -> str:
+    """Format a taxon result into a single string containing taxon ID, rank, and name
+    (including common name, if available).
+    """
+    if not taxon:
+        return 'unknown taxon'
+    common_name = taxon.get('preferred_common_name')
+    name = f"{taxon['name']}" + (f' ({common_name})' if common_name else '')
+    rank = taxon['rank'].title()
+
+    # Visually align taxon IDs (< 7 chars) and ranks (< 11 chars)
+    if align:
+        return f"{taxon['id']:>8}: {rank:>12} {name}"
+    else:
+        return f'{rank}: {name}'
+
+
+def simplify_observation(obs):
+    """Simplify some nested data structures in an observation record"""
+    # Reduce annotations to IDs and values
+    obs = deepcopy(obs)
+    obs['annotations'] = [
+        (a['controlled_attribute_id'], a['controlled_value_id']) for a in obs['annotations']
+    ]
+
+    # Reduce identifications to just a list of identification IDs and taxon IDs
+    obs['identifications'] = [(i['id'], i['taxon_id']) for i in obs['identifications']]
+    obs['non_owner_ids'] = [(i['id'], i['taxon_id']) for i in obs['non_owner_ids']]
+
+    # Reduce comments to usernames and comment text
+    obs['comments'] = [(c['user']['login'], c['body']) for c in obs['comments']]
+    del obs['observation_photos']
+
+    return obs
+
+
+def to_dataframe(observations):
+    """Normalize observation JSON into a DataFrame"""
+    import pandas as pd
+
+    return pd.json_normalize([simplify_observation(obs) for obs in observations])


Test patch:
diff --git a/test/sample_data/get_taxa_autocomplete_minified.json b/test/sample_data/get_taxa_autocomplete_minified.json
deleted file mode 100644
index 39bcb94..0000000
--- a/test/sample_data/get_taxa_autocomplete_minified.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
-    "total_results": 47,
-    "page": 1,
-    "per_page": 30,
-    "results": [
-        "   52747:       Family Vespidae (Hornets, Paper Wasps, Potter Wasps, and Allies)",
-        "   84738:    Subfamily Vespinae (Hornets and Yellowjackets)",
-        "  131878:      Species Nicrophorus vespillo (Vespillo Burying Beetle)",
-        "  621585:      Species Vespicula trachinoides (Vespicula Waspfish)",
-        "  495392:      Species Vespidae st1",
-        "   70118:      Species Nicrophorus vespilloides (Lesser Vespillo Burying Beetle)",
-        "   92786:        Genus Vespicula",
-        "  646195:        Genus Vespiodes",
-        "  621584:      Species Vespicula cypho",
-        "  621586:      Species Vespicula zollingeri"
-    ]
-}
diff --git a/test/sample_data/get_taxa_autocomplete_minified.py b/test/sample_data/get_taxa_autocomplete_minified.py
new file mode 100644
index 0000000..1652f17
--- /dev/null
+++ b/test/sample_data/get_taxa_autocomplete_minified.py
@@ -0,0 +1,12 @@
+[
+    "   52747:       Family Vespidae (Hornets, Paper Wasps, Potter Wasps, and Allies)",
+    "   84738:    Subfamily Vespinae (Hornets and Yellowjackets)",
+    "  131878:      Species Nicrophorus vespillo (Vespillo Burying Beetle)",
+    "  621585:      Species Vespicula trachinoides (Vespicula Waspfish)",
+    "  495392:      Species Vespidae st1",
+    "   70118:      Species Nicrophorus vespilloides (Lesser Vespillo Burying Beetle)",
+    "   92786:        Genus Vespicula",
+    "  646195:        Genus Vespiodes",
+    "  621584:      Species Vespicula cypho",
+    "  621586:      Species Vespicula zollingeri",
+]
diff --git a/test/test_node_api.py b/test/test_node_api.py
index 1f6bdf9..ede53d7 100644
--- a/test/test_node_api.py
+++ b/test/test_node_api.py
@@ -2,7 +2,6 @@ import os
 import pytest
 from datetime import datetime
 from dateutil.tz import tzoffset, tzutc
-from pprint import pprint
 from unittest.mock import patch
 from urllib.parse import urlencode
 
@@ -554,32 +553,6 @@ def test_get_taxa_autocomplete(requests_mock):
     assert len(first_result['ancestor_ids']) == 11
 
 
-# Test usage of format_taxon() with get_taxa_autocomplete()
-def test_get_taxa_autocomplete_minified(requests_mock):
-    requests_mock.get(
-        f'{API_V1_BASE_URL}/taxa/autocomplete',
-        json=load_sample_data('get_taxa_autocomplete.json'),
-        status_code=200,
-    )
-
-    expected_results = [
-        '   52747:       Family Vespidae (Hornets, Paper Wasps, Potter Wasps, and Allies)',
-        '   84738:    Subfamily Vespinae (Hornets and Yellowjackets)',
-        '  131878:      Species Nicrophorus vespillo (Vespillo Burying Beetle)',
-        '  621585:      Species Vespicula trachinoides (Vespicula Waspfish)',
-        '  495392:      Species Vespidae st1',
-        '   70118:      Species Nicrophorus vespilloides (Lesser Vespillo Burying Beetle)',
-        '   92786:        Genus Vespicula',
-        '  646195:        Genus Vespiodes',
-        '  621584:      Species Vespicula cypho',
-        '  621586:      Species Vespicula zollingeri',
-    ]
-
-    response = get_taxa_autocomplete(q='vespi', minify=True)
-    pprint(response['results'])
-    assert response['results'] == expected_results
-
-
 @patch.dict(os.environ, {'GITHUB_REF': 'refs/heads/master'}, clear=True)
 def test_user_agent(requests_mock):
     requests_mock.get(
diff --git a/test/test_response_format.py b/test/test_response_format.py
index 6e20b53..49d0a3f 100644
--- a/test/test_response_format.py
+++ b/test/test_response_format.py
@@ -6,8 +6,6 @@ from pyinaturalist.response_format import (
     convert_observation_timestamps,
     convert_offset,
     format_histogram,
-    format_observation,
-    format_taxon,
 )
 from test.conftest import load_sample_data
 
@@ -28,30 +26,6 @@ def test_format_histogram__int_keys():
     assert all([isinstance(v, int) for v in histogram.values()])
 
 
-def test_format_observation():
-    observation = load_sample_data('get_observation.json')['results'][0]
-    assert format_observation(observation) == (
-        '[16227955] Species: Lixus bardanae '
-        'observed by niconoe on 2018-09-05 at 54 rue des Badauds'
-    )
-
-
-def test_format_taxon__with_common_name():
-    taxon = load_sample_data('get_taxa.json')['results'][0]
-    assert (
-        format_taxon(taxon) == 'Species: Nicrophorus vespilloides (Lesser Vespillo Burying Beetle)'
-    )
-
-
-def test_format_taxon__without_common_name():
-    taxon = load_sample_data('get_taxa.json')['results'][2]
-    assert format_taxon(taxon) == 'Species: Temnostoma vespiforme'
-
-
-def test_format_taxon__invalid():
-    assert format_taxon(None) == 'unknown taxon'
-
-
 @pytest.mark.parametrize(
     'observed_on, created_at, tz_offset, tz_name, expected_observed, expected_created',
     [
diff --git a/test/test_response_utils.py b/test/test_response_utils.py
new file mode 100644
index 0000000..ded40fb
--- /dev/null
+++ b/test/test_response_utils.py
@@ -0,0 +1,36 @@
+from pyinaturalist.response_utils import (
+    format_observation_str,
+    format_taxon_str,
+    simplify_observation,
+)
+from test.conftest import load_sample_data
+
+
+def test_format_observation():
+    observation = load_sample_data('get_observation.json')['results'][0]
+    assert format_observation_str(observation) == (
+        '[16227955] Species: Lixus bardanae '
+        'observed by niconoe on 2018-09-05 at 54 rue des Badauds'
+    )
+
+
+def test_format_taxon__with_common_name():
+    taxon = load_sample_data('get_taxa.json')['results'][0]
+    expected_str = 'Species: Nicrophorus vespilloides (Lesser Vespillo Burying Beetle)'
+    assert format_taxon_str(taxon) == expected_str
+
+
+def test_format_taxon__without_common_name():
+    taxon = load_sample_data('get_taxa.json')['results'][2]
+    assert format_taxon_str(taxon) == 'Species: Temnostoma vespiforme'
+
+
+def test_format_taxon__invalid():
+    assert format_taxon_str(None) == 'unknown taxon'
+
+
+def test_simplify_observation():
+    observation = load_sample_data('get_observation.json')['results'][0]
+    simplified_obs = simplify_observation(observation)
+    # Not much worth testing here, just make sure it returns something that can be formatted
+    assert format_observation_str(simplified_obs)