Repository: niconoe/pyinaturalist
Pull request #193 (2021-06-30 18:58:53)

Model collections
* Use UserList.data for model collections
* Add an 'Observations' model for some operations on a collection of observations
* Code reuse between model collections (currently just Observations, TaxonCounts, and LifeList)
* Move 'id' attr to BaseModel


Patch:
diff --git a/pyinaturalist/converters.py b/pyinaturalist/converters.py
index c9fb487..10c54d2 100644
--- a/pyinaturalist/converters.py
+++ b/pyinaturalist/converters.py
@@ -1,5 +1,4 @@
 """Type conversion utilities"""
-from collections import UserList
 from datetime import date, datetime, timedelta
 from dateutil.parser import UnknownTimezoneWarning  # type: ignore  # (missing from type stubs)
 from dateutil.parser import parse as parse_date
@@ -8,7 +7,7 @@ from io import BytesIO
 from logging import getLogger
 from os.path import abspath, expanduser
 from pathlib import Path
-from typing import IO, Any, Dict, List, Optional, Union
+from typing import IO, Any, Dict, List, MutableSequence, Optional, Union
 from warnings import catch_warnings, simplefilter
 
 from pyinaturalist.constants import (
@@ -246,9 +245,9 @@ def parse_offset(tz_offset: str, tz_name: str = None) -> tzoffset:
     return tzoffset(tz_name, delta.total_seconds() * multiplier)
 
 
-def safe_split(value: Any, delimiter: str = '|') -> List:
+def safe_split(value: Any, delimiter: str = '|') -> List[str]:
     """Split a pipe-(or other token)-delimited string"""
-    return ensure_list(value, convert_csv=True, delimiter=delimiter)
+    return list(ensure_list(value, convert_csv=True, delimiter=delimiter))
 
 
 def strip_empty_values(values: Dict) -> Dict:
@@ -256,11 +255,11 @@ def strip_empty_values(values: Dict) -> Dict:
     return {k: v for k, v in values.items() if v or v in [False, 0, 0.0]}
 
 
-# ?
+# Type conversion functions for collections
 # -------------------
 
 
-def ensure_list(value: Any, convert_csv: bool = False, delimiter: str = ',') -> List[Any]:
+def ensure_list(value: Any, convert_csv: bool = False, delimiter: str = ',') -> MutableSequence[Any]:
     """Convert an object, response, or (optionally) comma-separated string into a list"""
     if not value:
         return []
@@ -269,7 +268,9 @@ def ensure_list(value: Any, convert_csv: bool = False, delimiter: str = ',') ->
     elif convert_csv and isinstance(value, str) and delimiter in value:
         return [s.strip() for s in value.split(delimiter)]
 
-    if isinstance(value, (UserList, list, tuple, set)):
+    if isinstance(value, MutableSequence):
+        return value
+    elif isinstance(value, (tuple, set)):
         return list(value)
     else:
         return [value]
diff --git a/pyinaturalist/formatters.py b/pyinaturalist/formatters.py
index 0157cb8..ae40616 100644
--- a/pyinaturalist/formatters.py
+++ b/pyinaturalist/formatters.py
@@ -17,6 +17,7 @@ from pyinaturalist.converters import ensure_list
 from pyinaturalist.models import (
     Annotation,
     BaseModel,
+    BaseModelCollection,
     Comment,
     ControlledTerm,
     ControlledTermValue,
@@ -125,8 +126,8 @@ def ensure_model_list(values: ResponseOrObjects) -> List[BaseModel]:
     model objects
     """
     values = ensure_list(values)
-    if isinstance(values[0], BaseModel):
-        return values
+    if isinstance(values, BaseModelCollection) or isinstance(values[0], BaseModel):
+        return values  # type: ignore
 
     cls = detect_type(values[0])
     return [cls.from_json(value) for value in values]
diff --git a/pyinaturalist/models/__init__.py b/pyinaturalist/models/__init__.py
index 2eb8f98..abb0852 100644
--- a/pyinaturalist/models/__init__.py
+++ b/pyinaturalist/models/__init__.py
@@ -1,6 +1,5 @@
 """Dataclasses for modeling iNaturalist API response objects"""
 # flake8: noqa: F401
-# TODO: Validators for multiple choice fields?
 from typing import Callable
 from attr import define
 
@@ -18,8 +17,9 @@ from pyinaturalist.models.base import (
 )
 from pyinaturalist.models.lazy_property import LazyProperty, add_lazy_attrs, get_model_fields
 
-# attrs class decorator with the most commonly used options
+# attrs class decorators with the most commonly used options
 define_model: Callable = define(auto_attribs=False, field_transformer=add_lazy_attrs)
+define_model_collection: Callable = define(auto_attribs=False, order=False, slots=False)
 
 # Imported in order of model dependencies
 from pyinaturalist.models.photo import Photo
@@ -43,7 +43,7 @@ from pyinaturalist.models.project import (
     ProjectUser,
 )
 from pyinaturalist.models.observation_field import ObservationField, ObservationFieldValue
-from pyinaturalist.models.observation import Observation
+from pyinaturalist.models.observation import Observation, Observations
 from pyinaturalist.models.search import SearchResult
 
 
diff --git a/pyinaturalist/models/base.py b/pyinaturalist/models/base.py
index 3bc66ed..e81e132 100644
--- a/pyinaturalist/models/base.py
+++ b/pyinaturalist/models/base.py
@@ -37,7 +37,7 @@ datetime_now_attr = field(converter=try_datetime, factory=utcnow)  # type: ignor
 class BaseModel:
     """Base class for data models"""
 
-    partial: bool = field(default=None, repr=False)
+    id: int = kwarg
     temp_attrs: List[str] = []
     headers: Dict[str, str] = {}
 
@@ -78,10 +78,20 @@ class BaseModel:
 # TODO: Better workaround for mypy not accepting subclasses in overridden attributes and methods
 @define(auto_attribs=False, order=False, slots=False)
 class BaseModelCollection(BaseModel, UserList, Generic[T]):
-    """Base class for collections of data models"""
+    """Base class for data model collections. These will behave the same as lists but enable some
+    additional operations on contained items.
+    """
 
     data: List[T] = field(factory=list, init=False, repr=False)
 
+    @classmethod
+    def from_json(cls, value: JsonResponse, **kwargs) -> 'BaseModelCollection':
+        if 'results' in value:
+            value = value['results']
+        if 'data' not in value:
+            value = {'data': value}
+        return super(BaseModelCollection, cls).from_json(value)  # type: ignore
+
     @classmethod
     def from_json_list(cls, value: JsonResponse, **kwargs) -> 'BaseModelCollection':  # type: ignore
         """For model collections, initializing from a list should return an instance of ``cls``
@@ -89,9 +99,6 @@ class BaseModelCollection(BaseModel, UserList, Generic[T]):
         """
         return cls.from_json(value)  # type: ignore
 
-    def __attrs_post_init__(self):
-        self.data = self.taxa
-
 
 # Type aliases
 ModelObjects = Union[BaseModel, BaseModelCollection, Iterable[BaseModel]]
diff --git a/pyinaturalist/models/comment.py b/pyinaturalist/models/comment.py
index 5c086fa..3786d23 100644
--- a/pyinaturalist/models/comment.py
+++ b/pyinaturalist/models/comment.py
@@ -13,7 +13,6 @@ class Comment(BaseModel):
     body: str = kwarg
     created_at: datetime = datetime_now_attr
     hidden: bool = kwarg
-    id: int = kwarg
     uuid: str = kwarg
 
     # Lazy-loaded nested model objects
diff --git a/pyinaturalist/models/controlled_term.py b/pyinaturalist/models/controlled_term.py
index 2c65428..0bdae3f 100644
--- a/pyinaturalist/models/controlled_term.py
+++ b/pyinaturalist/models/controlled_term.py
@@ -52,7 +52,6 @@ class ControlledTermValue(BaseModel):
     """
 
     blocking: bool = kwarg
-    id: int = kwarg
     label: str = kwarg
     ontology_uri: str = kwarg
     uri: str = kwarg
@@ -69,7 +68,6 @@ class ControlledTerm(BaseModel):
     `GET /controlled_terms <https://api.inaturalist.org/v1/docs/#!/Controlled_Terms/get_controlled_terms>`_.
     """
 
-    id: int = kwarg
     is_value: bool = kwarg
     multivalued: bool = kwarg
     label: str = kwarg
diff --git a/pyinaturalist/models/identification.py b/pyinaturalist/models/identification.py
index 3163e3d..a25abb9 100644
--- a/pyinaturalist/models/identification.py
+++ b/pyinaturalist/models/identification.py
@@ -25,7 +25,6 @@ class Identification(BaseModel):
     current_taxon: bool = kwarg
     disagreement: bool = kwarg
     hidden: bool = kwarg
-    id: int = kwarg
     own_observation: bool = kwarg
     previous_observation_taxon_id: int = kwarg
     taxon_change: bool = kwarg  # TODO: confirm type
diff --git a/pyinaturalist/models/life_list.py b/pyinaturalist/models/life_list.py
index e2de1ac..1d9e63f 100644
--- a/pyinaturalist/models/life_list.py
+++ b/pyinaturalist/models/life_list.py
@@ -4,7 +4,14 @@ from typing import List
 from attr import field
 
 from pyinaturalist.constants import JsonResponse
-from pyinaturalist.models import Taxon, TaxonCount, TaxonCounts, define_model, kwarg
+from pyinaturalist.models import (
+    Taxon,
+    TaxonCount,
+    TaxonCounts,
+    define_model,
+    define_model_collection,
+    kwarg,
+)
 
 
 @define_model
@@ -24,12 +31,12 @@ class LifeListTaxon(TaxonCount):
         return f'[{self.id:<8}] {padding} {self.rank.title()} {self.name}: {self.count}'
 
 
-@define_model
+@define_model_collection
 class LifeList(TaxonCounts):
     """A user's life list, based on the schema of ``GET /observations/taxonomy``"""
 
     count_without_taxon: int = field(default=0)
-    taxa: List[LifeListTaxon] = field(factory=list, converter=LifeListTaxon.from_json_list)  # type: ignore
+    data: List[LifeListTaxon] = field(factory=list, converter=LifeListTaxon.from_json_list)  # type: ignore
     user_id: int = kwarg
 
     @classmethod
@@ -38,7 +45,7 @@ class LifeList(TaxonCounts):
         if 'results' in value:
             value = value['results']
 
-        life_list_json = {'taxa': value, 'user_id': user_id, 'count_without_taxon': count_without_taxon}
+        life_list_json = {'data': value, 'user_id': user_id, 'count_without_taxon': count_without_taxon}
         return super(LifeList, cls).from_json(life_list_json)  # type: ignore
 
     def tree(self):
@@ -48,7 +55,7 @@ class LifeList(TaxonCounts):
         Returns:
             :py:class:`rich.tree.Tree`
         """
-        return make_tree(self.taxa)
+        return make_tree(self.data)
 
 
 def make_tree(taxa: List[Taxon]):
diff --git a/pyinaturalist/models/observation.py b/pyinaturalist/models/observation.py
index 7210426..811ac4c 100644
--- a/pyinaturalist/models/observation.py
+++ b/pyinaturalist/models/observation.py
@@ -8,6 +8,7 @@ from pyinaturalist.converters import convert_observation_timestamp
 from pyinaturalist.models import (
     Annotation,
     BaseModel,
+    BaseModelCollection,
     Comment,
     Identification,
     LazyProperty,
@@ -20,6 +21,7 @@ from pyinaturalist.models import (
     coordinate_pair,
     datetime_attr,
     datetime_now_attr,
+    define_model_collection,
     kwarg,
 )
 from pyinaturalist.v1 import get_observation
@@ -39,7 +41,6 @@ class Observation(BaseModel):
     description: str = kwarg
     faves_count: int = kwarg
     geoprivacy: str = kwarg  # Enum
-    id: int = kwarg
     id_please: bool = kwarg
     identifications_count: int = kwarg
     identifications_most_agree: bool = kwarg
@@ -164,3 +165,33 @@ class Observation(BaseModel):
             f'observed on {self.observed_on} by {self.user.login} '
             f'at {self.place_guess or self.location}'
         )
+
+
+@define_model_collection
+class Observations(BaseModelCollection):
+    """A collection of observation records"""
+
+    data: List[Observation] = field(factory=list, converter=Observation.from_json_list)
+
+    @property
+    def identifiers(self) -> List[User]:
+        """Get all unique identifiers"""
+        unique_users = {ident.user.id: ident.user for obs in self.data for ident in obs.identifications}
+        return list(unique_users.values())
+
+    @property
+    def taxa(self) -> List[Taxon]:
+        """Get all unique taxa"""
+        unique_taxa = {obs.taxon.id: obs.taxon for obs in self.data}
+        return list(unique_taxa.values())
+
+    @property
+    def thumbnail_urls(self) -> List[str]:
+        """Get thumbnails for all observation default photos"""
+        return [obs.thumbnail_url for obs in self.data if obs.thumbnail_url]
+
+    @property
+    def observers(self) -> List[User]:
+        """Get all unique observers"""
+        unique_users = {obs.user.id: obs.user for obs in self.data}
+        return list(unique_users.values())
diff --git a/pyinaturalist/models/observation_field.py b/pyinaturalist/models/observation_field.py
index 9d3ee04..399b715 100644
--- a/pyinaturalist/models/observation_field.py
+++ b/pyinaturalist/models/observation_field.py
@@ -38,7 +38,6 @@ class ObservationField(BaseModel):
     created_at: datetime = datetime_now_attr
     datatype: str = kwarg  # Enum
     description: str = kwarg
-    id: int = kwarg
     name: str = kwarg
     updated_at: datetime = datetime_now_attr
     user_id: int = kwarg
@@ -68,7 +67,6 @@ class ObservationFieldValue(BaseModel):
 
     datatype: str = kwarg  # Enum
     field_id: int = kwarg
-    id: int = kwarg
     name: str = kwarg
     taxon_id: int = kwarg
     user_id: int = kwarg
diff --git a/pyinaturalist/models/photo.py b/pyinaturalist/models/photo.py
index 76624c1..48b26b1 100644
--- a/pyinaturalist/models/photo.py
+++ b/pyinaturalist/models/photo.py
@@ -15,7 +15,6 @@ class Photo(BaseModel):
     """
 
     attribution: str = kwarg
-    id: int = kwarg
     license_code: str = field(converter=format_license, default=None)  # Enum
     original_dimensions: Tuple[int, int] = field(converter=format_dimensions, default=(0, 0))
     url: str = kwarg
diff --git a/pyinaturalist/models/place.py b/pyinaturalist/models/place.py
index 97e3794..ef31f02 100644
--- a/pyinaturalist/models/place.py
+++ b/pyinaturalist/models/place.py
@@ -35,7 +35,6 @@ class Place(BaseModel):
     category: str = kwarg  # Either 'standard' or 'community'
     display_name: str = kwarg
     geometry_geojson: GeoJson = field(factory=dict)
-    id: int = kwarg
     location: Coordinates = field(converter=convert_optional_lat_long, default=None)
     name: str = kwarg
     place_type: int = kwarg
diff --git a/pyinaturalist/models/project.py b/pyinaturalist/models/project.py
index 526d428..657b959 100644
--- a/pyinaturalist/models/project.py
+++ b/pyinaturalist/models/project.py
@@ -21,7 +21,6 @@ from pyinaturalist.models.observation_field import ObservationField
 class ProjectObservation(ObservationField):
     """Metadata about an observation that has been added to a project"""
 
-    id: int = kwarg
     preferences: Dict = field(factory=dict)  # Example: {'allows_curator_coordinate_access': True}
     project: Dict = field(factory=dict)  # Example: {'id': 24237}
     user_id: int = kwarg
@@ -80,7 +79,6 @@ class Project(BaseModel):
     hide_title: bool = kwarg
     icon_file_name: str = kwarg
     icon: str = kwarg
-    id: int = kwarg
     is_umbrella: bool = kwarg
     location: Optional[Coordinates] = coordinate_pair
     place_id: int = kwarg
diff --git a/pyinaturalist/models/taxon.py b/pyinaturalist/models/taxon.py
index 5f6e938..b7c32b8 100644
--- a/pyinaturalist/models/taxon.py
+++ b/pyinaturalist/models/taxon.py
@@ -20,6 +20,7 @@ from pyinaturalist.models import (
     User,
     datetime_attr,
     define_model,
+    define_model_collection,
     kwarg,
 )
 from pyinaturalist.v1 import get_taxa_by_id
@@ -74,7 +75,6 @@ class Taxon(BaseModel):
     extinct: bool = kwarg  #: Indicates if the taxon is extinct
     iconic_taxon_id: int = field(default=0)  #: ID of the iconic taxon or taxon "category"
     iconic_taxon_name: str = field(default='unknown')  #: Name of the iconic taxon or taxon "category"
-    id: int = kwarg  #: Taxon ID
     is_active: bool = kwarg  #: Indicates if the taxon is active (and not renamed, moved, etc.)
     listed_taxa_count: int = kwarg  #: Number of listed taxa from this taxon + descendants
     name: str = kwarg  #: Taxon name; contains full scientific name at species level and below
@@ -108,6 +108,7 @@ class Taxon(BaseModel):
     # flag_counts: Dict[str, int] = field(factory=dict)  # {"unresolved": 1, "resolved": 2}
     # min_species_ancestry: str = kwarg  #: Used internally by iNaturalist for Elasticsearch aggregations
     # min_species_taxon_id: int = kwarg
+    # partial: bool = field(default=None, repr=False)
     # photos_locked: bool = kwarg
     # universal_search_rank: int = kwarg
 
@@ -218,36 +219,28 @@ class TaxonCount(Taxon):
         return f'[{self.id}] {self.full_name}: {self.count}'
 
 
-@define_model
+@define_model_collection
 class TaxonCounts(BaseModelCollection):
     """A collection of taxa with an associated counts. Used with
     `GET /observations/species_counts <https://api.inaturalist.org/v1/docs/#!/Observations/get_observations_species_counts>`_.
     as well as :py:class:`.LifeList`.
     """
 
-    taxa: List[TaxonCount] = field(factory=list, converter=TaxonCount.from_json_list)
+    data: List[TaxonCount] = field(factory=list, converter=TaxonCount.from_json_list)
     _taxon_counts: Dict[int, int] = field(default=None, init=False, repr=False)
 
-    @classmethod
-    def from_json(cls, value: JsonResponse, **kwargs) -> 'TaxonCounts':
-        if 'results' in value:
-            value = value['results']
-        if 'taxa' not in value:
-            value = {'taxa': value}
-        return super(TaxonCounts, cls).from_json(value)  # type: ignore
-
     def count(self, taxon_id: int) -> int:
         """Get an observation count for the specified taxon and its descendants, and handle unlisted taxa.
         **Note:** ``-1`` can be used an alias for ``count_without_taxon``.
         """
         # Make and cache an index of taxon IDs and observation counts
         if self._taxon_counts is None:
-            self._taxon_counts = {t.id: t.descendant_obs_count for t in self.taxa}
+            self._taxon_counts = {t.id: t.descendant_obs_count for t in self.data}
             self._taxon_counts[-1] = self.count_without_taxon
         return self._taxon_counts.get(taxon_id, 0)
 
     def __str__(self) -> str:
-        return '\n'.join([str(taxon) for taxon in self.taxa])
+        return '\n'.join([str(taxon) for taxon in self.data])
 
 
 # Since these use Taxon classmethods, they must be added after Taxon is defined
diff --git a/pyinaturalist/models/user.py b/pyinaturalist/models/user.py
index 4d467cf..7d2b3c2 100644
--- a/pyinaturalist/models/user.py
+++ b/pyinaturalist/models/user.py
@@ -17,7 +17,6 @@ class User(BaseModel):
     created_at: datetime = datetime_now_attr
     icon: str = kwarg
     icon_url: str = kwarg
-    id: int = kwarg
     identifications_count: int = kwarg
     journal_posts_count: int = kwarg
     login: str = kwarg


Test patch:
diff --git a/test/test_models.py b/test/test_models.py
index 6d689dc..b3a560c 100644
--- a/test/test_models.py
+++ b/test/test_models.py
@@ -147,14 +147,14 @@ def test_identification__str():
 
 def test_life_list__converters():
     life_list = LifeList.from_json(j_life_list)
-    assert life_list.taxa[0] == life_list[0]
+    assert life_list.data[0] == life_list[0]
     assert len(life_list) == 9
-    assert isinstance(life_list.taxa[0], LifeListTaxon) and life_list.taxa[0].id == 1
+    assert isinstance(life_list.data[0], LifeListTaxon) and life_list.data[0].id == 1
 
 
 def test_life_list__empty():
     life_list = LifeList()
-    life_list.taxa == []
+    life_list.data == []
     life_list._taxon_counts is None
 
 
@@ -215,6 +215,20 @@ def test_observation__thumbnail_url():
     assert obs.thumbnail_url == 'https://static.inaturalist.org/photos/92152429/square.jpg?1598551272'
 
 
+def test_observations():
+    obs_list = Observations.from_json_list(
+        [j_observation_1, j_observation_1, j_observation_2, j_observation_3_ofvs]
+    )
+    assert isinstance(obs_list.identifiers[0], User) and len(obs_list.identifiers) == 6
+    assert isinstance(obs_list.observers[0], User) and len(obs_list.observers) == 3
+    assert isinstance(obs_list.taxa[0], Taxon) and len(obs_list.taxa) == 3
+    assert len(obs_list.thumbnail_urls) == 4
+    assert (
+        obs_list.thumbnail_urls[0]
+        == 'https://static.inaturalist.org/photos/24355315/square.jpeg?1536150664'
+    )
+
+
 # Observation Fields
 # --------------------
 
@@ -486,14 +500,14 @@ def test_taxon__update_from_full_record():
 
 def test_taxon_counts__converters():
     taxon_counts = TaxonCounts.from_json(j_obs_species_counts)
-    assert taxon_counts.taxa[0] == taxon_counts[0]
+    assert taxon_counts.data[0] == taxon_counts[0]
     assert len(taxon_counts) == 9
-    assert isinstance(taxon_counts.taxa[0], TaxonCount) and taxon_counts.taxa[0].count == 31
+    assert isinstance(taxon_counts.data[0], TaxonCount) and taxon_counts.data[0].count == 31
 
 
 def test_taxon_counts__empty():
     taxon_counts = TaxonCounts()
-    assert taxon_counts.taxa == []
+    assert taxon_counts.data == []
 
 
 # Users