Repository: niconoe/pyinaturalist (python)

Issue:
Parse timestamps into python datetime objects
I just realized that observation timestamps (`observed_on_string`) aren't in a consistent format. This field may come directly from user-submitted photo metadata. The main issue here is timezone offset. 

Some timestamps declare this explicitly, for example: `'Sat Sep 26 2020 12:09:51 GMT-0700 (PDT)'`
While others only have the timezone code: `'2020-09-27 9:20:02 AM PDT'`
python-dateutil is able to handle most other variations, but can't automatically convert a timezone code to a timezone offset.

Because of this, I think it would be useful to handle this in pyinaturalist observation endpoints and any other endpoints that return timestamps. Otherwise, any client code that wants to make use of observation time information would have to handle this. There is a separate `time_zone_offset` field that contains an offset string like `'-08:00'`, which could be parsed and added to a timezone-unaware datetime object.

- [x] Parse timestamps
- [x] Parse and apply timezone offsets
- [x] Convert observation `created_at`
- [x] Convert observation `observed_on`
- [x] Convert observation field `created_at`
- [x] Convert observation field `observed_on`
- [x] Convert project `created_at`
- [x] Convert project `updated_at`
- [x] Convert taxon `created_at`
- [x] Convert taxon `updated_at`
- [x] Unit tests

Pull request description:
Convert datetime fields in Observation, Observation Field, Taxon, and Project responses
Closes #81 

Patch:
diff --git a/pyinaturalist/constants.py b/pyinaturalist/constants.py
index 529ba8d..fb8fa16 100644
--- a/pyinaturalist/constants.py
+++ b/pyinaturalist/constants.py
@@ -22,6 +22,7 @@ JsonResponse = Dict[str, Any]
 ListResponse = List[Dict[str, Any]]
 ObsFieldValues = Union[Dict, List[Dict]]
 RequestParams = Dict[str, Any]
+ResponseObject = Dict[str, Any]
 MultiInt = Union[int, List[int]]
 MultiStr = Union[str, List[str]]
 TemplateFunction = Any  # Cannot use Callable/Protocol, as they will not allow a mix of signatures
diff --git a/pyinaturalist/node_api.py b/pyinaturalist/node_api.py
index 1b967b5..453e492 100644
--- a/pyinaturalist/node_api.py
+++ b/pyinaturalist/node_api.py
@@ -39,7 +39,10 @@ from pyinaturalist.request_params import (
 )
 from pyinaturalist.response_format import (
     as_geojson_feature_collection,
-    convert_location_to_float,
+    convert_all_coordinates,
+    convert_all_place_coordinates,
+    convert_all_timestamps,
+    convert_observation_timestamps,
     flatten_nested_params,
     format_taxon,
 )
@@ -167,7 +170,7 @@ def get_observation(observation_id: int, user_agent: str = None) -> JsonResponse
 
     r = get_observations(id=observation_id, user_agent=user_agent)
     if r['results']:
-        return r['results'][0]
+        return convert_observation_timestamps(r['results'][0])
 
     raise ObservationNotFound()
 
@@ -180,9 +183,10 @@ def get_observations(user_agent: str = None, **params) -> JsonResponse:
 
     Example:
 
-        >>> # Get observations of Monarch butterflies with photos + public location info,
-        >>> # on a specific date in the provice of Saskatchewan, CA
-        >>> observations = get_observations(
+        Get observations of Monarch butterflies with photos + public location info,
+        on a specific date in the provice of Saskatchewan, CA:
+
+        >>> response = get_observations(
         >>>     taxon_name='Danaus plexippus',
         >>>     created_on='2020-08-27',
         >>>     photos=True,
@@ -191,6 +195,14 @@ def get_observations(user_agent: str = None, **params) -> JsonResponse:
         >>>     place_id=7953,
         >>> )
 
+        Get basic info for observations in response:
+
+        >>> from pyinaturalist.response_format import format_observation
+        >>> for obs in response['results']:
+        >>>     print(format_observation(obs))
+        '[57754375] Species: Danaus plexippus (Monarch) observed by samroom on 2020-08-27 at Railway Ave, Wilcox, SK'
+        '[57707611] Species: Danaus plexippus (Monarch) observed by ingridt3 on 2020-08-26 at Michener Dr, Regina, SK'
+
         .. admonition:: Example Response
             :class: toggle
 
@@ -202,7 +214,13 @@ def get_observations(user_agent: str = None, **params) -> JsonResponse:
     """
     validate_multiple_choice_param(params, 'order_by', NODE_OBS_ORDER_BY_PROPERTIES)
     r = make_inaturalist_api_get_call('observations', params=params, user_agent=user_agent)
-    return r.json()
+    r.raise_for_status()
+
+    observations = r.json()
+    observations['results'] = convert_all_coordinates(observations['results'])
+    observations['results'] = convert_all_timestamps(observations['results'])
+
+    return observations
 
 
 # TODO: Should use a requests Session here for several subsequent requests
@@ -358,17 +376,6 @@ def get_geojson_observations(properties: List[str] = None, **params) -> JsonResp
 # Places
 # --------------------
 
-# TODO: Use updated sample responses with converted coords
-
-
-def _convert_all_locations_to_float(response):
-    """ Convert locations for both standard (curated) and community-contributed places to floats """
-    response['results'] = {
-        'standard': convert_location_to_float(response['results'].get('standard')),
-        'community': convert_location_to_float(response['results'].get('community')),
-    }
-    return response
-
 
 def get_places_by_id(place_id: MultiInt, user_agent: str = None) -> JsonResponse:
     """
@@ -401,7 +408,7 @@ def get_places_by_id(place_id: MultiInt, user_agent: str = None) -> JsonResponse
 
     # Convert coordinates to floats
     response = r.json()
-    response['results'] = convert_location_to_float(response['results'])
+    response['results'] = convert_all_coordinates(response['results'])
     return response
 
 
@@ -417,7 +424,7 @@ def get_places_nearby(user_agent: str = None, **params) -> JsonResponse:
         >>> bounding_box = (150.0, -50.0, -149.999, -49.999)
         >>> response = get_places_nearby(*bounding_box)
 
-        Show basic info for 'standard' places in response:
+        Show basic info for standard (curated) places in response:
 
         >>> print({p['id']: p['name'] for p in  response['results']['standard']})
         {
@@ -433,7 +440,7 @@ def get_places_nearby(user_agent: str = None, **params) -> JsonResponse:
             10316: 'United States Minor Outlying Islands',
         }
 
-        Show basic info for 'community' places in response:
+        Show basic info for community-contributed places in response:
 
         >>> print({p['id']: p['name'] for p in  response['results']['community']})
         {
@@ -460,7 +467,7 @@ def get_places_nearby(user_agent: str = None, **params) -> JsonResponse:
     """
     r = make_inaturalist_api_get_call('places/nearby', params=params, user_agent=user_agent)
     r.raise_for_status()
-    return _convert_all_locations_to_float(r.json())
+    return convert_all_place_coordinates(r.json())
 
 
 def get_places_autocomplete(q: str, user_agent: str = None) -> JsonResponse:
@@ -495,7 +502,7 @@ def get_places_autocomplete(q: str, user_agent: str = None) -> JsonResponse:
 
     # Convert coordinates to floats
     response = r.json()
-    response['results'] = convert_location_to_float(response['results'])
+    response['results'] = convert_all_coordinates(response['results'])
     return response
 
 
@@ -545,9 +552,9 @@ def get_projects(user_agent: str = None, **params) -> JsonResponse:
     r = make_inaturalist_api_get_call('projects', params=params, user_agent=user_agent)
     r.raise_for_status()
 
-    # Convert coordinates to floats
     response = r.json()
-    response['results'] = convert_location_to_float(response['results'])
+    response['results'] = convert_all_coordinates(response['results'])
+    response['results'] = convert_all_timestamps(response['results'])
     return response
 
 
@@ -584,9 +591,9 @@ def get_projects_by_id(
     )
     r.raise_for_status()
 
-    # Convert coordinates to floats
     response = r.json()
-    response['results'] = convert_location_to_float(response['results'])
+    response['results'] = convert_all_coordinates(response['results'])
+    response['results'] = convert_all_timestamps(response['results'])
     return response
 
 
@@ -618,7 +625,10 @@ def get_taxa(user_agent: str = None, **params) -> JsonResponse:
     params = translate_rank_range(params)
     r = make_inaturalist_api_get_call('taxa', params=params, user_agent=user_agent)
     r.raise_for_status()
-    return r.json()
+
+    taxa = r.json()
+    taxa['results'] = convert_all_timestamps(taxa['results'])
+    return taxa
 
 
 def get_taxa_by_id(taxon_id: MultiInt, user_agent: str = None) -> JsonResponse:
@@ -652,7 +662,10 @@ def get_taxa_by_id(taxon_id: MultiInt, user_agent: str = None) -> JsonResponse:
     """
     r = make_inaturalist_api_get_call('taxa', ids=taxon_id, user_agent=user_agent)
     r.raise_for_status()
-    return r.json()
+
+    taxa = r.json()
+    taxa['results'] = convert_all_timestamps(taxa['results'])
+    return taxa
 
 
 @document_request_params([docs._taxon_params, docs._minify])
@@ -692,5 +705,5 @@ def get_taxa_autocomplete(user_agent: str = None, **params) -> JsonResponse:
     json_response = r.json()
 
     if params.get('minify'):
-        json_response['results'] = [format_taxon(t) for t in json_response['results']]
+        json_response['results'] = [format_taxon(t, align=True) for t in json_response['results']]
     return json_response
diff --git a/pyinaturalist/response_format.py b/pyinaturalist/response_format.py
index c2818c5..58ac6fe 100644
--- a/pyinaturalist/response_format.py
+++ b/pyinaturalist/response_format.py
@@ -1,9 +1,33 @@
 """ Helper functions for formatting API responses """
+from datetime import datetime, timedelta
+from dateutil.parser import parse as parse_date
+from dateutil.parser._parser import UnknownTimezoneWarning
+from dateutil.tz import tzoffset
+from logging import getLogger
 from typing import Any, Dict, Iterable, List, Optional
+from warnings import catch_warnings, simplefilter
+
+from pyinaturalist.constants import JsonResponse, ResponseObject
+
+GENERIC_TIME_FIELDS = ('created_at', 'updated_at')
+OBSERVATION_TIME_FIELDS = (
+    'created_at_details',
+    'created_time_zone',
+    'observed_on',
+    'observed_on_details',
+    'observed_on_string',
+    'observed_time_zone',
+    'time_zone_offset',
+)
+logger = getLogger(__name__)
+
+
+# GeoJSON conversion
+# --------------------
 
 
 def as_geojson_feature_collection(
-    results: Iterable[Dict[str, Any]], properties: List[str] = None
+    results: Iterable[ResponseObject], properties: List[str] = None
 ) -> Dict[str, Any]:
     """
     Convert results from an API response into a
@@ -16,12 +40,12 @@ def as_geojson_feature_collection(
         properties: Whitelist of specific properties to include
     """
     return {
-        "type": "FeatureCollection",
-        "features": [as_geojson_feature(record, properties) for record in results],
+        'type': 'FeatureCollection',
+        'features': [as_geojson_feature(record, properties) for record in results],
     }
 
 
-def as_geojson_feature(result: Dict[str, Any], properties: List[str] = None) -> Dict[str, Any]:
+def as_geojson_feature(result: ResponseObject, properties: List[str] = None) -> ResponseObject:
     """
     Convert an individual response item to a geojson Feature object, optionally with specific
     response properties included.
@@ -30,73 +54,227 @@ def as_geojson_feature(result: Dict[str, Any], properties: List[str] = None) ->
         result: A single response item
         properties: Whitelist of specific properties to include
     """
-    result["geojson"]["coordinates"] = [float(i) for i in result["geojson"]["coordinates"]]
+    result['geojson']['coordinates'] = [float(i) for i in result['geojson']['coordinates']]
     return {
-        "type": "Feature",
-        "geometry": result["geojson"],
-        "properties": {k: result.get(k) for k in properties or []},
+        'type': 'Feature',
+        'geometry': result['geojson'],
+        'properties': {k: result.get(k) for k in properties or []},
     }
 
 
-def flatten_nested_params(observation: Dict[str, Any]) -> Dict[str, Any]:
-    """Extract some nested observation properties to include at the top level;
-     this makes it easier to specify these as properties for
-     :py:func:`.as_as_geojson_feature_collection`.
+# Wrapper functions to apply conversions to all response objects
+# --------------------
+
+
+def convert_all_coordinates(results: List[ResponseObject]) -> List[ResponseObject]:
+    """Convert coordinate pairs in response items from strings to floats, if valid
 
     Args:
-        observation: A single observation result
+        results: Results from API response; expects coordinates in either 'location' key or
+            'latitude' and 'longitude' keys
     """
-    taxon = observation.get("taxon", {})
-    photos = observation.get("photos", [{}])
-    observation["taxon_id"] = taxon.get("id")
-    observation["taxon_name"] = taxon.get("name")
-    observation["taxon_rank"] = taxon.get("rank")
-    observation["preferred_common_name"] = taxon.get("preferred_common_name")
-    observation["photo_url"] = photos[0].get("url")
-    return observation
+    results = [convert_lat_long(result) for result in results]
+    results = [convert_location(result) for result in results]
+    return results
 
 
-def format_taxon(taxon: Dict) -> str:
-    """Format a taxon result into a single string containing taxon ID, rank, and name
-    (including common name, if available).
-    """
-    # Visually align taxon IDs (< 7 chars) and ranks (< 11 chars)
-    common = taxon.get("preferred_common_name")
-    return "{:>8}: {:>12} {}{}".format(
-        taxon["id"],
-        taxon["rank"].title(),
-        taxon["name"],
-        " ({})".format(common) if common else "",
-    )
+def convert_all_place_coordinates(response: JsonResponse) -> JsonResponse:
+    """ Convert locations for both standard and community-contributed places to floats """
+    response['results'] = {
+        'standard': convert_all_coordinates(response['results'].get('standard')),
+        'community': convert_all_coordinates(response['results'].get('community')),
+    }
+    return response
 
 
-def convert_location_to_float(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    """Convert coordinate pairs in response items from strings to floats, if valid.
+def convert_all_timestamps(results: List[ResponseObject]) -> List[ResponseObject]:
+    """Replace all date/time info with datetime objects, where possible"""
+    results = [convert_generic_timestamps(result) for result in results]
+    results = [convert_observation_timestamps(result) for result in results]
+    return results
 
-    Args:
-        results: Results from API response; expects coordinates in the "location" key
+
+# Conversion functions
+# --------------------
+
+
+def convert_lat_long(result: ResponseObject) -> ResponseObject:
+    """Convert a coordinate pair in a response item from strings to floats, if valid"""
+    if 'latitude' in result and 'longitude' in result:
+        result['latitude'] = try_float(result['latitude'])
+        result['longitude'] = try_float(result['longitude'])
+    return result
+
+
+def convert_location(result: ResponseObject):
+    """Convert a coordinate pairs in a response item from strings to floats, if valid"""
+    if ',' in (result.get('location') or ''):
+        result['location'] = [try_float(coord) for coord in result['location'].split(',')]
+    return result
+
+
+def convert_generic_timestamps(result: ResponseObject) -> ResponseObject:
+    """Replace generic created/updated info that's returned by multiple endpoints.
+    **Note:** Compared to observation timestamps, these are generally more reliable. These seem to
+    be consistently in ISO 8601 format.
     """
-    for result in results or []:
-        if "," in (result["location"] or ""):
-            result["location"] = [convert_float(coord) for coord in result["location"].split(",")]
-    return results
+    for field in GENERIC_TIME_FIELDS:
+        datetime_obj = try_datetime(result.get(field, ''))
+        if datetime_obj:
+            result[field] = datetime_obj
+    return result
 
 
-def convert_lat_long_to_float(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    """Convert coordinate pairs in response items from strings to floats, if valid
+def convert_observation_timestamps(result: ResponseObject) -> ResponseObject:
+    """Replace observation date/time info with datetime objects"""
+    if 'created_at_details' not in result and 'observed_on_string' not in result:
+        return result
+
+    observation = result.copy()
+    tz_offset = observation.get('time_zone_offset', '')
+    tz_name = observation.get('observed_time_zone')
+
+    created_datetime = observation.get('created_at')
+    if not isinstance(created_datetime, datetime):
+        created_datetime = try_datetime(observation.get('created_at_details', {}).get('date'))
+        created_datetime = convert_offset(created_datetime, tz_offset, tz_name)
+
+    # Ignore any timezone info in observed_on timestamp; offset field is more reliable
+    observed_datetime = try_datetime(observation.get('observed_on_string', ''), ignoretz=True)
+    observed_datetime = convert_offset(observed_datetime, tz_offset, tz_name)
+
+    # If valid, add the datetime objects and remove all other redundant date/time fields
+    if created_datetime and observed_datetime:
+        for field in OBSERVATION_TIME_FIELDS:
+            observation.pop(field, None)
+    observation['created_at'] = created_datetime
+    observation['observed_on'] = observed_datetime
+
+    return observation
+
+
+def convert_offset(
+    datetime_obj: Optional[datetime], tz_offset: str = None, tz_name: str = None
+) -> Optional[datetime]:
+    """Use timezone offset info to replace a datetime's tzinfo"""
+    if not datetime_obj or not tz_offset:
+        return datetime_obj
+
+    try:
+        offset = parse_offset(tz_offset, tz_name)
+        return datetime_obj.replace(tzinfo=offset)
+    except (AttributeError, TypeError, ValueError) as e:
+        logger.warning(f'Could not parse offset: {tz_offset}: {str(e)}')
+        return None
+
+
+def parse_offset(tz_offset: str, tz_name: str = None) -> tzoffset:
+    """Convert a timezone offset string to a tzoffset object, accounting for some common variations
+    in format
+
+    Examples:
+
+        >>> parse_offset('GMT-08:00', 'PST')
+        tzoffset('PST', -28800)
+        >>> parse_offset('-06:00')
+        tzoffset(None, -21600)
+        >>> parse_offset('+05:30')
+        tzoffset(None, 19800)
+        >>> parse_offset('0530')
+        tzoffset(None, 19800)
 
-    Args:
-        results: Results from API response; expects coordinates in "latitude" and "longitude" keys
     """
-    for result in results:
-        result["latitude"] = convert_float(result["latitude"])
-        result["longitude"] = convert_float(result["longitude"])
-    return results
 
+    def remove_prefixes(text, prefixes):
+        for prefix in prefixes:
+            if text.startswith(prefix):
+                text = text[len(prefix) :]  # noqa  # black and flake8 fight over this one
+        return text
+
+    # Parse hours, minutes, and sign from offset string; account for either `hh:mm` or `hhmm` format
+    tz_offset = remove_prefixes(tz_offset, ['GMT', 'UTC']).strip()
+    multiplier = -1 if tz_offset.startswith('-') else 1
+    tz_offset = tz_offset.lstrip('+-')
+    if ':' in tz_offset:
+        hours, minutes = tz_offset.split(':')
+    else:
+        hours, minutes = tz_offset[:2], tz_offset[2:]
+
+    # Convert to a timezone offset in +/- seconds
+    delta = timedelta(hours=int(hours), minutes=int(minutes))
+    return tzoffset(tz_name, delta.total_seconds() * multiplier)
 
-def convert_float(value: Any) -> Optional[float]:
+
+def try_datetime(timestamp: str, **kwargs) -> Optional[datetime]:
+    """ Parse a timestamp string into a datetime, if valid; return ``None`` otherwise """
+    if isinstance(timestamp, datetime):
+        return timestamp
+
+    try:
+        # Suppress UnknownTimezoneWarning
+        with catch_warnings():
+            simplefilter('ignore', category=UnknownTimezoneWarning)
+            return parse_date(timestamp, **kwargs)
+    except (AttributeError, TypeError, ValueError) as e:
+        logger.warning(f'Could not parse timestamp: {timestamp}: {str(e)}')
+        return None
+
+
+def try_float(value: Any) -> Optional[float]:
     """ Convert a value to a float, if valid; return ``None`` otherwise """
     try:
         return float(value)
     except (TypeError, ValueError):
         return None
+
+
+# Formatting functions
+# --------------------
+
+
+def flatten_nested_params(observation: ResponseObject) -> ResponseObject:
+    """Extract some nested observation properties to include at the top level;
+     this makes it easier to specify these as properties for
+     :py:func:`.as_as_geojson_feature_collection`.
+
+    Args:
+        observation: A single observation result
+    """
+    taxon = observation.get('taxon', {})
+    photos = observation.get('photos', [{}])
+    observation['taxon_id'] = taxon.get('id')
+    observation['taxon_name'] = taxon.get('name')
+    observation['taxon_rank'] = taxon.get('rank')
+    observation['taxon_common_name'] = taxon.get('preferred_common_name')
+    observation['photo_url'] = photos[0].get('url')
+    return observation
+
+
+def format_observation(observation: ResponseObject) -> str:
+    """Make a condensed summary from basic observation details: what, who, when, where"""
+    taxon_str = format_taxon(observation.get('taxon') or {})
+    location = observation.get('place_guess') or observation.get('location')
+    return (
+        f"[{observation['id']}] {taxon_str} "
+        f"observed by {observation['user']['login']} "
+        f"on {observation['observed_on']} "
+        f'at {location}'
+    )
+
+
+def format_taxon(taxon: ResponseObject, align: bool = False) -> str:
+    """Format a taxon result into a single string containing taxon ID, rank, and name
+    (including common name, if available).
+    """
+    if not taxon:
+        return 'unknown taxon'
+    common_name = taxon.get('preferred_common_name')
+    name = f"{taxon['name']}" + (f' ({common_name})' if common_name else '')
+    rank = taxon['rank'].title()
+
+    # Visually align taxon IDs (< 7 chars) and ranks (< 11 chars)
+    if align:
+        # return '{:>8}: {:>12} {}'.format(taxon['id'], rank, name)
+        return f"{taxon['id']:>8}: {rank:>12} {name}"
+    else:
+        return f'{rank}: {name}'
diff --git a/pyinaturalist/rest_api.py b/pyinaturalist/rest_api.py
index 7a717db..15a58fd 100644
--- a/pyinaturalist/rest_api.py
+++ b/pyinaturalist/rest_api.py
@@ -33,7 +33,7 @@ from pyinaturalist.request_params import (
     ensure_file_objs,
     validate_multiple_choice_param,
 )
-from pyinaturalist.response_format import convert_lat_long_to_float
+from pyinaturalist.response_format import convert_all_coordinates, convert_all_timestamps
 
 
 @document_request_params(
@@ -108,7 +108,10 @@ def get_observations(user_agent: str = None, **params) -> Union[List, str]:
     )
 
     if response_format == 'json':
-        return convert_lat_long_to_float(response.json())
+        observations = response.json()
+        observations = convert_all_coordinates(observations)
+        observations = convert_all_timestamps(observations)
+        return observations
     else:
         return response.text
 
@@ -141,7 +144,11 @@ def get_observation_fields(user_agent: str = None, **params) -> ListResponse:
         params=params,
         user_agent=user_agent,
     )
-    return response.json()
+    response.raise_for_status()
+
+    obs_fields = response.json()
+    obs_fields = convert_all_timestamps(obs_fields)
+    return obs_fields
 
 
 @document_request_params([docs._search_query])
@@ -161,12 +168,12 @@ def get_all_observation_fields(**params) -> ListResponse:
     page = 1
 
     while True:
-        r = get_observation_fields(page=page, **params)
+        response = get_observation_fields(page=page, **params)
 
-        if not r:
+        if not response:
             return results
 
-        results += r
+        results += response
         page += 1
         sleep(THROTTLING_DELAY)
 


Test patch:
diff --git a/test/test_node_api.py b/test/test_node_api.py
index 2c4cbd0..43fceaf 100644
--- a/test/test_node_api.py
+++ b/test/test_node_api.py
@@ -1,5 +1,7 @@
 import os
 import pytest
+from datetime import datetime
+from dateutil.tz import tzoffset, tzutc
 from pprint import pprint
 from unittest.mock import patch
 from urllib.parse import urlencode, urljoin
@@ -106,7 +108,10 @@ def test_get_observations(requests_mock):
     assert observations['total_results'] == 2
     assert len(observations['results']) == 1
     assert first_result['taxon_geoprivacy'] == 'open'
-    assert first_result['observed_on'] == '2020-08-27'
+    assert first_result['created_at'] == datetime(2020, 8, 27, 18, 0, 51, tzinfo=tzutc())
+    assert first_result['observed_on'] == datetime(
+        2020, 8, 27, 8, 57, 22, tzinfo=tzoffset('Etc/UTC', 0)
+    )
     assert first_result['taxon']['id'] == 48662
     assert len(first_result['place_ids']) == 13
 
@@ -304,6 +309,8 @@ def test_get_projects(requests_mock):
     assert first_result['title'] == 'PNW Invasive Plant EDDR'
     assert first_result['is_umbrella'] is False
     assert len(first_result['user_ids']) == 33
+    assert first_result['created_at'] == datetime(2016, 7, 20, 23, 0, 5, tzinfo=tzutc())
+    assert first_result['updated_at'] == datetime(2020, 7, 28, 20, 9, 49, tzinfo=tzutc())
 
 
 def test_get_projects_by_id(requests_mock):
@@ -320,6 +327,8 @@ def test_get_projects_by_id(requests_mock):
     assert first_result['title'] == 'Tucson High Native and Invasive Species Inventory'
     assert first_result['place_id'] == 96103
     assert first_result['location'] == [32.2264416406, -110.9617278383]
+    assert first_result['created_at'] == datetime(2016, 7, 26, 23, 8, 47, tzinfo=tzutc())
+    assert first_result['updated_at'] == datetime(2017, 9, 16, 1, 51, 1, tzinfo=tzutc())
 
 
 # Taxa
@@ -446,7 +455,6 @@ def test_get_taxa_autocomplete_minified(requests_mock):
         '  621586:      Species Vespicula zollingeri',
     ]
 
-
     response = get_taxa_autocomplete(q='vespi', minify=True)
     pprint(response['results'])
     assert response['results'] == expected_results
diff --git a/test/test_request_params.py b/test/test_request_params.py
index 843c050..3fa0492 100644
--- a/test/test_request_params.py
+++ b/test/test_request_params.py
@@ -133,20 +133,20 @@ def test_preprocess_request_params(mock_bool, mock_datetime, mock_list, mock_str
 )
 @patch('pyinaturalist.request_params.translate_rank_range')
 @patch('pyinaturalist.response_format.as_geojson_feature')
-@patch('pyinaturalist.node_api.convert_location_to_float')
-@patch('pyinaturalist.node_api._convert_all_locations_to_float')
+@patch('pyinaturalist.node_api.convert_all_coordinates', side_effect=lambda x: x)
+@patch('pyinaturalist.node_api.convert_all_place_coordinates', side_effect=lambda x: x)
 @patch('pyinaturalist.api_requests.preprocess_request_params')
 @patch('pyinaturalist.api_requests.requests.request')
 def test_all_node_requests_use_param_conversion(
     request,
     preprocess_request_params,
-    convert_all_locations_to_float,
-    convert_location_to_float,
+    convert_all_place_coordinates,
+    convert_all_coordinates,
     as_geojson,
     translate_rank_range,
     http_function,
 ):
-    request().json.return_value = {'total_results': 1, 'results': [{}]}
+    request().json.return_value = {'total_results': 1, 'results': [{'id': 1}]}
     mock_args = get_mock_args_for_signature(http_function)
     http_function(*mock_args)
     assert preprocess_request_params.call_count == 1
@@ -156,15 +156,18 @@ def test_all_node_requests_use_param_conversion(
     'http_function', get_module_http_functions(pyinaturalist.rest_api).values()
 )
 @patch.dict(os.environ, MOCK_CREDS_ENV)
-@patch('pyinaturalist.rest_api.convert_lat_long_to_float')
+@patch('pyinaturalist.rest_api.convert_all_coordinates')
 @patch('pyinaturalist.rest_api.sleep')
 @patch('pyinaturalist.api_requests.preprocess_request_params')
 @patch('pyinaturalist.api_requests.requests.request')
 def test_all_rest_requests_use_param_conversion(
-    request, preprocess_request_params, sleep, convert_lat_long_to_float, http_function
+    request, preprocess_request_params, sleep, convert_all_place_coordinates, http_function
 ):
     # Handle the one API response that returns a list instead of a dict
-    if http_function == pyinaturalist.rest_api.get_all_observation_fields:
+    if http_function in [
+        pyinaturalist.rest_api.get_observation_fields,
+        pyinaturalist.rest_api.get_all_observation_fields,
+    ]:
         request().json.return_value = []
     else:
         request().json.return_value = {
diff --git a/test/test_response_format.py b/test/test_response_format.py
new file mode 100644
index 0000000..8db6b83
--- /dev/null
+++ b/test/test_response_format.py
@@ -0,0 +1,117 @@
+import pytest
+from datetime import datetime
+from dateutil.tz import tzoffset
+
+from pyinaturalist.response_format import (
+    convert_observation_timestamps,
+    convert_offset,
+    format_observation,
+    format_taxon,
+)
+from test.conftest import load_sample_data
+
+
+def test_format_observation():
+    observation = load_sample_data('get_observation.json')['results'][0]
+    assert format_observation(observation) == (
+        '[16227955] Species: Lixus bardanae '
+        'observed by niconoe on 2018-09-05 at 54 rue des Badauds'
+    )
+
+
+def test_format_taxon__with_common_name():
+    taxon = load_sample_data('get_taxa.json')['results'][0]
+    assert (
+        format_taxon(taxon) == 'Species: Nicrophorus vespilloides (Lesser Vespillo Burying Beetle)'
+    )
+
+
+def test_format_taxon__without_common_name():
+    taxon = load_sample_data('get_taxa.json')['results'][2]
+    assert format_taxon(taxon) == 'Species: Temnostoma vespiforme'
+
+
+def test_format_taxon__invalid():
+    assert format_taxon(None) == 'unknown taxon'
+
+
+@pytest.mark.parametrize(
+    'observed_on, created_at, tz_offset, tz_name, expected_observed, expected_created',
+    [
+        (
+            'Sat Sep 26 2020 12:09:51 GMT-0700 (PDT)',
+            None,
+            '-07:00',
+            None,
+            datetime(2020, 9, 26, 12, 9, 51, tzinfo=tzoffset(None, -25200)),
+            None,
+        ),
+        (
+            '2020-09-27 9:20:02 AM PST',
+            '2020-10-01',
+            'GMT-08:00',
+            'PST',
+            datetime(2020, 9, 27, 9, 20, 2, tzinfo=tzoffset('PST', -28800)),
+            datetime(2020, 10, 1, tzinfo=tzoffset('PST', -28800)),
+        ),
+        (
+            'Dec 04 2020 21:00:00',
+            'Dec 10 2020',
+            'UTC +02:00',
+            'EET',
+            datetime(2020, 12, 4, 21, 0, 0, tzinfo=tzoffset('EET', 7200)),
+            datetime(2020, 12, 10, tzinfo=tzoffset('EET', 7200)),
+        ),
+        (
+            None,
+            'Dec 10 2020',
+            'UTC +02:00',
+            'EET',
+            None,
+            datetime(2020, 12, 10, tzinfo=tzoffset('EET', 7200)),
+        ),
+    ],
+)
+def test_convert_observation_timestamps(
+    observed_on, created_at, tz_offset, tz_name, expected_observed, expected_created
+):
+    observation = {
+        'created_at_details': {'date': created_at},
+        'observed_on_string': observed_on,
+        'time_zone_offset': tz_offset,
+        'observed_time_zone': tz_name,
+    }
+    converted = convert_observation_timestamps(observation)
+    assert converted['observed_on'] == expected_observed
+    assert converted['created_at'] == expected_created
+
+
+@pytest.mark.parametrize(
+    'datetime_obj, tz_offset, tz_name, expected_date',
+    [
+        (
+            datetime(2020, 9, 26, 12, 9, 51),
+            '-07:00',
+            None,
+            datetime(2020, 9, 26, 12, 9, 51, tzinfo=tzoffset(None, -25200)),
+        ),
+        (
+            datetime(2020, 9, 27, 9, 20, 2),
+            'GMT-08:00',
+            'PST',
+            datetime(2020, 9, 27, 9, 20, 2, tzinfo=tzoffset('PST', -28800)),
+        ),
+        (
+            datetime(2020, 12, 4, 21, 0, 0),
+            'UTC +02:00',
+            'EET',
+            datetime(2020, 12, 4, 21, 0, 0, tzinfo=tzoffset('EET', 7200)),
+        ),
+    ],
+)
+def test_convert_offset(datetime_obj, tz_offset, tz_name, expected_date):
+    assert convert_offset(datetime_obj, tz_offset, tz_name) == expected_date
+
+
+def test_convert_offset__invalid_offset():
+    assert convert_offset(datetime(2020, 1, 1), 'invalid') is None
diff --git a/test/test_rest_api.py b/test/test_rest_api.py
index 96ee164..11d531d 100644
--- a/test/test_rest_api.py
+++ b/test/test_rest_api.py
@@ -1,5 +1,7 @@
 import pytest
+from copy import deepcopy
 from datetime import datetime, timedelta
+from dateutil.tz import tzutc
 from io import BytesIO
 from unittest.mock import patch
 
@@ -32,22 +34,29 @@ def get_observations_response(response_format):
 @pytest.mark.parametrize('response_format', OBSERVATION_FORMATS)
 def test_get_observations(response_format, requests_mock):
     """ Test all supported observation data formats """
-    response = get_observations_response(response_format)
+    mock_response = get_observations_response(response_format)
     key = 'json' if response_format == 'json' else 'text'
 
     requests_mock.get(
         f'{INAT_BASE_URL}/observations.{response_format}',
         status_code=200,
-        **{key: response},
+        **{key: mock_response},
     )
 
     observations = get_observations(taxon_id=493595, response_format=response_format)
 
-    # Ensure coordinate strings were converted to floats, for JSON format only
+    # For JSON format, ensure type conversions were performed
     if response_format == 'json':
-        response[0]['latitude'] = 50.646894
-        response[0]['longitude'] = 4.360086
-    assert observations == response
+        assert observations[0]['latitude'] == 50.646894
+        assert observations[0]['longitude'] == 4.360086
+        assert observations[0]['created_at'] == datetime(
+            2018, 9, 5, 12, 31, 8, 48000, tzinfo=tzutc()
+        )
+        assert observations[0]['updated_at'] == datetime(
+            2018, 9, 22, 17, 19, 27, 80000, tzinfo=tzutc()
+        )
+    else:
+        assert observations == mock_response
 
 
 @pytest.mark.parametrize('response_format', ['geojson', 'yaml'])
@@ -58,41 +67,46 @@ def test_get_observations__invalid_format(response_format):
 
 def test_get_observation_fields(requests_mock):
     """ get_observation_fields() work as expected (basic use)"""
-
     requests_mock.get(
         f'{INAT_BASE_URL}/observation_fields.json?q=sex&page=2',
         json=PAGE_2_JSON_RESPONSE,
         status_code=200,
     )
+    expected_response = deepcopy(PAGE_2_JSON_RESPONSE)
 
-    obs_fields = get_observation_fields(q='sex', page=2)
-    assert obs_fields == PAGE_2_JSON_RESPONSE
+    response = get_observation_fields(q='sex', page=2)
+    first_result = response[0]
+    assert len(response) == 3
+    assert first_result['id'] == 5
+    assert first_result['datatype'] == 'text'
+    assert first_result['created_at'] == datetime(2012, 1, 23, 8, 12, 0, 138000, tzinfo=tzutc())
+    assert first_result['updated_at'] == datetime(2018, 10, 16, 6, 47, 43, 975000, tzinfo=tzutc())
 
 
 def test_get_all_observation_fields(requests_mock):
     """get_all_observation_fields() is able to paginate, accepts a search query and return correct results"""
-
     requests_mock.get(
         f'{INAT_BASE_URL}/observation_fields.json?q=sex&page=1',
         json=PAGE_1_JSON_RESPONSE,
         status_code=200,
     )
-
     requests_mock.get(
         f'{INAT_BASE_URL}/observation_fields.json?q=sex&page=2',
         json=PAGE_2_JSON_RESPONSE,
         status_code=200,
     )
-
-    page_3_json_response = []
     requests_mock.get(
         f'{INAT_BASE_URL}/observation_fields.json?q=sex&page=3',
-        json=page_3_json_response,
+        json=[],
         status_code=200,
     )
+    expected_response = PAGE_1_JSON_RESPONSE + PAGE_2_JSON_RESPONSE
 
-    all_fields = get_all_observation_fields(q='sex')
-    assert all_fields == PAGE_1_JSON_RESPONSE + PAGE_2_JSON_RESPONSE
+    response = get_all_observation_fields(q='sex')
+    assert len(response) == len(expected_response)
+    first_result = response[0]
+    assert first_result['created_at'] == datetime(2016, 5, 29, 16, 17, 8, 51000, tzinfo=tzutc())
+    assert first_result['updated_at'] == datetime(2018, 1, 1, 1, 17, 56, 7000, tzinfo=tzutc())
 
 
 def test_get_all_observation_fields_noparam(requests_mock):
@@ -113,17 +127,17 @@ def test_put_observation_field_values(requests_mock):
         status_code=200,
     )
 
-    r = put_observation_field_values(
+    response = put_observation_field_values(
         observation_id=18166477,
         observation_field_id=31,  # Animal behavior
         value='fouraging',
         access_token='valid token',
     )
 
-    assert r['id'] == 31
-    assert r['observation_field_id'] == 31
-    assert r['observation_id'] == 18166477
-    assert r['value'] == 'fouraging'
+    assert response['id'] == 31
+    assert response['observation_field_id'] == 31
+    assert response['observation_id'] == 18166477
+    assert response['value'] == 'fouraging'
 
 
 def test_update_observation(requests_mock):
@@ -137,12 +151,12 @@ def test_update_observation(requests_mock):
         'ignore_photos': 1,
         'description': 'updated description v2 !',
     }
-    r = update_observation(17932425, access_token='valid token', **params)
+    response = update_observation(17932425, access_token='valid token', **params)
 
     # If all goes well we got a single element representing the updated observation, enclosed in a list.
-    assert len(r) == 1
-    assert r[0]['id'] == 17932425
-    assert r[0]['description'] == 'updated description v2 !'
+    assert len(response) == 1
+    assert response[0]['id'] == 17932425
+    assert response[0]['description'] == 'updated description v2 !'
 
 
 @patch('pyinaturalist.rest_api.ensure_file_objs')
@@ -202,10 +216,10 @@ def test_create_observation(requests_mock):
         status_code=200,
     )
 
-    r = create_observation(species_guess='Pieris rapae', access_token='valid token')
-    assert len(r) == 1  # We added a single one
-    assert r[0]['latitude'] is None
-    assert r[0]['taxon_id'] == 55626  # Pieris Rapae @ iNaturalist
+    response = create_observation(species_guess='Pieris rapae', access_token='valid token')
+    assert len(response) == 1  # We added a single one
+    assert response[0]['latitude'] is None
+    assert response[0]['taxon_id'] == 55626  # Pieris Rapae @ iNaturalist
 
 
 @patch('pyinaturalist.rest_api.ensure_file_objs')