Repository: azavea/python-omgeo

Issue:
magicKey specification breaks geocoding
When a `findAddressCandidates` request is made without specifying `magicKey`, the response includes a `Loc_name` attribute for candidates:

```bash
curl  -X GET \
  'https://geocode.arcgis.com/arcgis/rest/services/World/GeocodeServer/findAddressCandidates?f=json&outFields=Loc_name%2CScore%2CMatch_addr%2CAddr_type%2CAddNum%2CStPreDir%2CStPreType%2CStName%2CStType%2CStDir%2CCity%2CSubregion%2CRegionAbbr%2CPostal%2CCountry%2CDisplayX%2CDisplayY&maxLocations=20&singleLine=New+York%2C+USA&sourceCountry=US'
```
```json
{
  "spatialReference": {
    "wkid": 4326,
    "latestWkid": 4326
  },
  "candidates": [
    {
      "address": "New York",
      "location": {
        "x": -74.00712,
        "y": 40.71453
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "New York",
        "Subregion": "New York County",
        "RegionAbbr": "NY",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -74.00712,
        "DisplayY": 40.71453
      },
      "extent": {
        "xmin": -74.25712,
        "ymin": 40.46453,
        "xmax": -73.75712,
        "ymax": 40.96453
      }
    },
    {
      "address": "New York",
      "location": {
        "x": -75.610701447,
        "y": 42.937083979
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "",
        "Subregion": "",
        "RegionAbbr": "NY",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -75.610701447,
        "DisplayY": 42.937083979
      },
      "extent": {
        "xmin": -78.716701447,
        "ymin": 39.831083979,
        "xmax": -72.504701447,
        "ymax": 46.043083979
      }
    },
    {
      "address": "New York",
      "location": {
        "x": -74.00597,
        "y": 40.71427
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "New York",
        "Subregion": "",
        "RegionAbbr": "NY",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -74.00597,
        "DisplayY": 40.71427
      },
      "extent": {
        "xmin": -74.25097,
        "ymin": 40.46927,
        "xmax": -73.76097,
        "ymax": 40.95927
      }
    },
    {
      "address": "nyc, New York",
      "location": {
        "x": -73.96625,
        "y": 40.78343
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "nyc, New York",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "Manhattan",
        "Subregion": "New York County",
        "RegionAbbr": "NY",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -73.96625,
        "DisplayY": 40.78343
      },
      "extent": {
        "xmin": -74.05125,
        "ymin": 40.69843,
        "xmax": -73.88125,
        "ymax": 40.86843
      }
    },
    {
      "address": "New York, Florida",
      "location": {
        "x": -87.20034,
        "y": 30.8382
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Florida",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "New York",
        "Subregion": "Santa Rosa County",
        "RegionAbbr": "FL",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -87.20034,
        "DisplayY": 30.8382
      },
      "extent": {
        "xmin": -87.21034,
        "ymin": 30.8282,
        "xmax": -87.19034,
        "ymax": 30.8482
      }
    },
    {
      "address": "New York, Iowa",
      "location": {
        "x": -93.25527,
        "y": 40.85156
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Iowa",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "New York",
        "Subregion": "Wayne County",
        "RegionAbbr": "IA",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -93.25527,
        "DisplayY": 40.85156
      },
      "extent": {
        "xmin": -93.26527,
        "ymin": 40.84156,
        "xmax": -93.24527,
        "ymax": 40.86156
      }
    },
    {
      "address": "New York, Kentucky",
      "location": {
        "x": -88.97562,
        "y": 36.99013
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Kentucky",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "New York",
        "Subregion": "Ballard County",
        "RegionAbbr": "KY",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -88.97562,
        "DisplayY": 36.99013
      },
      "extent": {
        "xmin": -88.98562,
        "ymin": 36.98013,
        "xmax": -88.96562,
        "ymax": 37.00013
      }
    },
    {
      "address": "New York, Missouri",
      "location": {
        "x": -93.90937,
        "y": 39.68488
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Missouri",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "New York",
        "Subregion": "Caldwell County",
        "RegionAbbr": "MO",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -93.90937,
        "DisplayY": 39.68488
      },
      "extent": {
        "xmin": -93.91937,
        "ymin": 39.67488,
        "xmax": -93.89937,
        "ymax": 39.69488
      }
    },
    {
      "address": "New York, New Mexico",
      "location": {
        "x": -107.52664,
        "y": 35.05945
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, New Mexico",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "New York",
        "Subregion": "Cibola County",
        "RegionAbbr": "NM",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -107.52664,
        "DisplayY": 35.05945
      },
      "extent": {
        "xmin": -107.53664,
        "ymin": 35.04945,
        "xmax": -107.51664,
        "ymax": 35.06945
      }
    },
    {
      "address": "New York, Texas",
      "location": {
        "x": -95.66936,
        "y": 32.16919
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Texas",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "New York",
        "Subregion": "Henderson County",
        "RegionAbbr": "TX",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -95.66936,
        "DisplayY": 32.16919
      },
      "extent": {
        "xmin": -95.67936,
        "ymin": 32.15919,
        "xmax": -95.65936,
        "ymax": 32.17919
      }
    },
    {
      "address": "New York, Nebraska",
      "location": {
        "x": -97.592,
        "y": 40.86807
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Nebraska",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "York",
        "Subregion": "York County",
        "RegionAbbr": "NE",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -97.592,
        "DisplayY": 40.86807
      },
      "extent": {
        "xmin": -97.635,
        "ymin": 40.82507,
        "xmax": -97.549,
        "ymax": 40.91107
      }
    },
    {
      "address": "New York, Wisconsin",
      "location": {
        "x": -88.86511,
        "y": 42.55779
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Wisconsin",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "Clinton",
        "Subregion": "Rock County",
        "RegionAbbr": "WI",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -88.86511,
        "DisplayY": 42.55779
      },
      "extent": {
        "xmin": -88.87911,
        "ymin": 42.54379,
        "xmax": -88.85111,
        "ymax": 42.57179
      }
    },
    {
      "address": "New York",
      "location": {
        "x": -75.4999,
        "y": 43.00035
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "",
        "Subregion": "",
        "RegionAbbr": "NY",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -75.4999,
        "DisplayY": 43.00035
      },
      "extent": {
        "xmin": -78.6429,
        "ymin": 39.85735,
        "xmax": -72.3569,
        "ymax": 46.14335
      }
    },
    {
      "address": "New York",
      "location": {
        "x": -73.90264,
        "y": 40.82276
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York",
        "Addr_type": "POI",
        "AddNum": "789",
        "StPreDir": "E",
        "StPreType": "",
        "StName": "163rd",
        "StType": "St",
        "StDir": "",
        "City": "Bronx",
        "Subregion": "Bronx County",
        "RegionAbbr": "NY",
        "Postal": "10456",
        "Country": "USA",
        "DisplayX": -73.90264,
        "DisplayY": 40.82276
      },
      "extent": {
        "xmin": -73.90364,
        "ymin": 40.82176,
        "xmax": -73.90164,
        "ymax": 40.82376
      }
    },
    {
      "address": "New York",
      "location": {
        "x": -73.998301,
        "y": 40.716688
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York",
        "Addr_type": "POI",
        "AddNum": "81",
        "StPreDir": "",
        "StPreType": "",
        "StName": "Mott",
        "StType": "St",
        "StDir": "",
        "City": "New York",
        "Subregion": "New York County",
        "RegionAbbr": "NY",
        "Postal": "10013",
        "Country": "USA",
        "DisplayX": -73.998301,
        "DisplayY": 40.716688
      },
      "extent": {
        "xmin": -73.999301,
        "ymin": 40.715688,
        "xmax": -73.997301,
        "ymax": 40.717688
      }
    },
    {
      "address": "New York, Florida",
      "location": {
        "x": -87.2008,
        "y": 30.83852
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Florida",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "",
        "Subregion": "Santa Rosa County",
        "RegionAbbr": "FL",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -87.2008,
        "DisplayY": 30.83852
      },
      "extent": {
        "xmin": -87.2118,
        "ymin": 30.82752,
        "xmax": -87.1898,
        "ymax": 30.84952
      }
    },
    {
      "address": "New York, Township of Union, Iowa",
      "location": {
        "x": -93.25993,
        "y": 40.85167
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Township of Union, Iowa",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "Township of Union",
        "Subregion": "Wayne County",
        "RegionAbbr": "IA",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -93.25993,
        "DisplayY": 40.85167
      },
      "extent": {
        "xmin": -93.27093,
        "ymin": 40.84067,
        "xmax": -93.24893,
        "ymax": 40.86267
      }
    },
    {
      "address": "New York, Kentucky",
      "location": {
        "x": -88.95256,
        "y": 36.98894
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Kentucky",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "",
        "Subregion": "Ballard County",
        "RegionAbbr": "KY",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -88.95256,
        "DisplayY": 36.98894
      },
      "extent": {
        "xmin": -88.96356,
        "ymin": 36.97794,
        "xmax": -88.94156,
        "ymax": 36.99994
      }
    },
    {
      "address": "New York, Township of New York, Missouri",
      "location": {
        "x": -93.92688,
        "y": 39.68529
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Township of New York, Missouri",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "Township of New York",
        "Subregion": "Caldwell County",
        "RegionAbbr": "MO",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -93.92688,
        "DisplayY": 39.68529
      },
      "extent": {
        "xmin": -93.93788,
        "ymin": 39.67429,
        "xmax": -93.91588,
        "ymax": 39.69629
      }
    },
    {
      "address": "New York, Seama, New Mexico",
      "location": {
        "x": -107.52727,
        "y": 35.05865
      },
      "score": 100,
      "attributes": {
        "Loc_name": "World",
        "Score": 100,
        "Match_addr": "New York, Seama, New Mexico",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "Seama",
        "Subregion": "Cibola County",
        "RegionAbbr": "NM",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -107.52727,
        "DisplayY": 35.05865
      },
      "extent": {
        "xmin": -107.53827,
        "ymin": 35.04765,
        "xmax": -107.51627,
        "ymax": 35.06965
      }
    }
  ]
}
```

However, if the `magicKey` is included, the `Loc_name` is no longer returned:

```bash
curl  -X GET \
  'https://geocode.arcgis.com/arcgis/rest/services/World/GeocodeServer/findAddressCandidates?f=json&outFields=Loc_name%2CScore%2CMatch_addr%2CAddr_type%2CAddNum%2CStPreDir%2CStPreType%2CStName%2CStType%2CStDir%2CCity%2CSubregion%2CRegionAbbr%2CPostal%2CCountry%2CDisplayX%2CDisplayY&maxLocations=20&singleLine=New+York%2C+USA&sourceCountry=US&magicKey=dHA9MCN0dj02NTc3YTY2NiNsb2M9OTY5Mzg4MSNsbmc9NTUjcGw9NzAyMzczMiNsYnM9MTQ6MzQ5Mzg0MjI7MDo1MTAxMjg1OCNsbj1Xb3JsZA'
```
```json
{
  "spatialReference": {
    "wkid": 4326,
    "latestWkid": 4326
  },
  "candidates": [
    {
      "address": "New York",
      "location": {
        "x": -75.610701447,
        "y": 42.937083979
      },
      "score": 100,
      "attributes": {
        "Score": 100,
        "Match_addr": "New York",
        "Addr_type": "Locality",
        "AddNum": "",
        "StPreDir": "",
        "StPreType": "",
        "StName": "",
        "StType": "",
        "StDir": "",
        "City": "",
        "Subregion": "",
        "RegionAbbr": "NY",
        "Postal": "",
        "Country": "USA",
        "DisplayX": -75.610701447,
        "DisplayY": 42.937083979
      },
      "extent": {
        "xmin": -78.716701447,
        "ymin": 39.831083979,
        "xmax": -72.504701447,
        "ymax": 46.043083979
      }
    }
  ]
}
```

Furthermore, the returned `Addr_type` is `Locality`, which does not pass the default postprocessors:

https://github.com/azavea/python-omgeo/blob/329beedb0c19b7b98b2754f68f316022a49da82f/omgeo/services/esri.py#L56-L61

Make it so that results after specifying `magicKey` are tolerated.

NOTE: [ESRI REST endpoint documentation](https://developers.arcgis.com/rest/geocode/api-reference/geocoding-service-output.htm) does not have a CHANGELOG, but does say that `Loc_name` is an internal field that clients should not use.

Discussion:

Patch:
diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml
index ceadde8..5cebaa9 100644
--- a/.github/workflows/continuous_integration.yml
+++ b/.github/workflows/continuous_integration.yml
@@ -42,11 +42,11 @@ jobs:
 
       - name: Run tests
         run: python setup.py test
-        env:
-          BING_MAPS_API_KEY: ${{ secrets.BING_MAPS_API_KEY }}
-          ESRI_CLIENT_ID: ${{ secrets.ESRI_CLIENT_ID }}
-          ESRI_CLIENT_SECRET: ${{ secrets.ESRI_CLIENT_SECRET }}
-          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
-          MAPQUEST_API_KEY: ${{ secrets.MAPQUEST_API_KEY }}
-          PELIAS_API_KEY: ${{ secrets.PELIAS_API_KEY }}
+        # env:
+          # BING_MAPS_API_KEY: ${{ secrets.BING_MAPS_API_KEY }}
+          # ESRI_CLIENT_ID: ${{ secrets.ESRI_CLIENT_ID }}
+          # ESRI_CLIENT_SECRET: ${{ secrets.ESRI_CLIENT_SECRET }}
+          # GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          # MAPQUEST_API_KEY: ${{ secrets.MAPQUEST_API_KEY }}
+          # PELIAS_API_KEY: ${{ secrets.PELIAS_API_KEY }}
 
diff --git a/omgeo/services/esri.py b/omgeo/services/esri.py
index 437bd9d..99f9e99 100644
--- a/omgeo/services/esri.py
+++ b/omgeo/services/esri.py
@@ -55,6 +55,7 @@ class EsriWGS(GeocodeService):
     DEFAULT_POSTPROCESSORS = [
         AttrFilter(['PointAddress',
                     'StreetAddress',
+                    'Locality',
                     # 'PostalExt',
                     # 'Postal'
                     ],
@@ -62,6 +63,7 @@ class EsriWGS(GeocodeService):
         # AttrExclude(['USA_Postal'], 'locator'), #accept postal from everywhere but US (need PostalExt)
         AttrSorter(['PointAddress',
                     'StreetAddress',
+                    'Locality',
                     # 'PostalExt',
                     # 'Postal'
                     ],
@@ -195,7 +197,7 @@ class EsriWGS(GeocodeService):
                 c = Candidate()
                 attributes = location['attributes']
                 c.match_addr = attributes['Match_addr']
-                c.locator = attributes['Loc_name']
+                c.locator = attributes.get('Loc_name', '')
                 c.locator_type = attributes['Addr_type']
                 c.score = attributes['Score']
                 c.x = attributes['DisplayX']  # represents the actual location of the address.
@@ -210,7 +212,8 @@ class EsriWGS(GeocodeService):
                     setattr(c, out_key, attributes.get(in_key, ''))
                 setattr(c, 'match_streetaddr', self._street_addr_from_response(attributes))
                 returned_candidates.append(c)
-        except KeyError:
+        except KeyError as e:
+            logger.warning('Missing key: ' + e)
             pass
         return returned_candidates
 


Test patch:
diff --git a/omgeo/tests/tests.py b/omgeo/tests/tests.py
index bd133a0..5c94861 100755
--- a/omgeo/tests/tests.py
+++ b/omgeo/tests/tests.py
@@ -208,6 +208,16 @@ class GeocoderTest(OmgeoTestCase):
         self.assertEqual('340 N 12th' in candidates[0].match_addr, True,
                          '"340 N 12th" not found in match_addr. Got "%s"' % candidates[0].match_addr)
 
+    def test_geocode_esri_wgs_magicKey(self):
+        """Check that geocoding New York, USA with a magicKey returns one result."""
+        esri = self.g_esri_wgs._sources[0]
+        suggestions = esri._get_json_obj(
+            f'{esri._endpoint}/suggest',
+            {'f': 'json', 'text': 'New York, USA'})['suggestions']
+        pq = PlaceQuery(suggestions[0]['text'], key=suggestions[0]['magicKey'])
+        candidates = self.g_esri_wgs.get_candidates(pq)
+        self.assertOneCandidate(candidates)
+
     def test_geocode_esri_wgs_zip_plus_4(self):
         """Check that geocoding 19127-1112 returns one result."""
         candidates = self.g_esri_wgs_postal_ok.get_candidates(self.pq['zip_plus_4_in_postal_plus_country'])
@@ -229,6 +239,7 @@ class GeocoderTest(OmgeoTestCase):
         candidate = self.g_esri_wgs.get_candidates(self.pq["azavea"])[0]
         self.assertEqual(candidate.match_region, "PA")
 
+    @unittest.skipIf(GOOGLE_API_KEY is None, GOOGLE_KEY_REQUIRED_MSG)
     def test_google_short_region(self):
         """Ensure that Google uses region abbreviations"""
         candidate = self.g_google.get_candidates(self.pq["azavea"])[0]
@@ -272,8 +283,8 @@ class GeocoderTest(OmgeoTestCase):
         self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
 
     def test_geocode_census(self):
-        """Test Azavea's address using US Census geocoder."""
-        candidates = self.g_census.get_candidates(PlaceQuery('1200 Callowhill St, Philadelphia, PA'))
+        """Test Element 84's address using US Census geocoder."""
+        candidates = self.g_census.get_candidates(PlaceQuery('210 N. Lee Street, Alexandria, VA'))
         self.assertEqual(len(candidates) > 0, True, 'No candidates returned.')
 
     def test_EsriWGS_address_components(self):