Repository: alltheplaces/alltheplaces

Issue:
VISA ATM Locator
### Brand name

Visa

### Wikidata ID

[Q328840](https://www.wikidata.org/wiki/Q328840)

### Store finder url(s)

https://www.visa.com/locator/atm


Discussion:
I've just been playing with this because I though the data would be interesting.

The API taunts me with `totalATMCount` but won't let me get past the first 100 for any query. You can do any distance, but it'll never give you more that the 100. Which means we'd have to do multiple queries, as in a lat/lon search. I'm not comfortable doing that world wide.

Below is my code, in case anyone else wants a go.

```python
import hashlib
import json
import urllib.parse

import chompjs
from scrapy import Spider
from scrapy.http import JsonRequest

from locations.categories import Categories
from locations.dict_parser import DictParser
from locations.geo import point_locations


class VisaSpider(Spider):
    name = "visa"
    item_attributes = {"extras": Categories.ATM.value}

    @staticmethod
    def make_request(lat: float, lon: float) -> JsonRequest:
        return JsonRequest(
            url="https://www.visa.com/atmlocator_services/rest/findNearByATMs?{}".format(
                urllib.parse.urlencode(
                    {
                        "callback": "locations",
                        "request": json.dumps(
                            {
                                "wsRequestHeaderV2": {
                                    "applicationId": "LOCATORUI",
                                    "requestMessageId": "test12345678",
                                    "userId": "CDISIUserID",
                                    "userBid": "10000108",
                                    "correlationId": "909420141104053819418",
                                },
                                "requestData": {
                                    "distance": 100,
                                    "distanceUnit": "mi",
                                    "location": {"geocodes": {"latitude": lat, "longitude": lon}},
                                    "options": {
                                        "sort": {"primary": "distance", "direction": "asc"},
                                        "range": {"start": 0, "count": 100},
                                    },
                                },
                            }
                        ),
                    }
                )
            )
        )

    def start_requests(self):
        for lat, lon in point_locations("eu_centroids_80km_radius_country.csv", "UK"):
            yield self.make_request(lat, lon)

    def parse(self, response, **kwargs):
        js = chompjs.parse_js_object(response.text)
        if data := js.get("responseData"):
            for resp in data:
                for atm in resp.get("foundATMLocations"):
                    location = atm["location"]
                    location["address"]["street_address"] = ", ".join(
                        filter(None, [location["address"].pop("street"), location["address"].pop("street2")])
                    )

                    item = DictParser.parse(location)

                    item["ref"] = hashlib.sha256(str(atm).encode("utf_8")).hexdigest()

                    item["extras"] = {}
                    item["extras"]["operator"] = location.get("ownerBusName")
                    item["located_in"] = location.get("placeName")

                    yield item
```
I asked for 10000 locations within a 10000 mile radius per:

`https://www.visa.com/gateway/api/locators-service/locators/findNearByLocations?locatorRequest={"locatorRequestData":{"culture":"en-US","distance":10000,"distanceUnit":"mi","metaDataOptions":0,"locatorType":"ATM","location":{"address":null,"placeName":"","geocodes":{"latitude":-37.8136276,"longitude":144.9630576}},"options":{"sort":{"primary":"distance","direction":"asc"},"range":{"start":0,"count":10000},"operationName":"and","findFilters":[],"useFirstAmbiguous":true}}}`

And 10000 locations were returned (approx. 15MB JSON response) on the first instance. Subsequent requests failed with a HTTP 500 error (likely due to resource exhaustion/timeout on the server side).

I seem to get fairly consistent responses with 5000 locations requested, but it's slow (~7MB JSON response) and I did notice the odd timeout error, so I think a smaller request like 2000 locations may be better.

If you ask for a 100000 mile search radius (whole Earth), the API reports 1573470 ATMs existing. So at 2000 results returned per API call, it's best case 800 or so requests. But because search radiuses will necessarily overlap and ATMs aren't at the same density around the world, I suspect 2000-10000 requests may be needed. Total data needing to be downloaded and parsed appears to be in the order of 2.5GB.
Whenever you can paginate through an api, it's better than doing multiple searches.

```python
import json
import urllib.parse

from scrapy import Spider
from scrapy.http import JsonRequest

from locations.categories import Categories, apply_category
from locations.dict_parser import DictParser


class VisaSpider(Spider):
    name = "visa"
    offset = 0
    page_size = 1000
    no_refs = True
    custom_settings = {"AUTOTHROTTLE_ENABLED": True, "RETRY_TIMES": 5}

    def next_request(self) -> JsonRequest:
        return JsonRequest(
            url="https://www.visa.com/gateway/api/locators-service/locators/findNearByLocations?{}".format(
                urllib.parse.urlencode(
                    {
                        "locatorRequest": json.dumps(
                            {
                                "locatorRequestData": {
                                    "culture": "en-US",
                                    "distance": 100000,
                                    "distanceUnit": "mi",
                                    "metaDataOptions": 0,
                                    "locatorType": "ATM",
                                    "location": {
                                        "address": None,
                                        "placeName": "",
                                        "geocodes": {"latitude": 0, "longitude": 0},
                                    },
                                    "options": {
                                        "sort": {"primary": "distance", "direction": "asc"},
                                        "range": {"start": self.offset, "count": self.page_size},
                                        "operationName": "and",
                                        "findFilters": [],
                                        "useFirstAmbiguous": True,
                                    },
                                }
                            }
                        ),
                    }
                )
            ),
            errback=self.error,
        )

    def start_requests(self):
        yield self.next_request()

    def parse(self, response, **kwargs):
        if data := response.json().get("responseData"):
            for atm in data.get("foundATMLocations") or []:
                location = atm["location"]
                location["address"]["street_address"] = ", ".join(
                    filter(None, [location["address"].pop("street"), location["address"].pop("street2")])
                )

                item = DictParser.parse(location)

                item["extras"]["operator"] = location.get("ownerBusName")
                item["located_in"] = location.get("placeName")

                apply_category(Categories.ATM, item)

                yield item

            self.offset += self.page_size
            print([self.offset, data["totalATMCount"], (100 / data["totalATMCount"]) * self.offset])

            if self.offset < data["totalATMCount"]:
                yield self.next_request()

    def error(self, response, **kwargs):
        # Skip a page
        self.logger.error("Skipping offset: {}".format(self.offset))
        self.offset += self.page_size
        yield self.next_request()
```
Pull request incoming? Or concerns with the approach suggested? Need any assistance?
I just finished a run, 1,533,470 ATMs, and a few network errors. I was assuming Ian wouldn't want them, also "we have 4.5m pois, but 1.5m are ATMs" is a bit depressing. But yeah, I shouldn't assume, @iandees do you want a PR?

```python
{'atp/category/amenity/atm': 1533470,
 'atp/cdn/cloudflare/response_count': 1575,
 'atp/cdn/cloudflare/response_status_count/200': 1534,
 'atp/cdn/cloudflare/response_status_count/403': 1,
 'atp/cdn/cloudflare/response_status_count/500': 39,
 'atp/cdn/cloudflare/response_status_count/502': 1,
 'atp/field/brand/missing': 1533470,
 'atp/field/brand_wikidata/missing': 1533470,
 'atp/field/city/missing': 293,
 'atp/field/country/from_reverse_geocoding': 8170,
 'atp/field/country/invalid': 11,
 'atp/field/email/missing': 1533470,
 'atp/field/image/missing': 1533470,
 'atp/field/lat/invalid': 1,
 'atp/field/lat/missing': 1959,
 'atp/field/lon/missing': 1959,
 'atp/field/name/missing': 1533470,
 'atp/field/opening_hours/missing': 1533470,
 'atp/field/phone/missing': 1533470,
 'atp/field/postcode/missing': 303465,
 'atp/field/state/from_reverse_geocoding': 7117,
 'atp/field/state/missing': 461157,
 'atp/field/street_address/missing': 37,
 'atp/field/twitter/missing': 1533470,
 'atp/field/website/missing': 1533470,
 'atp/geometry/null_island': 38,
 'downloader/request_bytes': 3381853,
 'downloader/request_count': 1775,
 'downloader/request_method_count/GET': 1775,
 'downloader/response_bytes': 111283228,
 'downloader/response_count': 1775,
 'downloader/response_status_count/200': 1534,
 'downloader/response_status_count/403': 1,
 'downloader/response_status_count/500': 234,
 'downloader/response_status_count/502': 6,
 'elapsed_time_seconds': 1462.529259,
 'feedexport/success_count/FileFeedStorage': 1,
 'finish_reason': 'finished',
 'finish_time': datetime.datetime(2023, 8, 23, 18, 24, 57, 897554),
 'httpcache/hit': 1775,
 'httpcompression/response_bytes': 2125682804,
 'httpcompression/response_count': 1535,
 'item_scraped_count': 1533470,
 'log_count/ERROR': 80,
 'log_count/INFO': 33,
 'log_count/WARNING': 450,
 'memusage/max': 494075904,
 'memusage/startup': 344637440,
 'request_depth_max': 1573,
 'response_received_count': 1575,
 'retry/count': 200,
 'retry/max_reached': 40,
 'retry/reason_count/500 Internal Server Error': 195,
 'retry/reason_count/502 Bad Gateway': 5,
 'robotstxt/request_count': 1,
 'robotstxt/response_count': 1,
 'robotstxt/response_status_count/403': 1,
 'scheduler/dequeued': 1774,
 'scheduler/dequeued/memory': 1774,
 'scheduler/enqueued': 1774,
 'scheduler/enqueued/memory': 1774,
 'start_time': datetime.datetime(2023, 8, 23, 18, 0, 35, 368295)
}
`stadt_zuerich_ch` provides every lamp post in Zurich, so I'm not sure either haha
Sure, PR is ok.

This sort of skirts around my rule of "no aggregators". I imagine some of these ATMs exist inside of places we already scrape and might be run by banks that we are already scraping. But as long as we give tag them with ATM tags, I think we should be ok.
> I imagine some of these ATMs exist inside of places we already scrape and might be run by banks that we are already scraping. But as long as we give tag them with ATM tags, I think we should be ok.

There will no doubt the duplicates and ambiguities in data. It'll be interesting to see whether VISA has up-to-date data versus the bank's store locator sites, with the banks directly managing the ATM. It'll also be interesting to see whether coordinates for duplicates line up, or whether VISA has slightly more or less accurate coordinates for each ATM. VISA may also have additional fields of data available (reference: https://wiki.openstreetmap.org/wiki/Tag:amenity%3Datm) which bank store locators may not provide, for example, `tactile_writing:braille=yes/no`.

For banks, fuel stations, shops, etc which have one or more ATMs co-located within, generally `item["extras"]["atm:yes"]` is being set on the overall amenity, rather than a new item with `item["extras"]["amenity:atm"]` being generated just for the ATM. I have seen at least one spider that spawns multiple items per location, e.g. one for the convenience shop, one for the ATM, one for the car wash, etc. Coordinates are probably the same for each of the items spawned. Splitting amenities would be my preference, as then more detailed tags could be applied to each amenity, for example, an ATM and toilet which are otherwise tagged as part of the same overall amenity may have different values for `charge=x EUR`. Perhaps we should document the preference for "colocated amenities" and decide when splitting should occur (e.g. all the time, only when detailed tags need to be used on specific amenities, for certain amenities only, etc)?

Resolving patch:
diff --git a/locations/spiders/visa.py b/locations/spiders/visa.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/visa.py
@@ -0,0 +1,82 @@
+import json
+import urllib.parse
+
+from scrapy import Spider
+from scrapy.http import JsonRequest
+
+from locations.categories import Categories, apply_category
+from locations.dict_parser import DictParser
+
+
+class VisaSpider(Spider):
+    name = "visa"
+    no_refs = True
+    custom_settings = {"AUTOTHROTTLE_ENABLED": True, "RETRY_TIMES": 5}
+
+    offset = 0
+    page_size = 1000
+
+    def next_request(self) -> JsonRequest:
+        return JsonRequest(
+            url="https://www.visa.com/gateway/api/locators-service/locators/findNearByLocations?{}".format(
+                urllib.parse.urlencode(
+                    {
+                        "locatorRequest": json.dumps(
+                            {
+                                "locatorRequestData": {
+                                    "culture": "en-US",
+                                    "distance": 100000,
+                                    "distanceUnit": "mi",
+                                    "metaDataOptions": 0,
+                                    "locatorType": "ATM",
+                                    "location": {
+                                        "address": None,
+                                        "placeName": "",
+                                        "geocodes": {"latitude": 0, "longitude": 0},
+                                    },
+                                    "options": {
+                                        "sort": {"primary": "distance", "direction": "asc"},
+                                        "range": {"start": self.offset, "count": self.page_size},
+                                        "operationName": "and",
+                                        "findFilters": [],
+                                        "useFirstAmbiguous": True,
+                                    },
+                                }
+                            }
+                        ),
+                    }
+                )
+            ),
+            errback=self.error,
+        )
+
+    def start_requests(self):
+        yield self.next_request()
+
+    def parse(self, response, **kwargs):
+        if data := response.json().get("responseData"):
+            for atm in data.get("foundATMLocations") or []:
+                location = atm["location"]
+                location["address"]["street_address"] = ", ".join(
+                    filter(None, [location["address"].pop("street"), location["address"].pop("street2")])
+                )
+
+                item = DictParser.parse(location)
+
+                item["extras"]["operator"] = location.get("ownerBusName")
+                item["located_in"] = location.get("placeName")
+
+                apply_category(Categories.ATM, item)
+
+                yield item
+
+            self.offset += self.page_size
+
+            if self.offset < data["totalATMCount"]:
+                yield self.next_request()
+
+    def error(self, response, **kwargs):
+        # Skip a page
+        self.logger.error("Skipping offset: {}".format(self.offset))
+        self.offset += self.page_size
+        yield self.next_request()


Test patch: