Repository: alltheplaces/alltheplaces

Issue:
Spider bankofamerica is broken
During the global build at 2021-06-02-14-42-40, spider **bankofamerica** failed with **1069 features** and **6058 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-06-02-14-42-40/logs/bankofamerica.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-06-02-14-42-40/output/bankofamerica.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-06-02-14-42-40/output/bankofamerica.geojson))


Discussion:

Resolving patch:
diff --git a/locations/spiders/bankofamerica.py b/locations/spiders/bankofamerica.py
--- a/locations/spiders/bankofamerica.py
+++ b/locations/spiders/bankofamerica.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-
 import json
+
 import scrapy
 
 from locations.items import GeojsonPointItem
@@ -8,51 +8,72 @@
 
 class BankOfAmericaSpider(scrapy.Spider):
     name = "bankofamerica"
-    item_attributes = { 'brand': "Bank of America", 'brand_wikidata': "Q487907" }
+    item_attributes = {"brand": "Bank of America", "brand_wikidata": "Q487907"}
+    download_delay = 0.1
     allowed_domains = ["bankofamerica.com"]
-    start_urls = (
-        'https://locators.bankofamerica.com/',
-    )
+    start_urls = ("https://locators.bankofamerica.com/",)
 
     def parse(self, response):
         states = response.xpath('//ul//a[contains(@name, "State_")]/@href')
         for state in states:
             yield scrapy.Request(
-                response.urljoin(state.extract()),
-                callback=self.parse_state
+                response.urljoin(state.extract()), callback=self.parse_state
             )
 
     def parse_state(self, response):
         cities = response.xpath('//ul//a[contains(@name, "City_")]/@href')
         for city in cities:
             yield scrapy.Request(
-                response.urljoin(city.extract()),
-                callback=self.parse_city
+                response.urljoin(city.extract()), callback=self.parse_city
             )
 
     def parse_city(self, response):
-        centers = response.xpath('//div[@class="map-list-item"]//a[contains(@name, "View_")]/@href')
+        centers = response.xpath(
+            '//div[@class="map-list-item"]//a[contains(@name, "View_")]/@href'
+        )
 
         for center in centers:
             yield scrapy.Request(
-                response.urljoin(center.extract()),
-                callback=self.parse_center
+                response.urljoin(center.extract()), callback=self.parse_center
             )
 
     def parse_center(self, response):
-        data = json.loads(response.xpath('//script[@type="application/ld+json"]//text()').extract_first())
-        opening_hours = ";".join(x["openingHours"] for x in data if x.get("openingHours"))
-
-        yield GeojsonPointItem(
-            lat=float(response.xpath('//meta[@property="og:latitude"]/@content').extract_first()),
-            lon=float(response.xpath('//meta[@property="og:longitude"]/@content').extract_first()),
-            phone=response.xpath('//meta[@property="og:phone_number"]/@content').extract_first(),
-            website=response.url,
-            ref=response.xpath('//meta[@name="twitter:title"]/@content').extract_first(),
-            opening_hours=opening_hours,
-            addr_full=response.xpath('//meta[@property="og:street-address"]/@content').extract_first(),
-            city=response.xpath('//meta[@property="og:locality"]/@content').extract_first(),
-            state=response.xpath('//meta[@property="og:region"]/@content').extract_first(),
-            postcode=response.xpath('//meta[@property="og:postal-code"]/@content').extract_first(),
-            country=response.xpath('//meta[@property="og:country-name"]/@content').extract_first(),
-        )
+        ref = response.url.rsplit("-", 1)[-1].split(".")[0]
+        for ldjson in response.xpath(
+            '//script[@type="application/ld+json"]//text()'
+        ).extract():
+            if "SpecialAnnouncement" in ldjson:
+                # Contains broken json and is irrelevant
+                continue
+            yield self.parse_entity(ldjson, ref)
+
+    def parse_entity(self, ldjson, page_ref):
+        # Each feature has a BankOrCreditUnion and then either a FinancialService
+        # or an AutomatedTeller. Loop over them to find a single item.
+        properties = {"extras": {}}
+        for ent in json.loads(ldjson):
+            if "geo" not in ent:
+                continue
+            properties.update(
+                {
+                    "ref": page_ref,
+                    "website": ent["url"],
+                    "lat": ent["geo"]["latitude"],
+                    "lon": ent["geo"]["longitude"],
+                    "addr_full": ent["address"]["streetAddress"],
+                    "city": ent["address"]["addressLocality"],
+                    "state": ent["address"]["addressRegion"],
+                    "postcode": ent["address"]["postalCode"],
+                    "country": ent["address"]["addressCountry"]["name"],
+                }
+            )
+
+            if ent["@type"] != "AutomatedTeller":
+                # Skip over ATM customer service and the shorter name
+                properties.update({"name": ent["name"], "phone": ent["telephone"]})
+            if "openingHours" in ent:
+                properties.update({"opening_hours": ent["openingHours"]})
+            if ent["@type"] != "BankOrCreditUnion":
+                # i.e. the interesting of the two types
+                properties["extras"].update({"type": ent["@type"]})
+        return GeojsonPointItem(**properties)


Test patch: