Repository: alltheplaces/alltheplaces

Issue:
Spider providence_health_services is broken
During the global build at 2021-08-25-14-42-15, spider **providence_health_services** failed with **451 features** and **4 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-08-25-14-42-15/logs/providence_health_services.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-08-25-14-42-15/output/providence_health_services.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-08-25-14-42-15/output/providence_health_services.geojson))


Discussion:
During the global build at 2021-09-15-14-42-44, spider **providence_health_services** failed with **451 features** and **4 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-09-15-14-42-44/logs/providence_health_services.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-09-15-14-42-44/output/providence_health_services.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-09-15-14-42-44/output/providence_health_services.geojson))

Resolving patch:
diff --git a/locations/spiders/providence_health_services.py b/locations/spiders/providence_health_services.py
deleted file mode 100644
--- a/locations/spiders/providence_health_services.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import json
-
-from scrapy.spiders import SitemapSpider
-
-from locations.items import Feature
-
-
-class ProvidenceHealthServicesSpider(SitemapSpider):
-    name = "providence_health_services"
-    item_attributes = {
-        "brand": "Providence Health Services",
-        "brand_wikidata": "Q7252430",
-    }
-    allowed_domains = ["providence.org"]
-    download_delay = 0.2
-    sitemap_urls = ["https://www.providence.org/sitemap.xml"]
-    sitemap_rules = [
-        (
-            r"https:\/\/www\.providence\.org\/locations\/[-\w]+$",
-            "parse_location",
-        )
-    ]
-
-    def sitemap_filter(self, entries):
-        for entry in entries:
-            # sitemap now includes language codes, but are always redirected to the base url
-            entry["loc"] = entry["loc"].replace("https://www.providence.org/en/", "https://www.providence.org/")
-            # skip bad page
-            if entry["loc"] == "https://www.providence.org/locations/saint-johns-santa-monica-pediatrics-1811-wilshire":
-                continue
-
-            yield entry
-
-    def parse_location(self, response):
-        ldjson = json.loads(response.css('script[type="application/ld+json"]::text').get())
-
-        if ldjson["@type"] == "SpecialAnnouncement":
-            ldjson = ldjson["announcementLocation"]
-
-        lat = ldjson["geo"]["latitude"]
-        lon = ldjson["geo"]["longitude"]
-        if (lat, lon) == (0, 0):
-            return
-        properties = {
-            "lat": lat,
-            "lon": lon,
-            "ref": response.url,
-            "phone": ldjson.get("telephone"),
-            "name": ldjson["name"],
-            "website": response.url,
-        }
-        if "address" in ldjson:
-            address = ldjson["address"]
-            properties.update(
-                {
-                    "street_address": address["streetAddress"],
-                    "city": address["addressLocality"],
-                    "state": address["addressRegion"],
-                    "postcode": address["postalCode"],
-                }
-            )
-        yield Feature(**properties)
diff --git a/locations/spiders/providence_us.py b/locations/spiders/providence_us.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/providence_us.py
@@ -0,0 +1,55 @@
+import re
+
+from scrapy import Request, Spider
+
+from locations.hours import OpeningHours
+from locations.structured_data_spider import StructuredDataSpider
+
+
+class ProvidenceUSSpider(StructuredDataSpider):
+    name = "providence_us"
+    item_attributes = {"brand": "Providence", "brand_wikidata": "Q7252430"}
+    allowed_domains = ["www.providence.org"]
+    start_urls = ["https://www.providence.org/locations?postal=66102&latlng=39.103,-94.666&radius=100000&page=1"]
+    wanted_types = ["MedicalOrganization", "MedicalClinic", "Hospital", "EmergencyService", "SpecialAnnouncement"]
+
+    def parse(self, response):
+        self.parse_location_results(response)
+        pager_script = response.xpath('//script[contains(text(), "dig.modules.paging(")]/text()').get()
+        if total_pages_search_result := re.search(
+            r"\", (\d+), \d+, \"__locationSearchGetPageRequest\"\);", pager_script
+        ):
+            total_pages = int(total_pages_search_result.group(1))
+        else:
+            return
+        for page_number in range(2, total_pages, 1):
+            yield Request(url=response.url[:-1] + str(page_number), callback=self.parse_location_results)
+
+    def parse_location_results(self, response):
+        location_urls = response.xpath('//div[@class="location-card-title"]/h2/a/@href').getall()
+        for location_url in location_urls:
+            yield Request(url=f"https://www.providence.org{location_url}", callback=self.parse_sd)
+
+    def post_process_item(self, item, response, ld_data):
+        item["facebook"] = None
+        item["twitter"] = None
+        item["image"] = None
+        item["opening_hours"] = OpeningHours()
+        hours_string = " ".join(
+            filter(
+                None,
+                response.xpath(
+                    '//div[contains(@class, "location-detail")][1]/div[contains(@class, "hours-text")]//text()'
+                ).getall(),
+            )
+        )
+        hours_string = (
+            hours_string.strip()
+            .replace(";", ":")
+            .replace("a.m.", "AM")
+            .replace("a.m", "AM")
+            .replace("p.m.", "PM")
+            .replace("p.m", "PM")
+        )
+        item["opening_hours"].add_ranges_from_string(hours_string)
+        yield item


Test patch: