Repository: alltheplaces/alltheplaces

Issue:
Spider heb is broken
During the global build at 2021-08-18-14-42-26, spider **heb** failed with **320 features** and **1 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-08-18-14-42-26/logs/heb.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-08-18-14-42-26/output/heb.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-08-18-14-42-26/output/heb.geojson))


Discussion:
During the global build at 2021-09-01-14-42-16, spider **heb** failed with **320 features** and **1 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-09-01-14-42-16/logs/heb.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-09-01-14-42-16/output/heb.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-09-01-14-42-16/output/heb.geojson))

Resolving patch:
diff --git a/locations/spiders/h_e_b_us.py b/locations/spiders/h_e_b_us.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/h_e_b_us.py
@@ -0,0 +1,52 @@
+import re
+
+from scrapy import Spider
+from scrapy.http import JsonRequest
+
+from locations.dict_parser import DictParser
+from locations.hours import DAYS_FULL, OpeningHours
+
+
+class HEBUSSpider(Spider):
+    name = "h_e_b_us"
+    item_attributes = {"brand": "H-E-B", "brand_wikidata": "Q830621"}
+    allowed_domains = ["www.heb.com"]
+    start_urls = ["https://www.heb.com/graphql"]
+    custom_settings = {"ROBOTSTXT_OBEY": False}
+
+    def start_requests(self):
+        graphql_query = [
+            {
+                "extensions": {
+                    "persistedQuery": {
+                        "sha256Hash": "24dc544e4b33ce6995dff6650e4deb9ba53e8c4a928b33535a710b96bd2517d6",
+                        "version": 1,
+                    }
+                },
+                "operationName": "StoreDetailsSearch",
+                "variables": {"address": "90210", "radius": 10000, "size": 5000},
+            }
+        ]
+        for url in self.start_urls:
+            yield JsonRequest(url=url, method="POST", data=graphql_query)
+
+    def parse(self, response):
+        for location in response.json()[0]["data"]["searchStoresByAddress"]["stores"]:
+            item = DictParser.parse(location["store"])
+            item["website"] = (
+                "https://www.heb.com/heb-store/"
+                + item["country"]
+                + "/"
+                + item["state"].lower()
+                + "/"
+                + item["city"].lower()
+                + "/"
+                + re.sub(r"[^\w]", "-", item["name"].lower())
+                + "-"
+                + str(item["ref"])
+            )
+            item["opening_hours"] = OpeningHours()
+            for day in location["store"]["storeHours"]:
+                if day["day"].title() in DAYS_FULL:
+                    item["opening_hours"].add_range(day["day"].title(), day["opens"], day["closes"])
+            yield item
diff --git a/locations/spiders/heb.py b/locations/spiders/heb.py
deleted file mode 100644
--- a/locations/spiders/heb.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import json
-
-import scrapy
-
-from locations.hours import OpeningHours
-from locations.items import Feature
-
-
-class HEBSpider(scrapy.Spider):
-    name = "heb"
-    item_attributes = {"brand": "H-E-B", "brand_wikidata": "Q830621"}
-    allowed_domains = ["www.heb.com"]
-    download_delay = 0.2
-    start_urls = ("https://www.heb.com/sitemap/storeSitemap.xml",)
-
-    def parse(self, response):
-        xml = scrapy.selector.Selector(response)
-        xml.remove_namespaces()
-
-        urls = xml.xpath("//loc/text()").extract()
-        for url in urls:
-            yield scrapy.Request(url=url, callback=self.parse_store, meta={"url": url})
-
-    def parse_store(self, response):
-        if response.request.meta.get("redirect_urls"):
-            return
-
-        store_json = json.loads(response.xpath('//script[@type="application/ld+json"]/text()').extract_first())
-        yield Feature(
-            ref=response.url.split("/")[-1],
-            name=store_json["name"],
-            lat=float(store_json["geo"]["latitude"]),
-            lon=float(store_json["geo"]["longitude"]),
-            addr_full=store_json["address"]["streetAddress"],
-            city=store_json["address"]["addressLocality"],
-            state=store_json["address"]["addressRegion"],
-            postcode=store_json["address"]["postalCode"],
-            country=store_json["address"]["addressCountry"],
-            phone=store_json["telephone"],
-            website=response.url,
-            opening_hours=self.parse_hours(store_json["openingHoursSpecification"]),
-        )
-
-    def parse_hours(self, hours):
-        opening_hours = OpeningHours()
-
-        for hour in hours:
-            opening_hours.add_range(
-                day=hour["dayOfWeek"][0:2].capitalize(),
-                open_time=hour["opens"],
-                close_time=hour["closes"],
-            )
-
-        return opening_hours.as_opening_hours()


Test patch: