Repository: alltheplaces/alltheplaces

Issue:
Spider krystal is broken
During the global build at 2021-10-27-14-42-46, spider **krystal** failed with **0 features** and **0 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-10-27-14-42-46/logs/krystal.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-10-27-14-42-46/output/krystal.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-10-27-14-42-46/output/krystal.geojson))


Discussion:

Resolving patch:
diff --git a/locations/spiders/krystal.py b/locations/spiders/krystal.py
deleted file mode 100644
--- a/locations/spiders/krystal.py
+++ /dev/null
@@ -1,158 +0,0 @@
-import json
-
-import scrapy
-
-from locations.items import Feature
-
-STATES = [
-    "AL",
-    "AK",
-    "AZ",
-    "AR",
-    "CA",
-    "CO",
-    "CT",
-    "DC",
-    "DE",
-    "FL",
-    "GA",
-    "HI",
-    "ID",
-    "IL",
-    "IN",
-    "IA",
-    "KS",
-    "KY",
-    "LA",
-    "ME",
-    "MD",
-    "MA",
-    "MI",
-    "MN",
-    "MS",
-    "MO",
-    "MT",
-    "NE",
-    "NV",
-    "NH",
-    "NJ",
-    "NM",
-    "NY",
-    "NC",
-    "ND",
-    "OH",
-    "OK",
-    "OR",
-    "PA",
-    "RI",
-    "SC",
-    "SD",
-    "TN",
-    "TX",
-    "UT",
-    "VT",
-    "VA",
-    "WA",
-    "WV",
-    "WI",
-    "WY",
-]
-
-
-class KrystalSpider(scrapy.Spider):
-    name = "krystal"
-    item_attributes = {"brand": "Krystal"}
-    allowed_domains = ["krystal.com"]
-    download_delay = 1.5
-    start_urls = ("http://krystal.com/wp-admin/admin-ajax.php",)
-    custom_settings = {"ROBOTSTXT_OBEY": False}
-
-    def parse_hours(self, hours):
-        hours = json.loads(hours)
-        for key, epoch in hours.items():
-            if epoch["open"] == "99":
-                epoch["open"] = "00"
-
-            if set(epoch.values()) == {"99"}:
-                hours[key] = "00:00-24:00"
-            elif len(epoch["close"]) < 2 and len(epoch["open"]) < 2:
-                hours[key] = "0{}:00-0{}:00".format(epoch["open"], epoch["close"])
-
-            elif len(epoch["close"]) < 2 and len(epoch["open"]) >= 2:
-                hours[key] = "{}:00-0{}:00".format(epoch["open"], epoch["close"])
-
-            elif len(epoch["open"]) < 2 and len(epoch["close"]) >= 2:
-                hours[key] = "0{}:00-{}:00".format(epoch["open"], epoch["close"])
-
-            elif len(epoch["open"]) >= 2 and len(epoch["close"]) >= 2:
-                hours[key] = "{}:00-{}:00".format(epoch["open"], epoch["close"])
-
-        reversed_hours = {}
-        for key, epoch in hours.items():
-            reversed_hours.setdefault(epoch, [])
-            reversed_hours[epoch].append(key[:2].title())
-
-        if len(reversed_hours) == 1 and list(reversed_hours)[0] == "00:00-24:00":
-            return "24/7"
-
-        opening_hours = []
-
-        for key, value in reversed_hours.items():
-            if len(value) == 1:
-                opening_hours.append("{} {}".format(value[0], key))
-            else:
-                opening_hours.append("{}-{} {}".format(value[0], value[-1], key))
-
-        return ";".join(opening_hours)
-
-    def parse(self, response):
-        data = response.json()
-
-        for key, store_infor in data.items():
-            if isinstance(store_infor, dict):
-                opening_hours = self.parse_hours(store_infor["storehours"])
-
-                properties = {
-                    "addr_full": store_infor["address"],
-                    "phone": store_infor["phone"],
-                    "city": store_infor["city"],
-                    "state": store_infor["state"],
-                    "postcode": store_infor["zip"],
-                    "ref": store_infor["unit"],
-                    "website": "{}{}{}".format("http://", self.allowed_domains[0], store_infor["url"]),
-                    "lat": store_infor["lat"],
-                    "lon": store_infor["lng"],
-                    "opening_hours": opening_hours,
-                }
-                yield Feature(**properties)
-
-    def start_requests(self):
-        headers = {
-            "Accept-Language": "en-US,en;q=0.8,ru;q=0.6",
-            "Origin": "http://krystal.com",
-            "Accept-Encoding": "gzip, deflate",
-            "Accept": "*/*",
-            "Referer": "http://krystal.com/locations/",
-            "Connection": "keep-alive",
-            "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
-            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
-        }
-
-        for state in STATES:
-            form_data = {
-                "action": "get_locations_set_by_state",
-                "data[devicelng]": "-77.2888",
-                "data[maplng]": "-86.8073",
-                "data[maplat]": "32.799",
-                "data[devicelat]": "38.8318",
-                "data[state]": state,
-                "data[numresults]": "10",
-            }
-
-            yield scrapy.http.FormRequest(
-                url=self.start_urls[0],
-                method="POST",
-                formdata=form_data,
-                headers=headers,
-                callback=self.parse,
-            )
diff --git a/locations/spiders/krystal_us.py b/locations/spiders/krystal_us.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/krystal_us.py
@@ -0,0 +1,41 @@
+import re
+
+from scrapy.spiders import SitemapSpider
+
+from locations.hours import OpeningHours
+from locations.structured_data_spider import StructuredDataSpider
+
+
+class KrystalUSSpider(SitemapSpider, StructuredDataSpider):
+    name = "krystal_us"
+    item_attributes = {"brand": "Krystal", "brand_wikidata": "Q472195"}
+    allowed_domains = ["www.krystal.com"]
+    sitemap_urls = ["http://www.krystal.com/sitemap.xml"]
+    sitemap_rules = [(r"\/locations\/[a-z]{2}\/[\w\-]+\/[\w\-]+\/?$", "parse_sd")]
+
+    def post_process_item(self, item, response, ld_data):
+        store_ref = response.xpath("//h1[1]/text()").get()
+        if m := re.match(r"^KRYSTAL ([\w]+) \|", store_ref, re.IGNORECASE):
+            item["ref"] = m.group(1)
+        else:
+            item["ref"] = response.url
+        item["name"] = response.xpath("//title/text()").get().split(" | ", 1)[0]
+        item["addr_full"] = " ".join(filter(None, response.xpath('//div[@class="center-box"]/p[1]/text()').getall()))
+        item["website"] = response.url
+        item["opening_hours"] = OpeningHours()
+        hours_columns = response.xpath('//div[@class="hours flex"]/div[@class="col flex"]')
+        for hours_column in hours_columns:
+            day_names = hours_column.xpath('.//div[@class="day flex"]/p/span/text()').getall()
+            times = hours_column.xpath('.//div[@class="time flex"]/p/span/text()').getall()
+            days = zip(day_names, times)
+            for day in days:
+                times = day[1].upper().replace("OPEN 24 HOURS", "12:00AM-11:59PM")
+                if "CLOSED" in times:
+                    continue
+                open_time, close_time = times.split("-", 1)
+                if ":" not in open_time:
+                    open_time = open_time.replace("AM", ":00AM").replace("PM", ":00PM")
+                if ":" not in close_time:
+                    close_time = close_time.replace("AM", ":00AM").replace("PM", ":00PM")
+                item["opening_hours"].add_range(day[0], open_time, close_time, "%I:%M%p")
+        yield item


Test patch: