Repository: alltheplaces/alltheplaces

Issue:
Spider whitecastle is broken
During the global build at 2021-06-02-14-42-40, spider **whitecastle** failed with **0 features** and **0 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-06-02-14-42-40/logs/whitecastle.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-06-02-14-42-40/output/whitecastle.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-06-02-14-42-40/output/whitecastle.geojson))


Discussion:

Resolving patch:
diff --git a/locations/spiders/whitecastle.py b/locations/spiders/whitecastle.py
--- a/locations/spiders/whitecastle.py
+++ b/locations/spiders/whitecastle.py
@@ -1,98 +1,73 @@
 # -*- coding: utf-8 -*-
 import scrapy
+import csv
 import json
 import re
 
+from locations.hours import OpeningHours
 from locations.items import GeojsonPointItem
 
+
 class WhiteCastleSpider(scrapy.Spider):
     name = "whitecastle"
     item_attributes = { 'brand': "White Castle", 'brand_wikidata': "Q1244034" }
     allowed_domains = ["www.whitecastle.com"]
     timeregex = re.compile('^([0-9:]+)(AM|PM)$')
-    start_urls = (
-        'https://www.whitecastle.com/api/location/search?form=%7B%22origin%22%3A%7B%22latitude%22%3A40.75368539999999%2C%22longitude%22%3A-73.9991637%7D%2C%22count%22%3A9999999%2C%22skip%22%3A0%2C%22targets%22%3A%5B%22Castle%22%5D%7D',
-    )
-
-    def normalize_time(self, time_str):
-        time, ampm = self.timeregex.search(time_str).groups()
-        hour, minute = time.split(':')
-
-        hour = int(hour)
-        if ampm == 'PM' and hour != 12:
-            hour = hour + 12
-        return '%02d:%s' % (hour, minute)
-
-    def normalize_dayrange(self, dayrange):
-        replacements = [
-            ["Monday", "Mo"],
-            ["Tuesday", "Tu"],
-            ["Wednesday", "We"],
-            ["Thursday", "Th"],
-            ["Friday", "Fr"],
-            ["Saturday", "Sa"],
-            ["Sunday", "Su"],
-            ["Mon", "Mo"],
-            ["Tue", "Tu"],
-            ["Wed", "We"],
-            ["Thu", "Th"],
-            ["Fri", "Fr"],
-            ["Sat", "Sa"],
-            ["Sun", "Su"],
-            [" - ", "-"]
-        ]
-        for r in replacements:
-            dayrange = dayrange.replace(r[0], r[1])
-        return dayrange
-
-    def store_hours(self, timetable):
-        if (len(timetable) == 1):
-            hours = timetable[0]
-            if (hours['dayOfWeek'] == 'Sun - Sat') and \
-               ('24 Hours' in hours['formattedTitles'][0]):
-                return '24/7'
-
-        opening_hours = ""
-        for time in timetable:
-            dayrange = time['dayOfWeek']
-            dayrange = self.normalize_dayrange(dayrange)
-            times = time['formattedTitles'][0]
-
-            if ('24 Hours' in times):
-                hours = "00:00-24:00"
-            else:
-                opentime, closetime = times.split(' - ')
-                hours = '{}-{}'.format(
-                    self.normalize_time(opentime),
-                    self.normalize_time(closetime),
-                )
-            opening_hours += '{dayrange} {hours}; '.format(dayrange=dayrange, hours=hours)
-
-        return opening_hours[:-2]
+
+    def start_requests(self):
+        with open("./locations/searchable_points/us_centroids_100mile_radius.csv") as points:
+            for row in csv.DictReader(points):
+                latitude = row["latitude"]
+                longitude = row["longitude"]
+                url = f"https://www.whitecastle.com/wcapi/location-search?lat={latitude}&lng={longitude}&dist=100"
+                yield scrapy.Request(url, headers={"Accept": "application/json"})
+
+    def store_hours(self, days):
+        o = OpeningHours()
+
+        for day in days:
+            day_name = day.get('day')
+            if not day_name:
+                continue
+
+            if day.get('closed'):
+                continue
+
+            if day.get('open24Hours'):
+                o.add_range(day_name[:2], "00:00", "23:59")
+                continue
+
+            open_time, close_time = day.get('hours').split(' - ', 2)
+
+            if close_time in ('12:00 AM', 'Midnight'):
+                close_time = '00:00 AM'
+
+            o.add_range(day_name[:2], open_time, close_time, time_format='%H:%M %p')
+
+        return o.as_opening_hours()
 
     def parse(self, response):
-        data = json.loads(response.body_as_unicode())
+        data = json.loads(response.text)
 
         for store in data:
             properties = {
-                "ref": store.get('id'),
+                "ref": store.get('storeNumber'),
                 "name": store.get('name'),
                 "addr_full": store.get('address'),
                 "city": store.get('city'),
                 "state": store.get('state'),
                 "postcode": store.get('zip'),
                 "phone": store.get('telephone'),
+                "website": f'https://www.whitecastle.com/locations/{store.get("storeNumber")}',
+                "lat": store.get('lat'),
+                "lon": store.get('lng'),
             }
 
-            if store.get('url'):
-                properties['website'] = 'https://www.whitecastle.com' + store.get('url')
-
-            if store.get('latitude'): properties['lat'] = float(store.get('latitude'))
-            if store.get('longitude'): properties['lon'] = float(store.get('longitude'))
-
-            if store.get('timetable'):
-                opening_hours = self.store_hours(store.get('timetable'))
-                if opening_hours: properties['opening_hours'] = opening_hours
+            if store.get('open24x7'):
+                properties['opening_hours'] = '24/7'
+            elif store.get('days'):
+                opening_hours = self.store_hours(store.get('days'))
+                if opening_hours:
+                    properties['opening_hours'] = opening_hours
 
             yield GeojsonPointItem(**properties)
-


Test patch: