Repository: alltheplaces/alltheplaces

Issue:
Noah's NY Bagels
https://locations.noahs.com/us


Discussion:

Resolving patch:
diff --git a/locations/spiders/brueggers.py b/locations/spiders/brueggers.py
--- a/locations/spiders/brueggers.py
+++ b/locations/spiders/brueggers.py
@@ -1,104 +1,90 @@
 # -*- coding: utf-8 -*-
 import scrapy
-import json
 import re
 
 from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
 
 
+DAY_MAPPING = {
+    'Monday': 'Mo',
+    'Tuesday': 'Tu',
+    'Wednesday': 'We',
+    'Thursday': 'Th',
+    'Friday': 'Fr',
+    'Saturday': 'Sa',
+    'Sunday': 'Su'
+}
+
 class BrueggersSpider(scrapy.Spider):
+    """Copy of Einstein Bros. Bagels - all brands of the same parent company Coffee & Bagels"""
     name = "brueggers"
-    allowed_domains = ["www.brueggers.com"]
+    allowed_domains = ["brueggers.com"]
     start_urls = (
-        'https://www.brueggers.com/locations/',
+        'https://locations.brueggers.com/us',
     )
 
-    def normalize_time(self, time_str):
-        match = re.search(r'(\d{1,2}):(\d{2}) ([AP])M', time_str)
-        h, m, am_pm = match.groups()
-
-        return '%02d:%02d' % (
-            int(h) + 12 if am_pm == 'P' else int(h),
-            int(m),
-        )
+    def parse_hours(self, elements):
+        opening_hours = OpeningHours()
 
-    def store_hours(self, store_hours):
-        day_groups = []
-        this_day_group = None
-        for day in ('monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'):
-            day_open = store_hours[day + '_open']
-            day_close = store_hours[day + '_close']
+        for elem in elements:
+            day = elem.xpath('.//td[@class="c-location-hours-details-row-day"]/text()').extract_first()
+            intervals = elem.xpath('.//td[@class="c-location-hours-details-row-intervals"]')
 
-            if day_open is False:
-                # On days that they're closed they set the value to 'false'
+            if intervals.xpath('./text()').extract_first() == "Closed":
                 continue
+            if intervals.xpath('./span/text()').extract_first() == "Open 24 hours":
+                opening_hours.add_range(day=DAY_MAPPING[day],
+                                        open_time='0:00',
+                                        close_time='23:59')
+            else:
+                start_time = elem.xpath(
+                    './/span[@class="c-location-hours-details-row-intervals-instance-open"]/text()').extract_first()
+                end_time = elem.xpath(
+                    './/span[@class="c-location-hours-details-row-intervals-instance-close"]/text()').extract_first()
+                opening_hours.add_range(day=day[:2],
+                                        open_time=start_time,
+                                        close_time=end_time,
+                                        time_format='%H:%M %p')
+
+        return opening_hours.as_opening_hours()
+
+    def parse_store(self, response):
+        ref = re.search(r'.+/(.+)$', response.url).group(1)
+
+        address1 = response.xpath('//span[@class="c-address-street-1"]/text()').extract_first()
+        address2 = response.xpath('//span[@class="c-address-street-2"]/text()').extract_first() or ''
+
+        properties = {
+            'addr_full': " ".join([address1, address2]).strip(),
+            'phone': response.xpath('//span[@itemprop="telephone"]/text()').extract_first(),
+            'city': response.xpath('//span[@class="c-address-city"]/text()').extract_first(),
+            'state': response.xpath('//span[@itemprop="addressRegion"]/text()').extract_first(),
+            'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first(),
+            'country': response.xpath('//abbr[@itemprop="addressCountry"]/text()').extract_first(),
+            'ref': ref,
+            'website': response.url,
+            'lat': float(response.xpath('//meta[@itemprop="latitude"]/@content').extract_first()),
+            'lon': float(response.xpath('//meta[@itemprop="longitude"]/@content').extract_first()),
+            'name': response.xpath('//h1[@id="location-name"]/text()').extract_first()
+        }
+
+        hours = self.parse_hours(response.xpath('//table[@class="c-location-hours-details"]//tbody/tr'))
+
+        if hours:
+            properties['opening_hours'] = hours
+
+        yield GeojsonPointItem(**properties)
 
-            day_open = self.normalize_time(day_open)
-            day_close = self.normalize_time(day_close)
-
-            hours = day_open + "-" + day_close
-
-            day_short = day.title()[:2]
-
-            if not this_day_group:
-                this_day_group = dict(from_day=day_short, to_day=day_short, hours=hours)
-            elif this_day_group['hours'] == hours:
-                this_day_group['to_day'] = day_short
-            elif this_day_group['hours'] != hours:
-                day_groups.append(this_day_group)
-                this_day_group = dict(from_day=day_short, to_day=day_short, hours=hours)
-        day_groups.append(this_day_group)
-
-        if not day_groups:
-            return None
-
-        if len(day_groups) == 1:
-            opening_hours = day_groups[0]['hours']
-            if opening_hours == '07:00-07:00':
-                opening_hours = '24/7'
-        else:
-            opening_hours = ''
-            for day_group in day_groups:
-                if day_group['from_day'] == day_group['to_day']:
-                    opening_hours += '{from_day} {hours}; '.format(**day_group)
-                else:
-                    opening_hours += '{from_day}-{to_day} {hours}; '.format(**day_group)
-            opening_hours = opening_hours[:-2]
+    def parse(self, response):
+        urls = response.xpath('//a[@class="Directory-listLink"]/@href').extract()
+        is_store_list = response.xpath('//section[contains(@class,"LocationList")]').extract()
 
-        return opening_hours
+        if not urls and is_store_list:
+            urls = response.xpath('//a[contains(@class,"Teaser-titleLink")]/@href').extract()
 
-    def parse(self, response):
-        map_js = response.xpath('//script[@type="text/javascript"]/text()')[3].extract()
-        lat_lons = re.findall(r'LatLng\(\s+(\S*),(\S*)\);', map_js)
-
-        for lat, lon in lat_lons:
-            yield scrapy.Request(
-                'https://www.brueggers.com/wp-json/brueggers/v1/locations/search/?lat={}&lng={}'.format(
-                    lat, lon
-                ),
-                callback=self.parse_lookup_results,
-            )
-
-    def parse_lookup_results(self, response):
-        # There's extra JSON loading stuff before and after the JSON blob to skip around
-        store_list = json.loads(response.body_as_unicode())
-
-        for store_data in store_list:
-            properties = {
-                'phone': store_data['phone'],
-                'website': store_data['guid'],
-                'ref': store_data['location_id'],
-                'name': store_data['post_title'],
-                'addr_full': store_data['address'],
-                'postcode': store_data['zip'],
-                'state': store_data['state'],
-                'city': store_data['city'],
-                'lon': float(store_data['lng']),
-                'lat': float(store_data['lat']),
-            }
-
-            opening_hours = self.store_hours(store_data)
-            if opening_hours:
-                properties['opening_hours'] = opening_hours
-
-            yield GeojsonPointItem(**properties)
+        for url in urls:
+            if re.search(r'us/.{2}/.+/.+', url):
+                yield scrapy.Request(response.urljoin(url), callback=self.parse_store)
+            else:
+                yield scrapy.Request(response.urljoin(url))
diff --git a/locations/spiders/manhattanbagels.py b/locations/spiders/manhattanbagels.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/manhattanbagels.py
@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+DAY_MAPPING = {
+    'Monday': 'Mo',
+    'Tuesday': 'Tu',
+    'Wednesday': 'We',
+    'Thursday': 'Th',
+    'Friday': 'Fr',
+    'Saturday': 'Sa',
+    'Sunday': 'Su'
+}
+
+class ManhattanBagelSpider(scrapy.Spider):
+    """Copy of Einstein Bros. Bagels - all brands of the same parent company Coffee & Bagels"""
+    name = "manhattanbagels"
+    allowed_domains = ["manhattanbagel.com"]
+    start_urls = (
+        'https://locations.manhattanbagel.com/us',
+    )
+
+    def parse_hours(self, elements):
+        opening_hours = OpeningHours()
+
+        for elem in elements:
+            day = elem.xpath('.//td[@class="c-location-hours-details-row-day"]/text()').extract_first()
+            intervals = elem.xpath('.//td[@class="c-location-hours-details-row-intervals"]')
+
+            if intervals.xpath('./text()').extract_first() == "Closed":
+                continue
+            if intervals.xpath('./span/text()').extract_first() == "Open 24 hours":
+                opening_hours.add_range(day=DAY_MAPPING[day],
+                                        open_time='0:00',
+                                        close_time='23:59')
+            else:
+                start_time = elem.xpath(
+                    './/span[@class="c-location-hours-details-row-intervals-instance-open"]/text()').extract_first()
+                end_time = elem.xpath(
+                    './/span[@class="c-location-hours-details-row-intervals-instance-close"]/text()').extract_first()
+                opening_hours.add_range(day=day[:2],
+                                        open_time=start_time,
+                                        close_time=end_time,
+                                        time_format='%H:%M %p')
+
+        return opening_hours.as_opening_hours()
+
+    def parse_store(self, response):
+        ref = re.search(r'.+/(.+)$', response.url).group(1)
+
+        address1 = response.xpath('//span[@class="c-address-street-1"]/text()').extract_first()
+        address2 = response.xpath('//span[@class="c-address-street-2"]/text()').extract_first() or ''
+
+        properties = {
+            'addr_full': " ".join([address1, address2]).strip(),
+            'phone': response.xpath('//span[@itemprop="telephone"]/text()').extract_first(),
+            'city': response.xpath('//span[@class="c-address-city"]/text()').extract_first(),
+            'state': response.xpath('//span[@itemprop="addressRegion"]/text()').extract_first(),
+            'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first(),
+            'country': response.xpath('//abbr[@itemprop="addressCountry"]/text()').extract_first(),
+            'ref': ref,
+            'website': response.url,
+            'lat': float(response.xpath('//meta[@itemprop="latitude"]/@content').extract_first()),
+            'lon': float(response.xpath('//meta[@itemprop="longitude"]/@content').extract_first()),
+            'name': response.xpath('//h1[@id="location-name"]/text()').extract_first()
+        }
+
+        hours = self.parse_hours(response.xpath('//table[@class="c-location-hours-details"]//tbody/tr'))
+
+        if hours:
+            properties['opening_hours'] = hours
+
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        urls = response.xpath('//a[@class="Directory-listLink"]/@href').extract()
+        is_store_list = response.xpath('//section[contains(@class,"LocationList")]').extract()
+
+        if not urls and is_store_list:
+            urls = response.xpath('//a[contains(@class,"Teaser-titleLink")]/@href').extract()
+
+        for url in urls:
+            if re.search(r'us/.{2}/.+/.+', url):
+                yield scrapy.Request(response.urljoin(url), callback=self.parse_store)
+            else:
+                yield scrapy.Request(response.urljoin(url))
diff --git a/locations/spiders/noahsnybagels.py b/locations/spiders/noahsnybagels.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/noahsnybagels.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import datetime
+import re
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+DAY_MAPPING = {
+    'Monday': 'Mo',
+    'Tuesday': 'Tu',
+    'Wednesday': 'We',
+    'Thursday': 'Th',
+    'Friday': 'Fr',
+    'Saturday': 'Sa',
+    'Sunday': 'Su'
+}
+
+class NoahsNYBagelsSpider(scrapy.Spider):
+    """Copy of Einstein Bros. Bagels - all brands of the same parent company Coffee & Bagels"""
+    name = "noahsnybagels"
+    allowed_domains = ["noahs.com"]
+    start_urls = (
+        'https://locations.noahs.com/us',
+    )
+
+    def parse_hours(self, elements):
+        opening_hours = OpeningHours()
+
+        for elem in elements:
+            day = elem.xpath('.//td[@class="c-location-hours-details-row-day"]/text()').extract_first()
+            intervals = elem.xpath('.//td[@class="c-location-hours-details-row-intervals"]')
+
+            if intervals.xpath('./text()').extract_first() == "Closed":
+                continue
+            if intervals.xpath('./span/text()').extract_first() == "Open 24 hours":
+                opening_hours.add_range(day=DAY_MAPPING[day],
+                                        open_time='0:00',
+                                        close_time='23:59')
+            else:
+                start_time = elem.xpath(
+                    './/span[@class="c-location-hours-details-row-intervals-instance-open"]/text()').extract_first()
+                end_time = elem.xpath(
+                    './/span[@class="c-location-hours-details-row-intervals-instance-close"]/text()').extract_first()
+                opening_hours.add_range(day=day[:2],
+                                        open_time=start_time,
+                                        close_time=end_time,
+                                        time_format='%H:%M %p')
+
+        return opening_hours.as_opening_hours()
+
+    def parse_store(self, response):
+        ref = re.search(r'.+/(.+)$', response.url).group(1)
+
+        address1 = response.xpath('//span[@class="c-address-street-1"]/text()').extract_first()
+        address2 = response.xpath('//span[@class="c-address-street-2"]/text()').extract_first() or ''
+
+        properties = {
+            'addr_full': " ".join([address1, address2]).strip(),
+            'phone': response.xpath('//span[@itemprop="telephone"]/text()').extract_first(),
+            'city': response.xpath('//span[@class="c-address-city"]/text()').extract_first(),
+            'state': response.xpath('//span[@itemprop="addressRegion"]/text()').extract_first(),
+            'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first(),
+            'country': response.xpath('//abbr[@itemprop="addressCountry"]/text()').extract_first(),
+            'ref': ref,
+            'website': response.url,
+            'lat': float(response.xpath('//meta[@itemprop="latitude"]/@content').extract_first()),
+            'lon': float(response.xpath('//meta[@itemprop="longitude"]/@content').extract_first()),
+            'name': response.xpath('//h1[@id="location-name"]/text()').extract_first()
+        }
+
+        hours = self.parse_hours(response.xpath('//table[@class="c-location-hours-details"]//tbody/tr'))
+
+        if hours:
+            properties['opening_hours'] = hours
+
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        urls = response.xpath('//a[@class="Directory-listLink"]/@href').extract()
+        is_store_list = response.xpath('//section[contains(@class,"LocationList")]').extract()
+
+        if not urls and is_store_list:
+            urls = response.xpath('//a[contains(@class,"Teaser-titleLink")]/@href').extract()
+
+        for url in urls:
+            if re.search(r'us/.{2}/.+/.+', url):
+                yield scrapy.Request(response.urljoin(url), callback=self.parse_store)
+            else:
+                yield scrapy.Request(response.urljoin(url))


Test patch: