Repository: alltheplaces/alltheplaces

Issue:
Bloomingdale's
https://locations.bloomingdales.com/index.html


Discussion:

Resolving patch:
diff --git a/locations/spiders/bloomingdales.py b/locations/spiders/bloomingdales.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/bloomingdales.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+DAY_MAPPING = {
+    'SUNDAY': 'Su',
+    'MONDAY': 'Mo',
+    'TUESDAY': 'Tu',
+    'WEDNESDAY': 'We',
+    'THURSDAY': 'Th',
+    'FRIDAY': 'Fr',
+    'SATURDAY': 'Sa',
+}
+
+
+class BloomingdalesSpider(scrapy.Spider):
+    name = 'bloomingdales'
+    allowed_domains = ['bloomingdales.com']
+    download_delay = 0.2
+    start_urls = (
+        'https://locations.bloomingdales.com/',
+    )
+
+    def parse_hours(self, hours):
+        hours = json.loads(hours)
+        opening_hours = OpeningHours()
+
+        for hour in hours:
+            if hour["intervals"]:
+                opening_hours.add_range(day=DAY_MAPPING[hour["day"]],
+                                        open_time=str(hour["intervals"][0]["start"]),
+                                        close_time=str(hour["intervals"][0]["end"]),
+                                        time_format='%H%M')
+
+        return opening_hours.as_opening_hours()
+
+    def parse(self, response):
+        urls = response.xpath('//li[@class="c-LocationGridList-item"]//a[contains(text(), "Website")]/@href').extract()
+
+        for url in urls:
+            yield scrapy.Request(response.urljoin(url), callback=self.parse_store)
+
+    def parse_store(self, response):
+
+        brand = response.xpath('//span[@class="LocationName-brand"]/text()').extract_first()
+        name = response.xpath('//span[@class="LocationName-geo"]/text()').extract_first()
+
+        properties = {
+            'name': brand + ' ' + name,
+            'addr_full': response.xpath('//span[contains(@class, "c-address-street-1")]/text()').extract_first(),
+            'city': response.xpath('//span[@itemprop="addressLocality"]/text()').extract_first(),
+            'state': response.xpath('//abbr[@itemprop="addressRegion"]/text()').extract_first(),
+            'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first().strip(),
+            'country': response.xpath('//abbr[@itemprop="addressCountry"]/text()').extract_first(),
+            'ref': re.search(r'.*/(.*)$', response.url).groups()[0],
+            'lat': response.xpath('//span[@class="coordinates"]/meta[@itemprop="latitude"]/@content').extract_first(),
+            'lon': response.xpath('//span[@class="coordinates"]/meta[@itemprop="longitude"]/@content').extract_first(),
+            'phone': response.xpath('//span[@itemprop="telephone"]/text()').extract_first(),
+            'website': response.url
+        }
+
+        hours = self.parse_hours(response.xpath('//span[contains(@class, "c-location-hours-today")]/@data-days').extract_first())
+
+        if hours:
+            properties["opening_hours"] = hours
+
+        yield GeojsonPointItem(**properties)
+
diff --git a/locations/spiders/macys.py b/locations/spiders/macys.py
--- a/locations/spiders/macys.py
+++ b/locations/spiders/macys.py
@@ -1,7 +1,21 @@
 # -*- coding: utf-8 -*-
 import scrapy
 import json
+import re
 from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+DAY_MAPPING = {
+    'SUNDAY': 'Su',
+    'MONDAY': 'Mo',
+    'TUESDAY': 'Tu',
+    'WEDNESDAY': 'We',
+    'THURSDAY': 'Th',
+    'FRIDAY': 'Fr',
+    'SATURDAY': 'Sa',
+}
+
 
 class MacysSpider(scrapy.Spider):
     name = 'macys'
@@ -11,72 +25,53 @@ class MacysSpider(scrapy.Spider):
         'https://l.macys.com/',
     )
 
+    def parse_hours(self, hours):
+        hours = json.loads(hours)
+        opening_hours = OpeningHours()
+
+        for hour in hours:
+            if not hour["isClosed"]:
+                opening_hours.add_range(day=DAY_MAPPING[hour["day"]],
+                                        open_time=str(hour["intervals"][0]["start"]),
+                                        close_time=str(hour["intervals"][0]["end"]),
+                                        time_format='%H%M')
+
+        return opening_hours.as_opening_hours()
+
+
     def parse(self, response):
-        states = response.xpath('//div[@class="c-directory-list"]/div/ul/li/a/@href').extract()
-        for state in states:
-            yield scrapy.Request(response.urljoin(state), callback=self.parse_state)
-
-    def parse_state(self, response):
-        # States with few stores have listings in this page.
-        stores = response.xpath('//div[@class="c-location-grid-item"]')
-        for store in stores:
-          yield self.parse_info(store, response)
-        # States with several stores list cities.
-        cities = response.xpath('//a[@class="c-directory-list-content-item-link"]/@href').extract()
-        for city in cities:
-          yield scrapy.Request(response.urljoin(city), callback=self.parse_city)
-    
-    def parse_city(self, response):
-        stores = response.xpath('//div[@class="c-location-grid-item"]')
-        if len(stores) > 0:
-            # Muliple stores on the page.
-            for store in stores:
-              yield self.parse_info(store, response)
-        else:
-            # Only one store on the page.
-            store = response.xpath('//div[@id="location-info"]')
-            address = store.xpath('./address/span/span[@class="c-address-street-1"]/text()').extract_first()
-            city = store.xpath('./address/span/span[@itemprop="addressLocality"]/text()').extract_first()
-            state = store.xpath('./address/abbr[@itemprop="addressRegion"]/text()').extract_first()
-            postalCode = store.xpath('./address/span[@itemprop="postalCode"]/text()').extract_first().strip()
-            country = store.xpath('./address/abbr[@itemprop="addressCountry"]/text()').extract_first()
-            hours = ', '.join(store.xpath('./div[@class="c-location-hours"]/div/table/tbody/tr/@content').extract())
-            phone = store.xpath('./div[@class="c-location-info-phone"]/span/text()').extract_first()
-            prpoerties = {
-                'addr_full': '%s, %s, %s %s' % (address, city, state, postalCode),
-                'city': city,
-                'state': state,
-                'postcode': postalCode,
-                'country': country,
-                'ref': address,
-                'lat': store.xpath('./span[@class="coordinates"]/meta[@itemprop="latitude"]/@content').extract_first(),
-                'lon': store.xpath('./span[@class="coordinates"]/meta[@itemprop="longitude"]/@content').extract_first(),
-                'phone': phone,
-                'website': response.url,
-                'opening_hours': hours,
-                'extras': { 'name': response.xpath('//h1/text()').extract_first() } 
-            }
-            yield GeojsonPointItem(**prpoerties)
-    
-    def parse_info(self, store, response):
-        address = store.xpath('./div/address/span/span[@class="c-address-street-1"]/text()').extract_first().strip()
-        city = store.xpath('./div/address/span/span[@itemprop="addressLocality"]/text()').extract_first()
-        state = store.xpath('./div/address/abbr[@itemprop="addressRegion"]/text()').extract_first()
-        postalCode = store.xpath('./div/address/span[@itemprop="postalCode"]/text()').extract_first().strip()
-        country = store.xpath('./div/address/abbr[@itemprop="addressCountry"]/text()').extract_first()
-        hours = ', '.join(store.xpath('./div[@class="c-location-grid-item-hours-today"]/div/div/@content').extract())
+        urls = response.xpath('//div[@class="Directory-content"]//li/a/@href').extract()
+
+        if not urls:
+            urls = response.xpath('//div[@class="Directory-content"]//li//a[contains(text(), "Store Details")]/@href').extract()
+        for url in urls:
+            if '.html' not in url:
+                yield scrapy.Request(response.urljoin(url), callback=self.parse_store)
+            else:
+                yield scrapy.Request(response.urljoin(url))
+
+    def parse_store(self, response):
+
+        brand = response.xpath('//span[@class="LocationName-brand"]/text()').extract_first()
+        name = response.xpath('//span[@class="LocationName-geo"]/text()').extract_first()
+
         properties = {
-            'addr_full': '%s, %s, %s %s' % (address, city, state, postalCode),
-            'city': city,
-            'state': state,
-            'postcode': postalCode,
-            'country': country,
-            'ref': store.xpath('./div[@class="c-location-grid-item-link-wrapper"]/a[@data-ga-category="Get Directions"]/@href').extract_first().split('cid=')[-1],
-            'lat': store.xpath('./div/span/meta[@itemprop="latitude"]/@content').extract_first(),
-            'lon': store.xpath('./div/span/meta[@itemprop="longitude"]/@content').extract_first(),
-            'phone': store.xpath('./div[@class="c-location-grid-item-phone"]/div/span/text()').extract_first(),
-            'website': response.urljoin(store.xpath('./h5/a/@href').extract_first()),
-            'opening_hours': hours,
-            'extras': { 'name': store.xpath('./h5/a/text()').extract_first() }
+            'name': brand + ' ' + name,
+            'addr_full': response.xpath('//meta[@itemprop="streetAddress"]/@content').extract_first(),
+            'city': response.xpath('//meta[@itemprop="addressLocality"]/@content').extract_first(),
+            'state': response.xpath('//abbr[@itemprop="addressRegion"]/text()').extract_first(),
+            'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first(),
+            'country': response.xpath('//abbr[@itemprop="addressCountry"]/text()').extract_first(),
+            'ref': re.search(r'.*/(.*)$', response.url).groups()[0],
+            'lat': response.xpath('//span[@class="coordinates"]/meta[@itemprop="latitude"]/@content').extract_first(),
+            'lon': response.xpath('//span[@class="coordinates"]/meta[@itemprop="longitude"]/@content').extract_first(),
+            'phone': response.xpath('//div[@itemprop="telephone"]//a/text()').extract_first(),
+            'website': response.url,
         }
-        return GeojsonPointItem(**properties)
+        hours = self.parse_hours(response.xpath('//span[contains(@class, "c-hours-today")]/@data-days').extract_first())
+
+        if hours:
+            properties["opening_hours"] = hours
+
+        yield GeojsonPointItem(**properties)
+


Test patch: