Repository: alltheplaces/alltheplaces

Issue:
Best Buy
http://www.bestbuy.com/site/store-locator/


Discussion:
No progress for 6 days, re-opening this for others.
I had assigned it to myself on Friday night. I am done with Bestbuy US & Bestbuy Mexico. Working on Bestbuy Canada. @kristinepetrosyan are you working on this as well? I can push my code for US & Mexico if you want to work on it further.
Thanks for the update. In the future please only self assign when starting
work on an issue (not in anticipation of starting work).

On Mon, Dec 18, 2017 at 2:23 PM, Vikranth Katpally Reddy <
notifications@github.com> wrote:

> I had assigned it to myself on Friday night. I am done with Bestbuy US &
> Bestbuy Mexico. Working on Bestbuy Canada. @kristinepetrosyan
> <https://github.com/kristinepetrosyan> are you working on this as well? I
> can push my code for US & Mexico if you want to work on it further.
>
> —
> You are receiving this because you are subscribed to this thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/alltheplaces/alltheplaces/issues/21#issuecomment-352577117>,
> or mute the thread
> <https://github.com/notifications/unsubscribe-auth/AA0EO9isyTihmsMcem2bvfDJPTvKW0btks5tBuXKgaJpZM4Lj0lQ>
> .
>

Thank you for getting back. I did self-assign it after starting work. I'll continue and assign it to myself

Resolving patch:
diff --git a/locations/spiders/bestbuy.py b/locations/spiders/bestbuy.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/bestbuy.py
@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+from datetime import date
+
+
+from locations.items import GeojsonPointItem
+
+
+class BestBuySpider(scrapy.Spider):
+    name = "bestbuy"
+    allowed_domains = ["www.bestbuy.com"]
+    base_url = 'https://store.bestbuy.com/'
+    bb_url = 'https://www.bestbuy.com/browse-api/1.0/store-locator/raas?zipcode={}'
+
+    zip_code = '94040'
+    start_urls = (bb_url.format(zip_code), )
+    completed_requests = set()
+
+    def store_hours(self, store_hours):
+        day_groups = []
+        this_day_group = None
+        for i, line in enumerate(store_hours):
+            for key, value in line.items():
+                if key == 'date':
+                    match = re.search(r'^(\d{1,4})-(\d{1,2})-(\d{1,2})$', value)
+                    (y, m, d) = match.groups()
+                    day = date(int(y), int(m), int(d)).strftime('%A')
+                if key == 'open':
+                    hours = value
+                if key == 'close':
+                    hours += '-' + value
+            # Store hours has 2 weeks of data
+            if i >= 7:
+                break
+
+            if not this_day_group:
+                this_day_group = {
+                    'from_day': day,
+                    'to_day': day,
+                    'hours': hours
+                }
+            elif this_day_group['hours'] != hours:
+                day_groups.append(this_day_group)
+                this_day_group = {
+                    'from_day': day,
+                    'to_day': day,
+                    'hours': hours
+                }
+            elif this_day_group['hours'] == hours:
+                this_day_group['to_day'] = day
+
+        if this_day_group:
+            day_groups.append(this_day_group)
+
+        opening_hours = ""
+        if len(day_groups) == 1 and day_groups[0]['hours'] in ('00:00-23:59', '00:00-00:00'):
+            opening_hours = '24/7'
+        else:
+            for day_group in day_groups:
+                if day_group['from_day'] == day_group['to_day']:
+                    opening_hours += '{from_day} {hours}; '.format(**day_group)
+                elif day_group['from_day'] == 'Mo' and day_group['to_day'] == 'Su':
+                    opening_hours += '{hours}; '.format(**day_group)
+                else:
+                    opening_hours += '{from_day}-{to_day} {hours}; '.format(**day_group)
+            opening_hours = opening_hours[:-2]
+        return opening_hours
+
+    def parse(self, response):
+        json_str = response.body_as_unicode()
+        data = json.loads(json_str)['data']['stores']
+
+        bounding_box = {
+            'min_lat': 100,
+            'max_lat': -100,
+            'min_lon': 300,
+            'max_lon': -300,
+            'min_lat_zip': None,
+            'max_lat_zip': None,
+            'min_lon_zip': None,
+            'max_lon_zip': None,
+        }
+
+        for store in data:
+            (num, street) = store['addr1'].split(' ', 1)
+            properties = {
+                "phone": store['phone'],
+                "ref": store['id'],
+                "name": store['name'],
+                "opening_hours": self.store_hours(store['hours']),
+                "lat": store['latitude'],
+                "lon": store['longitude'],
+                "addr_full": store['addr1'],
+                "housenumber": num,
+                "street": street,
+                "city": store['city'],
+                "state": store['state'],
+                "postcode": store['zipCode'],
+                "country": store['country'],
+                "website": self.base_url + store['id'],
+            }
+            lat = float(store['latitude'])
+            lon = float(store['longitude'])
+            if lat < bounding_box['min_lat']:
+                bounding_box['min_lat'] = lat
+                bounding_box['min_lat_zip'] = store['zipCode']
+            if lat > bounding_box['max_lat']:
+                bounding_box['max_lat'] = lat
+                bounding_box['max_lat_zip'] = store['zipCode']
+            if lon < bounding_box['min_lon']:
+                bounding_box['min_lon'] = lon
+                bounding_box['min_lon_zip'] = store['zipCode']
+            if lon > bounding_box['max_lon']:
+                bounding_box['max_lon'] = lon
+                bounding_box['max_lon_zip'] = store['zipCode']
+
+            yield GeojsonPointItem(**properties)
+
+        for key, value in bounding_box.items():
+            if 'zip' in key and value is not None:
+                if value in self.completed_requests:
+                    self.logger.info("Skipping request for zipcode %s because we already did it", value)
+                else:
+                    self.completed_requests.add(value)
+                    yield scrapy.Request(self.bb_url.format(value))
diff --git a/locations/spiders/bestbuy_ca.py b/locations/spiders/bestbuy_ca.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/bestbuy_ca.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+from datetime import date
+from urllib.parse import urlparse, parse_qsl
+
+
+from locations.items import GeojsonPointItem
+
+
+class BestBuySpider(scrapy.Spider):
+    name = "bestbuy-ca"
+    allowed_domains = ["www.bestbuy.ca"]
+    base_url = 'https://www.bestbuy.ca/'
+    bb_url = 'https://www.bestbuy.ca/en-CA/secure/store-locator-view-all-stores.aspx'
+
+    start_urls = (bb_url, )
+    # print(start_urls)
+    completed_requests = set()
+
+    def store_hours(self, store_hours):
+        store_days = store_hours.xpath('./div[@class="day"]/text()').extract()
+        store_times = store_hours.xpath('./div[@class="hours"]/text()').extract()
+        day_groups = []
+        this_day_group = None
+
+        if not len(store_times):
+            store_times = ['00:00 AM - 00:00 AM' for i in range(7)]
+        else:
+            store_times = [' '.join(i.split(' ')[-5:]) for i in store_times]
+
+        print(store_days, store_times)
+        for i, day in enumerate(store_days):
+            day_hours = store_times[i]
+            if 'Closed' in day_hours:
+                hours = 'Closed'
+            elif 'Call store for hours' in day_hours:
+                hours = day_hours
+            else:
+                match = re.search(r'^^(\d{1,2})[:]?(\d{1,2})? (A|P)M - (\d{1,2})[:]?(\d{1,2})? (A|P)M$', day_hours)
+                (f_hr, f_min, f_ampm, t_hr, t_min, t_ampm) = match.groups()
+
+                f_hr = int(f_hr)
+                if f_ampm in ['p', 'P']:
+                    f_hr += 12
+                elif f_ampm in ['a', 'A'] and f_hr == 12:
+                    f_hr = 0
+                t_hr = int(t_hr)
+                if t_ampm in ['p', 'P']:
+                    t_hr += 12
+                elif t_ampm in ['a', 'A'] and t_hr == 12:
+                    t_hr = 0
+                if f_min is None:
+                    f_min = 0
+                if t_min is None:
+                    t_min = 0
+
+                hours = '{:02d}:00-{:02d}:00'.format(
+                    f_hr,
+                    t_hr,
+                )
+
+            if not this_day_group:
+                this_day_group = {
+                    'from_day': day,
+                    'to_day': day,
+                    'hours': hours
+                }
+            elif this_day_group['hours'] != hours:
+                day_groups.append(this_day_group)
+                this_day_group = {
+                    'from_day': day,
+                    'to_day': day,
+                    'hours': hours
+                }
+            elif this_day_group['hours'] == hours:
+                this_day_group['to_day'] = day
+
+        if this_day_group:
+            day_groups.append(this_day_group)
+
+        opening_hours = ""
+        if len(day_groups) == 1 and day_groups[0]['hours'] in ('00:00-23:59', '00:00-00:00'):
+            opening_hours = '24/7'
+        else:
+            for day_group in day_groups:
+                if day_group['from_day'] == day_group['to_day']:
+                    opening_hours += '{from_day} {hours}; '.format(**day_group)
+                elif day_group['from_day'] == 'Mo' and day_group['to_day'] == 'Su':
+                    opening_hours += '{hours}; '.format(**day_group)
+                else:
+                    opening_hours += '{from_day}-{to_day} {hours}; '.format(**day_group)
+            opening_hours = opening_hours[:-2]
+        return opening_hours
+
+    def parse_store(self, response):
+        store_details = response.xpath('//div[@class="searchresults"]/div[@class="store-details"]')
+
+        addr1 = store_details.xpath('ul[@class="litem"]/li[@class="store-address"]/text()').extract_first()
+        city_state_zip = store_details.xpath('ul[@class="litem"]/li[@class="store-address-details"]/text()')\
+            .extract_first().split('\r\n')
+        phone_num = store_details.xpath('ul[@class="litem"]/li')
+        for line in phone_num:
+            if line.xpath('./span[@class="strong"]/text()').extract_first() is not None:
+                phone = line.xpath('./text()').extract()[1].replace('\r\n', '').strip(' ')
+
+        (num, street) = addr1.split(' ', 1)
+        properties = {
+            "phone": phone,
+            "ref": response.url.split('=')[1],
+            "name": store_details.xpath('h1/text()').extract_first(),
+            "opening_hours": self.store_hours(store_details.xpath('ul[@class="litem store-hours"]/li')),
+            # "lat": store_details['latitude'],
+            # "lon": store_details['longitude'],
+            "addr_full": addr1,
+            "housenumber": num,
+            "street": street,
+            "city": city_state_zip[0].strip(' ').rstrip(','),
+            "state": city_state_zip[1].strip(' ').rstrip(','),
+            "postcode": city_state_zip[2].strip(' '),
+            "country": 'Canada',
+            "website": response.url,
+        }
+
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        data = response.xpath('//ul[@class="storeListProvinceWrap"]//div[@class="litem"]/a/@href').extract()
+
+        for store in data:
+            yield scrapy.Request(self.base_url + store, callback=self.parse_store)
+        # yield scrapy.Request(self.base_url + data[0], callback=self.parse_store)
+
diff --git a/locations/spiders/bestbuy_mx.py b/locations/spiders/bestbuy_mx.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/bestbuy_mx.py
@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+from datetime import date
+from urllib.parse import urlparse, parse_qsl
+
+
+from locations.items import GeojsonPointItem
+
+
+class BestBuyMexicoSpider(scrapy.Spider):
+    name = "bestbuy-mx"
+    allowed_domains = ["www.bestbuy.com.mx",]
+    bb_url = 'http://www.bestbuy.com.mx/storelocator/api/{}'
+
+    starting_zip_codes = ['06010', '97130', '44840', '66260', '94294', '55076', ]
+    bb_urls = list()
+    for z in starting_zip_codes:
+        bb_urls.append(bb_url.format(z))
+    start_urls = tuple(bb_urls)
+    completed_requests = set()
+
+    def store_hours(self, store_hours):
+        day_groups = []
+        this_day_group = None
+        for i, line in enumerate(store_hours):
+            for key, value in line.items():
+                if key == 'dateOpen':
+                    match = re.search(r'^(\d{1,4})-(\d{1,2})-(\d{1,2})$', value)
+                    (y, m, d) = match.groups()
+                    day = date(int(y), int(m), int(d)).strftime('%A')
+                if key == 'timeOpen':
+                    hours = value
+                if key == 'timeClosed':
+                    hours += '-' + value
+            # Store hours has 2 weeks of data
+            if i >= 7:
+                break
+
+            if not this_day_group:
+                this_day_group = {
+                    'from_day': day,
+                    'to_day': day,
+                    'hours': hours
+                }
+            elif this_day_group['hours'] != hours:
+                day_groups.append(this_day_group)
+                this_day_group = {
+                    'from_day': day,
+                    'to_day': day,
+                    'hours': hours
+                }
+            elif this_day_group['hours'] == hours:
+                this_day_group['to_day'] = day
+
+        if this_day_group:
+            day_groups.append(this_day_group)
+
+        opening_hours = ""
+        if len(day_groups) == 1 and day_groups[0]['hours'] in ('00:00-23:59', '00:00-00:00'):
+            opening_hours = '24/7'
+        else:
+            for day_group in day_groups:
+                if day_group['from_day'] == day_group['to_day']:
+                    opening_hours += '{from_day} {hours}; '.format(**day_group)
+                elif day_group['from_day'] == 'Mo' and day_group['to_day'] == 'Su':
+                    opening_hours += '{hours}; '.format(**day_group)
+                else:
+                    opening_hours += '{from_day}-{to_day} {hours}; '.format(**day_group)
+            opening_hours = opening_hours[:-2]
+        return opening_hours
+
+    def parse(self, response):
+        json_str = response.body_as_unicode()
+        data = json.loads(json_str)['data']['stores']
+
+        bounding_box = {
+            'min_lat': 100,
+            'max_lat': -100,
+            'min_lon': 300,
+            'max_lon': -300,
+            'min_lat_zip': None,
+            'max_lat_zip': None,
+            'min_lon_zip': None,
+            'max_lon_zip': None,
+        }
+
+        for store in data:
+            # Door# is at the end in address format in Spanish
+            parts = store['addr1'].split(' ')
+            num = ''
+            for part in parts:
+                try:
+                    num = '%d' % int(part)
+                except ValueError:
+                    pass
+            # strip out the door number from address for street name
+            street = store['addr1'].replace(num, '')
+            properties = {
+                "phone": store['phone'],
+                "ref": store['id'],
+                "name": store['name'],
+                "opening_hours": self.store_hours(store['hours']),
+                "lat": store['latitude'],
+                "lon": store['longitude'],
+                "addr_full": store['addr1'] + ', ' + store['addr2'],
+                "housenumber": num,
+                "street": street,
+                "city": store['city'],
+                "state": store['state'],
+                "postcode": store['postalCode'],
+                "country": store['country'],
+                "website": store['links']['self']['href'],
+            }
+            lat = float(store['latitude'])
+            lon = float(store['longitude'])
+            if lat < bounding_box['min_lat']:
+                bounding_box['min_lat'] = lat
+                bounding_box['min_lat_zip'] = store['postalCode']
+            if lat > bounding_box['max_lat']:
+                bounding_box['max_lat'] = lat
+                bounding_box['max_lat_zip'] = store['postalCode']
+            if lon < bounding_box['min_lon']:
+                bounding_box['min_lon'] = lon
+                bounding_box['min_lon_zip'] = store['postalCode']
+            if lon > bounding_box['max_lon']:
+                bounding_box['max_lon'] = lon
+                bounding_box['max_lon_zip'] = store['postalCode']
+
+            yield GeojsonPointItem(**properties)
+
+        for key, value in bounding_box.items():
+            if 'zip' in key and value is not None:
+                if value in self.completed_requests:
+                    self.logger.info("Skipping request for zipcode %s because we already did it", value)
+                else:
+                    self.completed_requests.add(value)
+                    yield scrapy.Request(self.bb_url.format(value))
+


Test patch: