Repository: alltheplaces/alltheplaces

Issue:
McMenamins
http://www.mcmenamins.com/Pubs


Discussion:
taken

Resolving patch:
diff --git a/locations/spiders/mcmenamins.py b/locations/spiders/mcmenamins.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/mcmenamins.py
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+
+from locations.items import GeojsonPointItem
+
+DAYS={
+    'Monday':'Mo',
+    'Tuesday':'Tu',
+    'Wednesday':'We',
+    'Friday':'Fr',
+    'Thursday':'Th',
+    'Saturday':'Sa',
+    'Sunday':'Su',
+}
+
+class McmenaminsSpider(scrapy.Spider):
+    name = "mcmenamins"
+    allowed_domains = ["mcmenamins.com","google.com"]
+    start_urls = (
+        'https://www.mcmenamins.com/eat-drink',
+    )
+
+    def store_hours(self, store_hours):
+        result=''
+        for line in store_hours:
+            sl=re.search(r'(\w+)\s*(through|and|-|&|–)?\s*(\w+)?,\s*(\d+):?(\d+)?\s*((a\.m\.)|(p\.m\.)|noon|midnight)\s*\'til\s*(\d+):?(\d+)?\s*((a\.m\.)|(p\.m\.)|noon|midnight)',line)
+
+            if not sl:
+                continue
+            if sl[3]=='daily' or sl[1]== 'daily':
+                result+='Mo-Su '
+            else:
+                result+=DAYS[sl[1]]+(('-'+DAYS[sl[3]]) if sl[3] else '')+' '
+
+            i=int(sl[4])
+
+            if sl[6]=='a.m.' or sl[6]=='noon':
+                result+='{:02d}'.format(int(sl[4]))
+            else:
+                result+='{:02d}'.format(i+12)
+
+            if sl[5]:
+                result+=':'+sl[5]+'-'
+            else:
+                result+=':00'+'-'
+
+            i=int(sl[9])
+
+            if sl[11]=='midnight':
+                result+='00'
+            elif sl[9]=='a.m.' or sl[6]=='noon':
+                result+='{:02d}'.format(int(sl[9]))
+            else:
+                result+='{:02d}'.format(i+12)
+
+            if sl[10]:
+                result+=':'+sl[10]+'; '
+            else:
+                result+=':00'+'; '
+
+        return result.rstrip('; ')
+
+    def phone_normalize(self, phone):
+        r=re.search(r'\+?(\s+)*(\d{1})?(\s|\()*(\d{3})(\s+|\))*(\d{3})(\s+|-)?(\d{2})(\s+|-)?(\d{2})',phone)
+        return ('('+r.group(4)+') '+r.group(6)+'-'+r.group(8)+'-'+r.group(10)) if r else phone
+
+    def parse(self, response): #high-level list of states
+        shops = response.xpath('//div[@id="MainContent_eatDrinkLocations"]/div[contains(@class,"all")]')
+        i=1
+        for path in shops:
+            if i<4:
+                yield scrapy.Request(response.urljoin(path.xpath('.//div/div[@class="tm-panel-titlebg"]/a/@href').extract_first()), callback=self.parse_store, meta={
+                       'ref':path.xpath('.//@id').extract_first(),
+                        })
+
+    def parse_store(self, response):
+        google_pos=response.xpath('//div[@class="mcm-logo-address"]')[0].xpath('.//a[contains(@href,"maps.google")]/@href')[0].extract()
+        address_full=response.xpath('//div[@class="mcm-logo-address"]')[0].xpath('.//a/p/text()').extract_first()
+        address_parts=re.match(r"(.{3,}),\s?(.{3,}),\s?(\w{2}) (\d{5})",address_full)
+        yield scrapy.Request(google_pos, callback=self.google_maps, meta={
+                'ref':response.meta.get('ref'),
+                'website':response.url,
+                'addr_full':address_parts[1].strip(),
+                'city':address_parts[2].strip(),
+                'state':address_parts[3].strip(),
+                'postcode':address_parts[4].strip(),
+                'phone':self.phone_normalize(response.xpath('//div[@class="mcm-logo-address"]')[0].xpath('.//ul/li/a/@href').extract_first()),
+                'open_hours':self.store_hours(response.xpath('//div[@id="MainContent_hoursText"]/p/text()').extract()),
+                })
+
+    def google_maps(self, response):
+        sc=response.xpath('//script[contains(.,"cacheResponse([")]').extract_first()
+        latlon=json.loads(re.search(r'cacheResponse\((.*)\);',sc)[1])[8][0][2]
+        yield GeojsonPointItem(
+            lat=float(latlon[0]),
+            lon=float(latlon[1]),
+            phone=response.meta.get('phone'),
+            website=response.meta.get('website'),
+            ref=response.meta.get('ref'),
+            opening_hours=response.meta.get('open_hours'),
+            addr_full=response.meta.get('addr_full'),
+            city=response.meta.get('city'),
+            state=response.meta.get('state'),
+            postcode=response.meta.get('postcode'),
+            country='US',
+        )


Test patch: