Repository: alltheplaces/alltheplaces

Issue:
Spider uhaul is broken
During the global build at 2021-06-30-14-42-26, spider **uhaul** failed with **0 features** and **0 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-06-30-14-42-26/logs/uhaul.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-06-30-14-42-26/output/uhaul.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-06-30-14-42-26/output/uhaul.geojson))


Discussion:

Resolving patch:
diff --git a/locations/spiders/uhaul.py b/locations/spiders/uhaul.py
--- a/locations/spiders/uhaul.py
+++ b/locations/spiders/uhaul.py
@@ -1,13 +1,9 @@
 # -*- coding: utf-8 -*-
+import json
 import scrapy
 import re
 from locations.items import GeojsonPointItem
-
-STATES = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
-          'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD',
-          'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ',
-          'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC',
-          'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY']
+from locations.hours import OpeningHours
 
 
 class UhaulSpider(scrapy.Spider):
@@ -15,68 +11,76 @@ class UhaulSpider(scrapy.Spider):
     item_attributes = { 'brand': "U-Haul" }
     allowed_domains = ["www.uhaul.com"]
 
-    def start_requests(self):
-        for state in STATES:
-            url = 'https://www.uhaul.com/Locations/{}/Results/?page=1'.format(
-                state
-            )
-            yield scrapy.Request(url)
+    start_urls = (
+        'https://www.uhaul.com/Locations/US_and_Canada/',
+    )
 
     def parse(self, response):
-        sections = response.css('#locationsResults .divider')
-        if sections:
-            next = sections[-1].css('#locationSearchNextLocations::attr(href)')
-
-            for section in sections[:-1]:
-                name = section.css('.map-location a::text').extract_first()
-                website = section.css('.map-location a::attr(href)').extract_first()
-                ref = website.split('/')[-2]
-                addr_data = section.css('.row .medium-8.columns .collapse::text').extract()
-                phone = section.css('.row .medium-8.columns .no-bullet li a::attr(href)').extract_first()[4:]
-                street = addr_data[1].strip()
-                city, state, zipcode = re.search(r'(.*), (.*) (\d+)', addr_data[2]).groups()
-                hours_data = section.css('.row .medium-6.columns .row .medium-6.columns .no-bullet')[0]
-
-                properties = {
-                    'ref': ref,
-                    'name': name,
-                    'website': website,
-                    'street': street,
-                    'opening_hours': self.hours(hours_data),
-                    'city': city.strip(),
-                    'state': state,
-                    'postcode': zipcode,
-                    'phone': phone,
-                }
-
-                yield scrapy.Request(
-                    'https://www.uhaul.com/Locations/Directions-to-{}/'.format(ref),
-                    callback=self.parse_directions,
-                    meta={'properties': properties},
-                )
-
-            if next:
-                yield scrapy.Request(
-                    'https://www.uhaul.com' + next.extract_first(),
-                    callback=self.parse
-                )
+        for cell in response.xpath('//ul/li[@class="cell"]/a/@href'):
+            yield scrapy.Request(
+                url=response.urljoin(cell.extract()),
+            )
 
-    def parse_directions(self, response):
-        script_str = response.xpath('//script')[-2].extract()
-        matches = re.search(r'"lat":([\-\d\.]*),"long":([\-\d\.]*),', script_str)
-        lat, lon = matches.groups() if matches else (None, None)
+        for store_nav in response.xpath('//ul/li/ul[@class="sub-nav"]'):
+            # Each store nav can have multiple services, each with a link under the sub-nav ul.
+            # We want to pick the first one to get to a store details page.
+            store_url = store_nav.xpath('.//a/@href').extract_first()
 
-        properties = response.meta['properties']
-        properties['lat'] = lat
-        properties['lon'] = lon
+            yield scrapy.Request(
+                url=response.urljoin(store_url),
+                callback=self.parse_store,
+            )
 
-        yield GeojsonPointItem(**properties)
+    def parse_store(self, response):
+        store_obj = None
+        for script in response.xpath('//script[@type="application/ld+json"]/text()').extract():
+            tmp_obj = json.loads(script)
+            ldjson_type = tmp_obj.get('@type')
+            if ldjson_type in ('SelfStorage', 'LocalBusiness'):
+                store_obj = tmp_obj
+                break
+
+        if store_obj is None:
+            return
+
+        if ldjson_type == 'SelfStorage':
+            ref = store_obj['url'].split('/')[-2]
+        elif ldjson_type == 'LocalBusiness':
+            ref = store_obj['@id'].split('/')[-2]
+
+        telephone = store_obj.get('telephone') or store_obj.get('contactPoint', {}).get('telephone')
+        hour_elements = response.xpath('//div[@class="callout flat radius hide-for-native"]/ul/li[@itemprop="openinghours"]/@datetime')
+
+        properties = {
+            'ref': ref,
+            'name': store_obj.get('name'),
+            'addr_full': store_obj.get('address', {}).get('streetAddress').strip(),
+            'city': store_obj.get('address', {}).get('addressLocality').strip(),
+            'state': store_obj.get('address', {}).get('addressRegion'),
+            'postcode': store_obj.get('address', {}).get('postalCode'),
+            'country': store_obj.get('address', {}).get('addressCountry'),
+            'phone': telephone,
+            'lat': store_obj.get('geo', {}).get('latitude'),
+            'lon': store_obj.get('geo', {}).get('longitude'),
+            'opening_hours': self.hours(hour_elements),
+        }
+
+        if properties['lat'] and properties['lon']:
+            # Can skip the call to the next one if this page happened to have lat/lon
+            yield GeojsonPointItem(**properties)
+
+        yield scrapy.Request(
+            url='https://www.uhaul.com/Locations/Directions-to-%s/' % ref,
+            callback=self.parse_directions,
+            meta={
+                'properties': properties,
+            },
+        )
 
     def hours(self, store_hours):
-        store_hours = store_hours.css('li::text').extract()
-        opening_hours = ''
+        opening_hours = []
 
-        for data in store_hours:
+        for data in store_hours.extract():
             day_text = data.split(':')[0]
             hours_text = data.split(':')[1]
 
@@ -94,9 +98,9 @@ def hours(self, store_hours):
             else:
                 hours = 'Closed'
 
-            opening_hours = opening_hours + '{} {}; '.format(day, hours)
+            opening_hours.append('{} {}'.format(day, hours))
 
-        return opening_hours
+        return '; '.join(opening_hours)
 
     def normalize_time(self, time_str, open):
         match = re.search(r'([0-9]{1,2}):([0-9]{1,2})', time_str)
@@ -111,3 +115,14 @@ def normalize_time(self, time_str, open):
             int(h) + 12 if not open else int(h),
             int(m),
         )
+
+    def parse_directions(self, response):
+        script_str = response.xpath('//script')[-1].extract()
+        matches = re.search(r'"lat":([\-\d\.]*),"long":([\-\d\.]*),', script_str)
+        lat, lon = matches.groups() if matches else (None, None)
+
+        properties = response.meta['properties']
+        properties['lat'] = lat
+        properties['lon'] = lon
+
+        yield GeojsonPointItem(**properties)


Test patch: