Repository: alltheplaces/alltheplaces

Issue:
Spider next_uk is broken
During the global build at 2021-06-23-14-42-18, spider **next_uk** failed with **0 features** and **0 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-06-23-14-42-18/logs/next_uk.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-06-23-14-42-18/output/next_uk.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-06-23-14-42-18/output/next_uk.geojson))


Discussion:

Resolving patch:
diff --git a/locations/spiders/next_uk.py b/locations/spiders/next_uk.py
--- a/locations/spiders/next_uk.py
+++ b/locations/spiders/next_uk.py
@@ -4,6 +4,7 @@
 import re
 
 from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
 
 DAYS = ['Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa', 'Su']
 
@@ -17,42 +18,25 @@ class NextSpider(scrapy.Spider):
     )
 
     def store_hours(self, store_hours):
-        lastday = DAYS[0]
-        lasttime = store_hours[DAYS[0]]
-        opening_hours = lastday
-
-        if not len(reduce(lambda x, y: x+y, store_hours)):
-            return ''
-
-        for day in range(1, 7):  # loop by days
-            if day == len(store_hours):
-                break
-
-            str_curr = store_hours[DAYS[day]]
-
-            if str_curr != lasttime:
-                if lastday == DAYS[day-1]:
-                    opening_hours += ' '+lasttime+';'+DAYS[day]
-                else:
-                    opening_hours += '-'+DAYS[day-1]+' '+lasttime+';'+DAYS[day]
-                lasttime = str_curr
-                lastday = DAYS[day]
-        if lasttime != '':
-            if lastday == DAYS[day]:
-                opening_hours += ' '+str_curr
+        o = OpeningHours()
+
+        for day in DAYS:
+            open_time, close_time = store_hours[day].split("-")
+            open_time = open_time.rstrip()
+            close_time = close_time.rstrip()
+            if (len(open_time) < 3 or len(open_time) > 4) and (len(close_time) < 3 or len(close_time) > 4):
+                pass
             else:
-                opening_hours += '-'+DAYS[6]+' '+str_curr
-        else:
-            opening_hours = opening_hours.rstrip(DAYS[6])
+                o.add_range(day=day, open_time=open_time, close_time=close_time, time_format='%H%M')
 
-        return opening_hours.rstrip(';').strip()
+        return o.as_opening_hours()
 
     def parse(self, response):
         countries = response.xpath('//select[@id="country-store"]/option/@value')
         for country in countries:
             req = {"country": country.extract()}
             yield scrapy.FormRequest(
-                "http://stores.next.co.uk/index/stores",
+                "https://stores.next.co.uk/index/stores",
                 formdata=req,
                 method="POST",
                 callback=self.parse_city
@@ -62,7 +46,7 @@ def parse_city(self, response):
         cities = response.xpath('//option/@value')
         for city in cities:
             yield scrapy.Request(
-                'http://stores.next.co.uk/stores/single/'+city.extract(),
+                'http://stores.next.co.uk/stores/single/' + city.extract(),
                 callback=self.parse_shop
             )
 
@@ -70,7 +54,7 @@ def parse_shop(self, response):
         text = response.xpath('//script[contains(.,"window.lctr.single_search")]/text()').extract_first()
         parts = text.split("window.lctr.results.push(")
 
-        for place in range(1, len(parts)):
+        for place in range(1, 2):
             shop = json.loads(parts[place].strip().rstrip(';').rstrip(')'))
             try:
                 hours = self.store_hours({
@@ -85,24 +69,40 @@ def parse_shop(self, response):
             except Exception:
                 hours = ''
 
-            props = {
-                'lat': float(shop['Latitude']),
-                'lon': float(shop['Longitude']),
-                'phone': shop['telephone'],
-                'website': response.url,
+            properties = {
                 'ref': shop['location_id'],
-                'addr_full': shop['street'],
+                'name': shop['branch_name'],
+                'addr_full': shop['AddressLine'] + ',' + shop['street'],
                 'city': shop['city'],
                 'country': shop['country'],
+                'lat': float(shop['Latitude']),
+                'lon': float(shop['Longitude']),
+                'phone': shop['telephone'],
+                'website': response.url
             }
 
             if hours:
-                props['opening_hours'] = hours
+                properties['opening_hours'] = hours
             if shop['PostalCode']:
-                props['postcode'] = shop['PostalCode']
-            if shop['county']:
-                props['state'] = shop['county']
+                properties['postcode'] = shop['PostalCode']
             if shop['county']:
-                props['state'] = shop['county']
+                properties['state'] = shop['county']
+
+            if shop['AddressLine'] and shop['centre'] and shop['street']:
+                properties['addr_full'] = shop['AddressLine'] + ', ' + shop['centre'] + ', ' + shop['street']
+            elif shop['street'] and shop['centre']:
+                properties['addr_full'] = shop['centre'] + ',' + shop['street']
+            elif shop['AddressLine'] and shop['centre']:
+                properties['addr_full'] = shop['AddressLine'] + ', ' + shop['centre']
+            elif shop['street']:
+                properties['addr_full'] = shop['street']
+            elif shop['centre']:
+                properties['addr_full'] = shop['centre']
+            elif shop['AddressLine']:
+                properties['addr_full'] = shop['AddressLine']
+            elif shop['town']:
+                properties['addr_full'] = shop['town']
+            else:
+                properties['addr_full'] = ""
 
-            yield GeojsonPointItem(**props)
+            yield GeojsonPointItem(**properties)


Test patch: