internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: alltheplaces/alltheplaces

Issue:
Boots
Boots (without an apostrophe) is a pharmacy chain in the UK.

https://www.boots.com/store-locator



Discussion:

Resolving patch:
diff --git a/locations/spiders/boots.py b/locations/spiders/boots.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/boots.py
@@ -0,0 +1,47 @@
+import scrapy
+import re
+import json
+from locations.items import GeojsonPointItem
+class ArgosSpider(scrapy.Spider):
+
+    name = "boots"
+    allowed_domains = ["www.boots.com"]
+    download_delay = 0.5
+    start_urls = (
+        'http://www.boots.com/store-a-z',
+    )
+    def parse_hours(self, lis):
+        hours = []
+        for li in lis:
+            day = li.xpath('normalize-space(./td[@class="store_hours_day"]/text())').extract_first()
+            times = li.xpath('normalize-space(./td[@class="store_hours_time"]/text())').extract_first()
+            if times and day:
+                hours.append(day[:2] + ' ' + times)
+
+        return "; ".join(hours)
+
+    def parse_stores(self, response):
+        addr_full = response.xpath('//section[@class="store_details_content rowContainer"]/dl[@class="store_info_list"][1]/dd[@class="store_info_list_item"]/text()').extract()
+        if(len(addr_full)<3):
+            return
+        properties = {
+            'addr_full':addr_full[0],
+            'phone': response.xpath('//section[@class="store_details_content rowContainer"]/dl[@class="store_info_list"][3]/dd[@class="store_info_list_item"]/a/text()').extract_first(),
+            'city':addr_full[1],
+            'state': addr_full[2],
+            'postcode':addr_full[3],
+            'country': 'United Kingdom',
+            'ref': re.findall(r"[^\/]+$" ,response.url)[0],
+            'website': response.url,
+            'lat': float(response.xpath('normalize-space(//input[@id="lat"]/@value)').extract_first()),
+            'lon': float(response.xpath('normalize-space(//input[@id="lon"]/@value)').extract_first()),
+        }
+        hours = self.parse_hours(response.xpath('//div[@class="row store_all_opening_hours"]/div[1]/table[@class="store_opening_hours "]/tbody/tr'))
+        if hours:
+            properties['opening_hours'] = hours
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        urls = response.xpath('//div[@class="brand_list_viewer"]/div[@class="column"]/ul/li/a/@href').extract()
+        for path in urls:
+            yield scrapy.Request(response.urljoin(path), callback=self.parse_stores)
\ No newline at end of file


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Carphone Warehouse
Carphone Warehouse is an electronics store chain in the UK which specialises in mobile phones.

https://www.carphonewarehouse.com/store-locator.html



Discussion:

Resolving patch:
diff --git a/locations/spiders/carphone_warehouse.py b/locations/spiders/carphone_warehouse.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/carphone_warehouse.py
@@ -0,0 +1,61 @@
+import json
+import re
+import scrapy
+from locations.items import GeojsonPointItem
+DAYS = {
+    'monday': 'Mo',
+    'tuesday': 'Tu',
+    'wednesday': 'We',
+    'friday': 'Fr',
+    'thursday': 'Th',
+    'saturday': 'Sa',
+    'sunday': 'Su',
+}
+class CarphoneWarehouseSpider(scrapy.Spider):
+    name = "carphonewarehouse"
+    allowed_domains = ["www.carphonewarehouse.com"]
+    def store_hours(self, store_hours):
+        clean_time=''
+        for key1 , value1 in DAYS.items():
+           if key1 in store_hours:
+               clean_time = clean_time + value1+" "+store_hours[key1]+" ;"
+        return clean_time
+    def start_requests(self):
+        url = 'https://www.carphonewarehouse.com/services/storedata?filter=&count=1000000&lat=54.2526491&lng=-2.0411242'
+        yield scrapy.Request(
+                url=url,  callback=self.parse
+            )
+
+    def parse(self, response):
+        data = json.loads(response.body_as_unicode())
+        for key , value in data.items():
+            if 'AddressLine' in value:
+             addr_full = value['AddressLine'].split(',')
+             address = ", ".join(addr_full[:len(addr_full)-1])
+             city = addr_full[len(addr_full)-1]
+            else:
+                address=""
+                city=""
+            if 'postcode' in value:
+                postcode = value['postcode']
+            else:
+                postcode = ""
+            properties = {
+                'ref': key ,
+                'name': value['branch_name'],
+                'addr_full': address,
+                'city': city,
+                'country':'United Kingdom',
+                'postcode':postcode,
+                'lat': value['Latitude'],
+                'lon':  value['Longitude'],
+                'phone':value['telephone'],
+            }
+
+            opening_hours = self.store_hours(value)
+            if opening_hours:
+                properties['opening_hours'] = opening_hours
+
+            yield GeojsonPointItem(**properties)
+
+


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Vea (Argentina)
Supermarket chain.

The [search engine](http://www.supermercadosvea.com.ar/buscador-de-sucursales.html) on the website asks for province and city, but the underlying JSON endpoint lets you leave those arguments off and get the entire list:
http://www.supermercadosvea.com.ar/sucursales-obtener.html

Interesting how it even contains the user ID and IP address of the employee who added it to the database (wtf).


Discussion:

Resolving patch:
diff --git a/locations/spiders/vea.py b/locations/spiders/vea.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/vea.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+from locations.items import GeojsonPointItem
+
+
+class VeaSpider(scrapy.Spider):
+
+    name = "vea"
+    allowed_domains = ["www.supermercadosvea.com"]
+    start_urls = (
+        'http://www.supermercadosvea.com.ar/sucursales-obtener.html',
+    )
+
+    def parse(self, response):
+
+        store_list = json.loads(response.body_as_unicode())
+        for store in store_list:
+            properties = {
+                'name': store['descripcion'],
+                'addr_full': store['direccion'],
+                'city': store['vea_localidades_desc'],
+                'opening_hours': store['horarios'],
+                'phone': store['telefonos'],
+                'ref': str(store['codigo']).replace('SM ', ''),
+            }
+            yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
El Taco Tote
http://tacotote.com/site/index.php/en-us/store-locator


Discussion:

Resolving patch:
diff --git a/locations/spiders/tacotote.py b/locations/spiders/tacotote.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/tacotote.py
@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+
+from locations.items import GeojsonPointItem
+
+DAYS = {
+    '&nbsp;':' ',
+    'open 24 hrs':'00:00-24:00',
+    'monday': 'Mo',
+    'tuesday': 'Tu',
+    'wednesday': 'We',
+    'friday': 'Fr',
+    'thursday': 'Th',
+    'saturday': 'Sa',
+    'sunday': 'Su',
+    'open daily':'Mo-Su',
+    'mon': 'Mo',
+    'sat': 'Sa',
+    'thurs': 'Th',
+    'sun': 'Su',
+    'fri': 'Fr',
+    'and':',',
+    ' from ':' ',
+    'm to ':'m-',
+    ' to ':'-',
+    '&':',',
+    '- ':'-',
+    ' -':'-',
+    ' :':' ',
+    ': ':' ',
+    '  ':'',
+}
+
+class MisterCarWashSpider(scrapy.Spider):
+    name = "tacotote"
+    allowed_domains = ["tacotote.com/"]
+    start_urls = (
+        'http://tacotote.com/site/index.php/en-us/?option=com_storelocator&view=map&format=raw&searchall=1',
+    )
+
+    def clear_str (self, str):
+        s=str.lower()
+        for keys in DAYS:
+            s=s.replace(keys,DAYS[keys])
+        return s
+
+    def store_hours(self, store_hours):
+        if not store_hours or len(store_hours)<5:
+            return
+
+        stri = ''
+        for line in store_hours.split('<br />'):
+            line = self.clear_str(line)
+
+            match = re.search(r'(\w*)(\s*-\s*(\w*))?\s*(\d{1,2})(:(\d{1,2}))?\s*(am|pm|mp)?\s*-\s*(\d{1,2})(:(\d{1,2}))?\s*(am|pm|mp)',line)
+            
+            if not match:
+                continue
+
+            stri += match[1][:2]
+            
+            try:
+                stri += match[2][:3]+' '
+            except Exception:
+                stri += ' '
+
+            if match[5]:
+                first_minutes=match[5]
+            else:
+                first_minutes=':00'
+
+            if match[9]:
+                second_minutes=match[9]
+            else:
+                second_minutes=':00'
+
+            stri += str(int(match[4])+(12 if match[7] in ['pm','mp'] else 0)) +first_minutes+'-'
+            stri += str(int(match[8])+(12 if match[10] in ['pm','mp'] else 0)) +second_minutes+';'
+
+        return stri.rstrip(';')
+
+    def parse(self, response):  # high-level list of states
+        tacos = response.xpath('//marker')
+
+        for taco in tacos:
+
+            yield GeojsonPointItem(
+                lat=float(taco.xpath('.//lat/text()').extract_first()),
+                lon=float(taco.xpath('.//lng/text()').extract_first()),
+                phone=taco.xpath('.//phone/text()').extract_first(),
+                ref=taco.xpath('.//name/text()').extract_first(),
+                opening_hours=self.store_hours(taco.xpath('.//fulladdress/text()').extract_first().replace('<p>','').replace('</p>','')),
+                addr_full=taco.xpath('.//address/text()').extract_first(),
+            )


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Bank of America
https://locators.bankofamerica.com/


Discussion:
No progress for 6 days, re-opening this for others.

Resolving patch:
diff --git a/locations/spiders/bankofamerica.py b/locations/spiders/bankofamerica.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/bankofamerica.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+import scrapy
+
+from locations.items import GeojsonPointItem
+
+
+class BankOfAmericaSpider(scrapy.Spider):
+    name = "bankofamerica"
+    allowed_domains = ["bankofamerica.com"]
+    start_urls = (
+        'https://locators.bankofamerica.com/',
+    )
+
+    def parse(self, response):
+        states = response.xpath('//ul[@id="maplist"]//a/@href')
+        for state in states:
+            yield scrapy.Request(
+                response.urljoin(state.extract()),
+                callback=self.parse_state
+            )
+
+    def parse_state(self, response):
+        cities = response.xpath('//ul[@id="maplist"]//a/@href')
+        for city in cities:
+            yield scrapy.Request(
+                response.urljoin(city.extract()),
+                callback=self.parse_city
+            )
+
+    def parse_city(self, response):
+        centers = response.xpath('//ul[@id="maplist"]//div[@class="location"]//a[contains(@class,"name")]/@href')
+
+        for center in centers:
+            yield scrapy.Request(
+                response.urljoin(center.extract()),
+                callback=self.parse_center
+            )
+
+    def parse_center(self, response):
+            yield GeojsonPointItem(
+                lat=float(response.xpath('//meta[@property="og:latitude"]/@content').extract_first()),
+                lon=float(response.xpath('//meta[@property="og:longitude"]/@content').extract_first()),
+                phone=response.xpath('//meta[@property="og:phone_number"]/@content').extract_first(),
+                website=response.url,
+                ref=response.xpath('//meta[@name="twitter:title"]/@content').extract_first(),
+                opening_hours='; '.join(response.xpath('//div[@class="hours_container"]//meta[@itemprop="openingHours"]/@content').extract()),
+                addr_full=response.xpath('//meta[@property="og:street-address"]/@content').extract_first(),
+                city=response.xpath('//meta[@property="og:locality"]/@content').extract_first(),
+                state=response.xpath('//meta[@property="og:region"]/@content').extract_first(),
+                postcode=response.xpath('//meta[@property="og:postal-code"]/@content').extract_first(),
+                country=response.xpath('//meta[@property="og:country-name"]/@content').extract_first(),
+            )


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Hampton Inn
e.g. http://hamptoninn3.hilton.com/en/about/locations/united-states-america/wisconsin.html?regionId=972&subregionId=3915


Discussion:
No progress for 6 days, re-opening this for others.

Resolving patch:
diff --git a/locations/spiders/hamptoninn.py b/locations/spiders/hamptoninn.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/hamptoninn.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+
+from locations.items import GeojsonPointItem
+
+
+class HamptonInnSpider(scrapy.Spider):
+    name = "hamptoninn"
+    allowed_domains = ['hamptoninn3.hilton.com']
+    start_urls = (
+        'http://hamptoninn3.hilton.com/sitemapurl-hp-00000.xml',
+    )
+
+    def parse(self, response):
+        response.selector.remove_namespaces()
+        city_urls = response.xpath('//url/loc/text()').extract()
+        regex = re.compile(r'http(://|s://)(www.|)hamptoninn\d.hilton.com/en/hotels/\w+/\S+/maps-directions/index.html')
+        for path in city_urls:
+            if re.search(regex, path):
+                yield scrapy.Request(
+                    path.strip(),
+                    callback=self.parse_store,
+                )
+
+    def parse_store(self, response):
+        regex = re.compile(r'http(://|s://)(www.|)hamptoninn\d.hilton.com/en/hotels/\w+/\S+/maps-directions/index.html')
+        if re.search(regex, response.url):
+            properties = {
+                'name': response.xpath('//span[@class="property-name"]/text()').extract_first(),
+                'ref': response.xpath('//span[@class="property-name"]/text()').extract_first(),
+                'addr_full': response.xpath('//span[@class="property-streetAddress"]/text()').extract_first(),
+                'city': response.xpath('//span[@class="property-addressLocality"]/text()').extract_first(),
+                'state': response.xpath('//span[@class="property-addressRegion"]/text()').extract_first(),
+                'postcode': response.xpath('//span[@class="property-postalCode"]/text()').extract_first(),
+                'phone': response.xpath('//span[@class="property-telephone"]/text()').extract_first(),
+                'website': response.xpath('//h1/a/@href').extract_first(),
+                'opening_hours': response.xpath('//div[@class="policy_component_left_pane"]/div[@class="title"]/text()').extract_first() + " " + " ".join(response.xpath('//div[@class="policy_component_left_pane"]/div[2]/text()').extract_first().split()) + " - " + response.xpath('//div[@class="policy_component_right_pane"]/div[@class="title"]/text()').extract_first() + " " + " ".join(response.xpath('//div[@class="policy_component_right_pane"]/div[2]/text()').extract_first().split()),
+                'lat': float(response.xpath('//meta[@name="geo.position"]/@content').extract_first().split(';')[0]),
+                'lon': float(response.xpath('//meta[@name="geo.position"]/@content').extract_first().split(';')[1]),
+            }
+
+            yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Co-op food
Co-op is a brand which, amongst other things, runs a supermarket chain in the UK.

https://finder.coop.co.uk/food



Discussion:

Resolving patch:
diff --git a/locations/spiders/coopfood.py b/locations/spiders/coopfood.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/coopfood.py
@@ -0,0 +1,46 @@
+import json
+import scrapy
+
+from locations.items import GeojsonPointItem
+
+
+class CoopFoodSpider(scrapy.Spider):
+    name = "coopfood"
+    allowed_domains = ["api.coop.co.uk"]
+    download_delay = 0.5
+    page_number = 1
+    start_urls = (
+        'https://api.coop.co.uk/locationservices/finder/food/?location=54.9966124%2C-7.308574799999974&distance=30000000000&always_one=true&format=json',
+    )
+
+    def parse(self, response):
+        data = json.loads(response.body_as_unicode())
+
+        for store in data['results']:
+            open_hours = store['opening_hours']
+            clean_hours = ''
+            for time in open_hours:
+                if time['opens'] is not None and time['closes'] is not None:
+                    clean_hours = clean_hours + time['name'][:2] + ' ' + time['opens'] + '-' + time['closes'] + ' ; '
+
+            properties = {
+                "ref": store['url'],
+                "name": store['name'],
+                "opening_hours": clean_hours,
+                "website": "https://finder.coop.co.uk"+store['url'],
+                "addr_full": store['street_address'] + " " + store['street_address2'] + " " + store['street_address3'],
+                "city": store['town'],
+                "postcode": store['postcode'],
+                "country": 'United Kingdom',
+                "lon": float(store['position']['x']),
+                "lat": float(store['position']['y']),
+                "phone": store["phone"],
+            }
+
+            yield GeojsonPointItem(**properties)
+
+        if data['next'] is not None:
+            self.page_number = self.page_number + 1
+            yield scrapy.Request(
+                self.start_urls[0] + '&page=' + str(self.page_number)
+            )


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Ace Hardware
http://www.acehardware.com/mystore/storeLocator.jsp


Discussion:
Interesting.
http://www.acehardware.com/acestoresitemap.xml
Can pull URLS from there.
When on store page however, using xpath or css the values are never what they should be. Checking source they change. Will come back to this.
Removing the assignee on this because there's been no update for 5 days.

Resolving patch:
diff --git a/locations/spiders/ace_hardware.py b/locations/spiders/ace_hardware.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/ace_hardware.py
@@ -0,0 +1,66 @@
+import scrapy
+import re
+import json
+from locations.items import GeojsonPointItem
+
+DAY_MAPPING = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
+
+
+class AceHardwareSpider(scrapy.Spider):
+    name = "ace_hardware"
+    allowed_domains = ["www.acehardware.com"]
+    download_delay = 0.5
+    start_urls = (
+        'http://www.acehardware.com/acestoresitemap.xml',
+    )
+
+    def parse_hours(self, lis):
+        days = []
+        for day in DAY_MAPPING:
+            if 'openingTime'+day in lis and 'closingTime'+day in lis:
+                open_time = str(lis['openingTime'+day]).zfill(4)
+                close_time = str(lis['closingTime'+day]).zfill(4)
+                days.append("%s %s:%s-%s:%s" % (
+                    day[:2],
+                    open_time[0:2],
+                    open_time[2:],
+                    close_time[0:2],
+                    close_time[2:],
+                ))
+        return '; '.join(days)
+
+    def parse_stores(self, response):
+        ref = response.meta['id']
+        json_data = json.loads(response.body_as_unicode())
+
+        if 'address1' not in json_data:
+            return
+
+        properties = {
+            'addr_full': json_data['address1'],
+            'phone': json_data['phoneNumber'],
+            'city': json_data['city'],
+            'state': json_data['stateCode'],
+            'postcode': json_data['postalCode'],
+            'ref': ref,
+            'website': "http://www.acehardware.com/mystore/index.jsp?store=" + ref,
+            'lat': float(json_data['latitude']),
+            'lon': float(json_data['longitude']),
+        }
+
+        hours = self.parse_hours(json_data['hours'])
+        if hours:
+            properties['opening_hours'] = hours
+
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        data = response.body_as_unicode()
+        store_ids = re.findall(r"[0-9]{5}", data)
+        for store_id in store_ids:
+            url = "http://www.acehardware.com/storeLocServ?heavy=true&token=ACE&operation=storeData&storeID=%s" % (store_id)
+            yield scrapy.Request(
+                url,
+                callback=self.parse_stores,
+                meta={'id': store_id}
+            )


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
SimonMed (medical imaging)
https://www.simonmed.com/locations

Only a map/AJAX from what I can see, alas.


Discussion:

Resolving patch:
diff --git a/locations/spiders/simonmed.py b/locations/spiders/simonmed.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/simonmed.py
@@ -0,0 +1,73 @@
+import json
+import scrapy
+import re
+from locations.items import GeojsonPointItem
+
+
+class SimonMedSpider(scrapy.Spider):
+    name = "simonmed"
+    allowed_domains = ["www.simonmed.com"]
+    start_urls = (
+        'https://www.simonmed.com/locations/getLocations',
+    )
+
+    def parse_times(self, hour):
+        if re.search('PM$', hour):
+            hour = re.sub('PM', '', hour).strip()
+            hour_min = hour.split(":")
+            if int(hour_min[0]) < 12:
+                hour_min[0] = str(12 + int(hour_min[0]))
+            return ":".join(hour_min)
+
+        if re.search('AM$', hour):
+            hour = re.sub('AM', '', hour).strip()
+            hour_min = hour.split(":")
+            if len(hour_min[0]) < 2:
+                hour_min[0] = hour_min[0].zfill(2)
+            else:
+                hour_min[0] = str(12 + int(hour_min[0]))
+            return ":".join(hour_min)
+
+    def parse(self, response):
+        data = json.loads(response.body_as_unicode())
+        for store in data:
+            opening_hours = []
+
+            for day_hour in store['Days']:
+                opening_hours.append(day_hour['Day'][:2]+" "+self.parse_times(day_hour['Open']) + "-" + self.parse_times(day_hour['Close']))
+
+            properties = {
+                'addr_full': store['Street1']+" "+store['Street2'],
+                'phone': store['PhoneNumber'],
+                'city': store['City'],
+                'state': store['State'],
+                'postcode': store['ZipCode'],
+                'name': store['Name'],
+                'ref': store['id'],
+                'website': "https://www.simonmed.com/locations",
+                'lat': store['Latitude'],
+                'lon': store['Longitude'],
+                'opening_hours': '; '.join(opening_hours)
+            }
+
+            yield GeojsonPointItem(**properties)
+
+    def start_requests(self):
+        headers = {
+            'Accept-Language': 'en-US,en;q=0.8,ru;q=0.6',
+            'Origin': 'www.simonmed.com',
+            'Accept-Encoding': 'gzip, deflate',
+            'Accept': '*/*',
+            'Referer': 'https://www.simonmed.com/locations',
+            'Connection': 'keep-alive',
+            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
+        }
+        form_data = { 'location': '85251', 'miles': '1000000', 'location_lat': '33.4986286', 'location_lng': '-111.9224398'}
+
+        yield scrapy.http.FormRequest(
+            self.start_urls[0],
+            method='POST',
+            formdata=form_data,
+            headers=headers,
+        )


Test patch: