internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: alltheplaces/alltheplaces

Issue:
Au Bon Pain
https://www.aubonpain.com/stores/all-stores


Discussion:
This issue hasn't been worked on in a while. I'm removing the assignment so others can work on it. If you're still working on it, please reassign yourself and reference a pull request or branch where you're doing your work.

Resolving patch:
diff --git a/locations/spiders/aubonpain.py b/locations/spiders/aubonpain.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/aubonpain.py
@@ -0,0 +1,70 @@
+import scrapy
+import re
+import json
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+class AuBonPainSpider(scrapy.Spider):
+    name = "aubonpain"
+    download_delay = 0.5
+    allowed_domains = [
+        "www.aubonpain.com",
+    ]
+    start_urls = (
+        'https://www.aubonpain.com/stores/all-stores',
+    )
+
+    def parse_hours(self, items):
+        opening_hours = OpeningHours()
+
+        for day in items:
+            open_time = day["Open"]
+            close_time = day["Close"]
+            if close_time == 'Closed' or open_time == 'Closed':
+                continue
+            elif close_time == 'Open 24 Hrs' or open_time == 'Open 24 Hrs':
+                open_time = '12:00 AM'
+                close_time = '12:00 AM'
+            elif close_time == 'Open for Special Events':
+                continue
+
+            opening_hours.add_range(day=day["Day"][:2],
+                                    open_time=open_time,
+                                    close_time=close_time,
+                                    time_format='%I:%M %p')
+        return opening_hours.as_opening_hours()
+
+    def parse_store(self, response):
+        ref = re.findall(r"[^(\/)]+$", response.url)[0]
+
+        scripts = "".join(response.xpath('//script/text()').extract())
+        lat, lon = re.search(r'.*Microsoft.Maps.Location\(([0-9.-]*),\s+([0-9-.]*)\).*', scripts).groups()
+
+        address1, address2 = response.xpath('//dt[contains(text(), "Address")]/following-sibling::dd/text()').extract()
+        city, state, zipcode = re.search(r'^(.*),\s+([a-z]{2})\s+([0-9]+)$', address2.strip(), re.IGNORECASE).groups()
+
+        properties = {
+            'addr_full': address1.strip(', '),
+            'phone': response.xpath('//dt[contains(text(), "Phone")]/following-sibling::dd/a/text()').extract_first(),
+            'city': city,
+            'state': state,
+            'postcode': zipcode,
+            'ref': ref,
+            'website': response.url,
+            'lat':  float(lat),
+            'lon':  float(lon),
+        }
+        hours = json.loads(re.search(r'.*var\shours\s*=\s*(.*?);.*', scripts).groups()[0])
+        hours = self.parse_hours(hours)
+        if hours:
+            properties['opening_hours'] = hours
+
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        urls = response.xpath('//section/div/div//a[contains(@href, "stores")]/@href').extract()
+
+        for url in urls:
+            url = url.replace('\r\n', '')
+            yield scrapy.Request(response.urljoin(url), callback=self.parse_store)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Captain D’s
https://locations.captainds.com/site-map/US


Discussion:

Resolving patch:
diff --git a/locations/spiders/captaind.py b/locations/spiders/captaind.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/captaind.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+NUMBER_DAY = {
+    '1': 'Mo',
+    '2': 'Tu',
+    '3': 'We',
+    '4': 'Th',
+    '5': 'Fr',
+    '6': 'Sa',
+    '7': 'Su'
+}
+
+
+class CaptainDSpider(scrapy.Spider):
+
+    name = "captaind"
+    allowed_domains = ["momentfeed-prod.apigee.net", ]
+
+    start_urls = (
+        'https://momentfeed-prod.apigee.net/api/llp.json?auth_token=AJXCZOENNNXKHAKZ&country=US&sitemap=true',
+    )
+
+    def parse_hours(self, hours):
+        "1,1030,2200;2,1030,2200;3,1030,2200;4,1030,2200;5,1030,2300;6,1030,2300;7,1030,2200;"
+        opening_hours = OpeningHours()
+        if hours:
+            days = [day for day in hours.split(';') if day]
+            for day in days:
+                day, from_hr, to_hr = day.split(',')
+                opening_hours.add_range(day=NUMBER_DAY[day],
+                                        open_time=from_hr,
+                                        close_time=to_hr,
+                                        time_format='%H%M')
+        return opening_hours.as_opening_hours()
+
+    def parse(self, response):
+
+        stores = json.loads(response.body_as_unicode())
+
+        for store in stores:
+            opening_hours = ''
+            if store.get('store_info', '').get('store_hours', ''):
+                opening_hours = self.parse_hours(
+                    store['store_info']['store_hours'])
+
+            props = {
+                'name': store['store_info']['brand_name'],
+                'ref': store['store_info']['corporate_id'],
+                'addr_full': store['store_info']['address'],
+                'postcode': store['store_info']['postcode'],
+                'state': store['store_info']['region'],
+                'website': store['store_info']['website'],
+                'city': store['store_info']['locality'],
+                'phone': store['store_info']['phone'],
+                'lat': float(store['store_info']['latitude']),
+                'lon': float(store['store_info']['longitude']),
+                'opening_hours': opening_hours
+            }
+
+            yield GeojsonPointItem(**props)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
 UPS Freight service centers
https://www.upsfreight.com/ProductsandServices/ServiceCenterDir/default.aspx

Seperate issue from UPS Store


Discussion:

Resolving patch:
diff --git a/locations/spiders/ups_freight_service_centers.py b/locations/spiders/ups_freight_service_centers.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/ups_freight_service_centers.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+import re
+
+import scrapy
+from locations.items import GeojsonPointItem
+
+
+class UPSFreightServiceCenter(scrapy.Spider):
+    download_delay = 0.2
+    name = "ups_freight_service_centers"
+    allowed_domains = ["upsfreight.com"]
+    start_urls = (
+        'https://www.upsfreight.com/ProductsandServices/ServiceCenterDir/default.aspx',
+    )
+
+    def parse_location(self, response):
+        ref = re.search(r'.+/(.+)', response.url).group(1)
+
+        properties = {
+            'addr_full': response.xpath('//span[contains(@id, "Address")]/text()').extract()[0],
+            'city': response.xpath('//span[contains(@id, "Zip")]/text()').extract()[0].split(',')[0],
+            'state': response.xpath('//span[contains(@id, "Zip")]/text()').extract()[0].split(', ')[1].split(' ')[0],
+            'postcode': response.xpath('//span[contains(@id, "Zip")]/text()').extract()[0].split(', ')[1].split(' ')[1],
+            'ref': ref,
+            'website': response.url,
+            'phone': response.xpath('//span[contains(@id, "Telephone")]/text()').extract()[0],
+            'name': response.xpath('//span[contains(@id, "lName")]/text()').extract()[0],
+            'country': ref.split('qcountry=')[1].split('&svc')[0]
+        }
+
+        yield GeojsonPointItem(**properties)
+
+    def parse_state(self, response):
+        location_urls = response.xpath('//*[@id="app_ctl00_scTable_hlDetail"]/@href').extract()
+
+        for url in location_urls:
+            yield scrapy.Request(response.urljoin(url), callback=self.parse_location)
+
+    def parse(self, response):
+        urls = response.xpath('//table//table//table//table//table//a/@href').extract()
+
+        for url in urls:
+            yield scrapy.Request(response.urljoin(url), callback=self.parse_state)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
XPO Logistics
https://www.xpo.com/global-locations/


Discussion:

Resolving patch:
diff --git a/locations/spiders/xpo_logistics.py b/locations/spiders/xpo_logistics.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/xpo_logistics.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+import ast
+from locations.items import GeojsonPointItem
+
+class XPOLogisticsSpider(scrapy.Spider):
+    name = "xpo_logistics"
+    allowed_domains = ["www.xpo.com"]
+    start_urls = (
+        'https://www.xpo.com/global-locations/',
+    )
+
+    def parse(self, response):
+        script = response.xpath('//script[contains(.,"globalLocationsArray")]').extract_first()
+        data = re.search(r'globalLocationsArray = (.*);', script).groups()[0]
+        data = ast.literal_eval(data)
+
+        for store in data:
+            yield GeojsonPointItem(
+                lat=float(store['latitude']),
+                lon=float(store['longitude'].replace(',','')),
+                phone=store['telephone'],
+                ref=store['office_name'],
+                addr_full=store['street'],
+                city=store['city'],
+                state=store['state'],
+                postcode=store['postal_code'],
+                country=store['country'],
+                name=store['office_name']
+            )
+
+


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
VCA
https://vcahospitals.com/find-a-hospital/location-directory


Discussion:

Resolving patch:
diff --git a/locations/spiders/vca.py b/locations/spiders/vca.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/vca.py
@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+
+from locations.items import GeojsonPointItem
+
+DAYS={
+    'Monday':'Mo',
+    'Tuesday':'Tu',
+    'Wednesday':'We',
+    'Friday':'Fr',
+    'Thursday':'Th',
+    'Saturday':'Sa',
+    'Sunday':'Su',
+}
+
+class VCASpider(scrapy.Spider):
+    name = "vca"
+    allowed_domains = ["vcahospitals.com"]
+    start_urls = (
+        'https://vcahospitals.com/find-a-hospital/location-directory',
+    )
+
+
+    def parse(self, response):
+        urls = response.xpath('//div[@class="hospital"]/section/h3/a/@href').extract()
+
+        for url in urls:
+            yield scrapy.Request(response.urljoin(url), callback=self.parse_store)
+
+
+    def parse_store(self, response):
+        try:
+            data = response.xpath('//script[@type="application/ld+json"]/text()').extract_first()
+            data = data.replace('\t',' ')
+            data = json.loads(data)
+        except AttributeError:
+            return
+        if not data:
+            return
+
+        yield GeojsonPointItem(
+            lat=float(data['geo']['latitude']),
+            lon=float(data['geo']['longitude']),
+            phone=data['telephone'],
+            website=response.url,
+            ref=response.url,
+            opening_hours=data['openingHours'],
+            addr_full=data['address']['streetAddress'],
+            city=data['address']['addressLocality'],
+            state=data['address']['addressRegion'],
+            postcode=data['address']['postalCode'],
+            country='US',
+            name=data['name'],
+        )
+
+


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Pizza Ranch 
Spider #29 is not collecting all locations, it's only capturing the 1st page on each state list.



Discussion:

Resolving patch:
diff --git a/locations/spiders/pizzaranch.py b/locations/spiders/pizzaranch.py
--- a/locations/spiders/pizzaranch.py
+++ b/locations/spiders/pizzaranch.py
@@ -51,6 +51,7 @@ def parse_stores(self, response):
         map_data = response.xpath('normalize-space(//script[@id="js-map-config-dir-map-nap-map"]/text())').extract_first()
         map_json = json.loads(map_data)
         properties = {
+            'name': response.xpath('//span[@class="location-name-geo"]/text()').extract_first(),
             'addr_full': response.xpath('normalize-space(//span[@itemprop="streetAddress"]/span/text())').extract_first(),
             'phone': response.xpath('normalize-space(//div[@class="c-phone-number c-phone-main-number"]/span[@class="c-phone-number-span c-phone-main-number-span"]/text())').extract_first(),
             'city': response.xpath('normalize-space(//span[@itemprop="addressLocality"]/text())').extract_first(),
@@ -68,11 +69,17 @@ def parse_stores(self, response):
 
         yield GeojsonPointItem(**properties)
 
+
     def parse_state_stores(self, response):
         stores = response.xpath('//h3[@class="title"]/a/@href').extract()
         for store in stores:
                 yield scrapy.Request(response.urljoin(store), callback=self.parse_stores)
 
+        next_page_url = response.xpath('//div[@class="pagination"]//li[@class="next"]/a/@href').extract_first()
+
+        if next_page_url:
+            yield scrapy.Request(next_page_url, callback=self.parse_state_stores)
+
     def parse(self, response):
         urls = response.xpath('//ol[@class="state-list"]/li/a/@href').extract()
         for path in urls:


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Benihana
Create spider for Benihana 


Discussion:

Resolving patch:
diff --git a/locations/spiders/benihana.py b/locations/spiders/benihana.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/benihana.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+import json
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+class BenihanaSpider(scrapy.Spider):
+    download_delay = 0.2
+    name = "benihana"
+    allowed_domains = ["benihana.com"]
+
+    def start_requests(self):
+        yield scrapy.Request('https://www.benihana.com/wp-content/themes/grandrestaurant-child/js/locations.json',
+                             callback=self.set_locations)
+        yield scrapy.Request('https://www.benihana.com/locations/')
+
+    def set_locations(self, response):
+        self.locations = json.loads(response.body_as_unicode())
+
+    def parse(self, response):
+        urls = response.xpath('//div[@class="location-link"]/a/@href').extract()
+
+        for url in urls:
+            if url != 'http://www.dignityhealthsportspark.com/' or url != 'https://www.sprintcenter.com/':
+                yield scrapy.Request(response.urljoin(url), callback=self.parse_stores)
+
+
+    def parse_stores(self, response):
+        delivery_only = response.xpath('//div[@class="hours-wrapper"]/p/strong/text()').extract_first()
+        if delivery_only != 'DELIVERY HOURS':
+            locationid = re.search(r'.*/locations/(.*)/$', response.url).groups()[0]
+            location = [loc for loc in self.locations if loc["LocationId"] == locationid][0]
+            properties = {
+                'addr_full': response.xpath('//span[@itemprop="streetAddress"]/text()').extract_first().strip(','),
+                'phone':  response.xpath('//span[@itemprop="telephone"]/text()').extract_first(),
+                'name': response.xpath('//h2[@class="ppb_title"]/text()').extract_first(),
+                'city': response.xpath('//span[@itemprop="addressLocality"]/text()').extract_first().strip(','),
+                'state': response.xpath('//span[@itemprop="addressRegion"]/text()').extract_first(),
+                'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first(),
+                'ref': locationid,
+                'website': response.url,
+                'lat': float(location["lat"]),
+                'lon': float(location["lng"]),
+            }
+
+
+            yield GeojsonPointItem(**properties)
\ No newline at end of file


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Harris Teeter
Mostly southeastern https://www.harristeeter.com/store/#/app/store-locator


Discussion:
working on this
hi, @iandees 
I am working on this issue.
but I found this website is not allowed crawling.
I've changed the User-Agent and tried, but always was blocked to scrape.
do you have any suggestion?
I'd recommend un-assigning and finding an easier one for now. We can come back to this later.
okay, got you.
let me find solution later.

Resolving patch:
diff --git a/locations/spiders/harristeeter.py b/locations/spiders/harristeeter.py
--- a/locations/spiders/harristeeter.py
+++ b/locations/spiders/harristeeter.py
@@ -79,16 +79,17 @@ def parse_shop(self, response):
 
         for shop in shops:
             props = {
-                'opening_hours': shop['StoreHours'].replace('Open 24 hrs', 'Mo-Su 0:00-24:00'),
-                'phone': shop['Telephone'],
-                'country': shop['Country'],
-                'ref': shop['Title'],
+                'ref': shop['StoreNumber'],
                 'addr_full': shop['Street'],
-                'postcode': shop.get('ZipCode'),
                 'city': shop.get('City'),
                 'state': shop.get('State'),
+                'postcode': shop.get('PostalCode'),
+                'country': shop['Country'],
+                'name': shop['StoreName'],
+                'phone': shop['Telephone'],
                 'lat': float(shop['Latitude']),
                 'lon': float(shop['Longitude']),
+                'opening_hours': shop['StoreHours'].replace('Open 24 Hours', 'Mo-Su 0:00-24:00')
             }
 
             yield GeojsonPointItem(**props)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
24 Hour Fitness
https://www.24hourfitness.com/health_clubs/find-a-gym/


Discussion:

Resolving patch:
diff --git a/locations/spiders/24_hour_fitness.py b/locations/spiders/24_hour_fitness.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/24_hour_fitness.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+from locations.items import GeojsonPointItem
+
+
+class TwentyFourHourFitnessSpider(scrapy.Spider):
+    name = "24_hour_fitness"
+    allowed_domains = ["www.24hourfitness.com"]
+    start_urls = (
+        'https://www.24hourfitness.com',
+    )
+    download_delay = 0.1
+
+    def start_requests(self):
+        url = 'https://www.24hourfitness.com/Website/ClubLocation/OpenClubs/'
+
+        headers = {
+            'Accept': 'application/json, text/javascript, */*; q=0.01',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Content-Type': 'application/json',
+            'Host': 'www.24hourfitness.com',
+            'Referer': 'https://www.24hourfitness.com/health_clubs/find-a-gym/',
+            'X-Requested-With': 'XMLHttpRequest'
+        }
+
+        yield scrapy.http.Request(
+            url,
+            self.parse,
+            method='GET',
+            headers=headers,
+        )
+
+    def parse(self, response):
+        result = json.loads(response.body_as_unicode())
+
+        if 'clubs' in result.keys():
+            for club in result.get('clubs'):
+
+                ref = club.get('clubNumber')
+                name = club.get('name')
+                street = club.get('address').get('street')
+                city = club.get('address').get('city')
+                state = club.get('address').get('state')
+                postalcode = club.get('address').get('zip')
+                lat = club.get('coordinate').get('latitude')
+                lon = club.get('coordinate').get('longitude')
+                phone = club.get('phoneNumber')
+                website = club.get('linkURL')
+
+                properties = {
+                    "ref": ref,
+                    "name": name,
+                    "street": street,
+                    "city": city,
+                    "state": state,
+                    "postcode": postalcode,
+                    "lat": lat,
+                    "lon": lon,
+                    "phone": phone,
+                    "website": website
+                }
+
+                yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Belk
https://www.belk.com/stores/?showForm=true


Discussion:

Resolving patch:
diff --git a/locations/spiders/belk.py b/locations/spiders/belk.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/belk.py
@@ -0,0 +1,74 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+import json
+from locations.items import GeojsonPointItem
+
+
+states = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
+          'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD',
+          'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ',
+          'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC',
+          'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY']
+
+
+class BelkSpider(scrapy.Spider):
+    name = "belk"
+    allowed_domains = ["www.belk.com"]
+    start_urls = (
+        'https://www.belk.com/stores-near-you/',
+    )
+    download_delay = 0.2
+
+    def start_requests(self):
+        url = 'https://www.belk.com/stores-near-you/'
+
+        for state in states:
+            headers = {
+                'origin': 'https://www.belk.com',
+                'Referer': 'https://www.belk.com/stores/?showForm=true'
+            }
+
+            formdata = {
+                'dwfrm_storelocator_state': '{}'.format(state),
+                'dwfrm_storelocator_findbystate': 'Search',
+            }
+
+            yield scrapy.http.FormRequest(
+                url,
+                self.parse,
+                method='POST',
+                headers=headers,
+                formdata=formdata
+            )
+
+    def parse(self, response):
+        store_urls = response.xpath('//a[@class="store-details-link"]/@href').extract()
+        for store_url in store_urls:
+            yield scrapy.Request(response.urljoin(store_url), callback=self.parse_store)
+
+    def parse_store(self, response):
+        store_data = response.xpath('//script[@type="application/ld+json"]/text()').extract_first()
+        data_obj = json.loads(store_data)
+        ref = re.search(r'.+/?StoreID=(.+)', response.url).group(1)
+        street = data_obj.get('address').get('streetAddress')
+        city = data_obj.get('address').get('addressLocality')
+        state = data_obj.get('address').get('addressRegion')
+        postcode = data_obj.get('address').get('postalCode')
+        lat = data_obj.get('geo').get('latitude')
+        lon = data_obj.get('geo').get('longitude')
+        phone = data_obj.get('telephone')
+
+        properties = {
+            'ref': ref,
+            'addr_full': street,
+            'city': city,
+            'state': state,
+            'postcode': postcode,
+            'lat': lat,
+            'lon': lon,
+            'phone': phone,
+            'website': response.request.url,
+        }
+
+        yield GeojsonPointItem(**properties)


Test patch: