internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: alltheplaces/alltheplaces

Issue:
Meijer
https://www.meijer.com/store_unknown.jsp


Discussion:

Resolving patch:
diff --git a/locations/spiders/meijer.py b/locations/spiders/meijer.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/meijer.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+from locations.items import GeojsonPointItem
+
+STATES = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
+          'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD',
+          'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ',
+          'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC',
+          'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY']
+          
+class MeijerSpider(scrapy.Spider):
+    name = 'meijer'
+    allowed_domains = ['www.meijer.com']
+
+    def start_requests(self):
+        for state in STATES:
+            yield scrapy.Request(
+                'https://www.meijer.com/custserv/locate_store_by_state.cmd?form_state=locateStoreByStateForm&state={}'.format(state),
+                callback = self.parse
+            )
+    
+    def parse(self, response):
+        stores = response.css('div.records_inner>script::text').extract_first()
+        
+        if stores:
+            stores = stores.strip()[13:-1]
+            stores = stores.replace('\',\'', '","')
+            stores = stores.replace('[\'', '["')
+            stores = stores.replace('\']', '"]')
+            stores = json.loads(stores)
+            loc_data = response.css('script').extract()[10]
+            lat_matches = re.findall(r'(\"LAT\"), (\")([+-]?([0-9]*[.])?[0-9]+)(\")', loc_data)
+            lon_matches = re.findall(r'(\"LNG\"), (\")([+-]?([0-9]*[.])?[0-9]+)(\")', loc_data)
+            
+            n = 0
+            for store in stores:
+                address1 = store[6].split(',')
+                city = address1[0].strip()
+                address2 = address1[1].strip().split(' ')
+                state = address2[0]
+                postcode = address2[1]
+                properties = {
+                    'ref': store[0],
+                    'name': store[1],
+                    'phone': store[7],
+                    'opening_hours': self.hours(store[8]),
+                    'lat': lat_matches[n][2],
+                    'lon': lon_matches[n][2],
+                    'street': store[2],
+                    'city': city,
+                    'state': state,
+                    'postcode': postcode
+                }
+                
+                n = n + 1
+                
+                yield GeojsonPointItem(**properties)
+
+    def hours(self, data):
+        if data == 'Open 24 hrs a day, 364 days a year.':
+            return '24/7'
+        else :
+            return data
+
+        
+        


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Red Robbin
http://www.redrobin.com/find-us/restaurants.html
Red Robbin
http://www.redrobin.com/find-us/restaurants.html


Discussion:
working on this.
working on this.

Resolving patch:
diff --git a/locations/spiders/red_robbin.py b/locations/spiders/red_robbin.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/red_robbin.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+
+from locations.items import GeojsonPointItem
+
+
+class RedRobbinSpider(scrapy.Spider):
+
+    name = "red_robbin"
+    allowed_domains = ["www.redrobin.com"]
+    start_urls = (
+        'https://www.redrobin.com/static/data.locationddp.json?lat=29.8067386&lng=-91.51919780000003&units=miles&maxresults=1200&maxdistance=5000',
+    )
+
+    def store_hours(self, store_hours):
+        opening_hours = ""
+
+        for key, day_hours in store_hours.items():
+            day_hours = day_hours.replace('Mon', 'Mo').replace('Tue', 'Tu').replace('Thu', 'Th').replace('Fri', 'Fr').replace('Sat', 'Sa').replace('Sun', 'Su')
+            day_hours = day_hours.replace('a-', ':00-')
+            day_hours = day_hours.replace('Midnight', '12:00')
+    
+            m = re.search('([0-9]{1,2})(p)', day_hours)
+            if m:
+                h = m.group(1)
+                new_h = int(h) + 12
+                day_hours = day_hours.replace(h + 'p', str(new_h) + ':00')
+            opening_hours += '{}; '.format(day_hours)            
+        
+        opening_hours = opening_hours[:-1]
+
+        return opening_hours
+
+    def parse(self, response):
+        results = json.loads(response.body_as_unicode())
+        for data in results['locations']:
+            properties = {
+                'ref': data['store_id'],
+                'name': data['restaurant_name'],
+                'lat': data['latitude'],
+                'lon': data['longitude'],
+                'addr_full': data['address'],
+                'city': data['city_name'],
+                'state': data['state'],
+                'postcode': data['zipcode'],
+                'country': data['country'],
+                'phone': data['phone'],
+                'website': data['onlineOrderingURL'],
+                "opening_hours": self.store_hours(data['hours'])
+
+            }
+
+            yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Whidbey Coffee
http://www.whidbeycoffee.com/pages/locations


Discussion:
Static HTML to parse. Opening hours format seems pretty consistent.
I have done the whidbeycoffee spider- Josh

Resolving patch:
diff --git a/locations/spiders/whidbeycoffee.py b/locations/spiders/whidbeycoffee.py
--- a/locations/spiders/whidbeycoffee.py
+++ b/locations/spiders/whidbeycoffee.py
@@ -76,7 +76,6 @@ def parse_hours(self, lis):
             else:
                 day = re.findall(r"^[^( |:)]+" ,li)[0]
             times = li.replace(day , "")[1:]
-            print(times)
             if times and day:
                 parsed_time = self.parse_times(times)
                 parsed_day = self.parse_day(day)
@@ -90,6 +89,7 @@ def parse_hours(self, lis):
         return "; ".join(hours)
 
     def parse(self, response):
+
       stores = response.xpath('//h5')
       for index , store in enumerate(stores):
         direction_link = store.xpath('normalize-space(./following-sibling::p/a/@href)').extract_first()
@@ -99,7 +99,7 @@ def parse(self, response):
             'city': store.xpath('./following-sibling::p/a/text()').extract()[1].split(',')[0],
             'state':  store.xpath('./following-sibling::p/a/text()').extract()[1].split(',')[1].split(' ')[1],
             'postcode':  store.xpath('./following-sibling::p/a/text()').extract()[1].split(',')[1].split(' ')[2],
-            'ref':response.url,
+            'ref':store.xpath('normalize-space(./text())').extract_first(),
             'lat':re.findall(r"\/@[^(\/)]+", direction_link)[0].split(',')[0][2:],
             'lon': re.findall(r"\/@[^(\/)]+", direction_link)[0].split(',')[1],
         }


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Dunkin Donuts
https://www.dunkindonuts.com/dunkindonuts/en/stores.html


Discussion:

Resolving patch:
diff --git a/locations/spiders/dunkin.py b/locations/spiders/dunkin.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/dunkin.py
@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+import scrapy
+from locations.items import GeojsonPointItem
+import json
+
+
+class DunkinSpider(scrapy.Spider):
+    name = "dunkindonuts"
+    allowed_domains = ["dunkindonuts.com", "mapquestapi.com"]
+
+    # Choosing key hub at geographic extremes of the US in order to get full coverage in the location search
+    search_hubs = ['washington+dc', 'los+angeles', 'miami', 'boston', 'chicago']
+
+    start_urls = [
+        'https://www.mapquestapi.com/search/v2/radius?'
+        'key=Gmjtd|lu6t2luan5%2C72%3Do5-larsq&origin=' + i + '&units=m&'
+        'maxMatches=4000&radius=3000&hostedData=mqap.33454_DunkinDonuts&'
+        'ambiguities=ignore' for i in search_hubs
+    ]
+
+    def parse(self, response):
+        jdata = json.loads(response.body_as_unicode())
+
+        for i in jdata['searchResults']:
+
+            properties = {
+                'ref': i['key'],
+                'country': i['fields']['country'],
+                'state': i['fields']['state'],
+                'city': i['fields']['city'],
+                'lat': i['fields']['mqap_geography']['latLng']['lat'],
+                'lon': i['fields']['mqap_geography']['latLng']['lng'],
+                'phone': i['fields']['phonenumber'],
+                'addr_full': i['fields']['address'],
+                'postcode': i['fields']['postal'],
+
+
+            }
+
+            yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Wawa
Gas/convenience chain, like Sheetz


Discussion:
https://www.wawa.com/StoreLocator.aspx
+1
JSON API is:

https://www.wawa.com/Handlers/LocationByLatLong.ashx?limit=50&lat=43.1272254&long=-87.9432837

Resolving patch:
diff --git a/locations/spiders/wawa.py b/locations/spiders/wawa.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/wawa.py
@@ -0,0 +1,86 @@
+import json
+import scrapy
+from urllib import parse
+from locations.items import GeojsonPointItem
+
+
+MAPQUEST_KEY = 'ybe4KeKi8ACKY0eVqJXAw2QxTKnxnor8'
+
+MAPQ = 'http://www.mapquestapi.com/geocoding/v1/address?'
+
+LAT_LONS = [
+    ('39.2601', '-76.3549'),
+    ('36.9874', '-77.0031'),
+    ('27.9998', '-81.6064'),
+    ('38.9300', '-77.1020'),
+    ('39.0495', '-75.4541'),
+    ('41.2033', '-77.1945'),
+    ('40.7995', '-77.8271'),
+]
+
+
+class WawaSpider(scrapy.Spider):
+    name = 'wawa'
+    start_urls = (
+        'https://www.wawa.com/Handlers/LocationByLatLong.ashx?',
+    )
+    download_delay = 1.5
+    allowed_domains = ('www.wawa.com', 'www.mapquestapi.com', )
+
+    def start_requests(self):
+        url = 'https://www.wawa.com/Handlers/LocationByLatLong.ashx?'
+        for lat, lon in LAT_LONS:
+            wawa_params = {'limit': 50, 'lat': lat, 'long': lon}
+
+            yield scrapy.Request(
+                url + parse.urlencode(wawa_params),
+                headers={
+                    'Accept': 'application/json',
+                },
+                callback=self.parse
+            )
+
+    def get_addr(self, addr):
+        return (v for k, v in addr.items() if k in ['address', 'city', 'state', 'zip'])
+
+    def get_lat_lng(self, physical_addr):
+        return physical_addr['loc']
+
+    def get_opening_hours(self, store):
+        open_time = store['storeOpen'][:5]
+        close_time = store['storeClose'][:5]
+
+        times = '{}-{}'.format(open_time, close_time)
+
+        if times == '00:00-00:00':
+            return '24/7'
+        else:
+            return times
+
+    def parse(self, response):
+        wawa_stores = json.loads(response.body_as_unicode())
+
+        for loc in wawa_stores['locations']:
+
+            addr, city, state, zipc = self.get_addr(loc['addresses'][0])
+            lat, lng = self.get_lat_lng(loc['addresses'][1])
+            opening_hours = self.get_opening_hours(loc) or None
+
+            properties = {
+                'addr_full': addr,
+                'name': loc['storeName'],
+                'phone': loc['telephone'],
+                'city': city,
+                'state': state,
+                'postcode': zipc,
+                'ref': loc['locationID'],
+                'website': "https://www.wawa.com/stores/{}/{}".format(
+                    loc['locationID'],
+                    loc['addressUrl'],
+                ),
+                'lat': float(lat),
+                'lon': float(lng),
+                'opening_hours': opening_hours
+            }
+
+            yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Shopko
http://www.shopko.com/custserv/locate_store.cmd


Discussion:

Resolving patch:
diff --git a/locations/spiders/shopko.py b/locations/spiders/shopko.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/shopko.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+
+from locations.items import GeojsonPointItem
+
+
+class ShopkoSpider(scrapy.Spider):
+    name = "shopko"
+    allowed_domains = ['shopko.com']
+    start_urls = (
+        'http://www.shopko.com/sitemaps_list.jsp?seq_no=1&typeName=store',
+    )
+
+    def parse(self, response):
+        response.selector.remove_namespaces()
+        city_urls = response.xpath('//url/loc/text()').extract()
+        for path in city_urls:
+            locationURL = re.compile(r'http:/(/|/www.)shopko.com/\S+/store_details\S+=\d+')
+            if not re.search(locationURL, path):
+                pass
+            else:
+                yield scrapy.Request(
+                    path.strip(),
+                    callback=self.parse_store,
+                )
+
+    def parse_store(self, response):
+
+        properties = {
+            'name': response.xpath('/html/body/div[1]/div[2]/div/div[2]/div[2]/h1').extract()[0].split('</i>')[1].strip('</h1>'),
+            'website': response.request.url,
+            'ref': response.xpath('/html/body/div[1]/div[2]/div/div[2]/div[2]/h1').extract()[0].split('</i>')[1].strip('</h1>'),
+            'addr_full': " ".join(response.xpath('/html/body/div[1]/div[2]/div/div[2]/div[2]/div[1]/text()').extract()[0].split()),
+            'city': " ".join(response.xpath('/html/body/div[1]/div[2]/div/div[2]/div[2]/div[1]/text()').extract()[1].split()).split(',')[0],
+            'state': " ".join(response.xpath('/html/body/div[1]/div[2]/div/div[2]/div[2]/div[1]/text()').extract()[1].split()).split(',')[1].split()[0],
+            'postcode': " ".join(response.xpath('/html/body/div[1]/div[2]/div/div[2]/div[2]/div[1]/text()').extract()[1].split()).split(',')[1].split()[1],
+            'opening_hours': str(" ".join(response.xpath('/html/body/div[1]/div[2]/div/div[2]/div[2]/div[4]/text()').extract()).split()).replace(',','').replace('[','').replace("'", ''),
+             'lon': float(response.xpath('/html/body/div[1]/div[2]/div/script[3]/text()').extract()[0].split('store.lng = ')[1].split(';')[0]),
+             'lat': float(response.xpath('/html/body/div[1]/div[2]/div/script[3]/text()').extract()[0].split('store.lat = ')[1].split(';')[0]),
+        }
+
+        yield GeojsonPointItem(**properties)
\ No newline at end of file


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Wegmans
https://www.wegmans.com/stores.html


Discussion:

Resolving patch:
diff --git a/locations/spiders/wegmans.py b/locations/spiders/wegmans.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/wegmans.py
@@ -0,0 +1,30 @@
+import scrapy
+import re
+from locations.items import GeojsonPointItem
+
+class WegmansSpider(scrapy.Spider):
+
+    name = "wegmans"
+    allowed_domains = ["www.wegmans.com"]
+    download_delay = 1.5
+    start_urls = (
+        'https://www.wegmans.com/stores.html',
+    )
+    def parse_stores(self, response):
+        properties = {
+            'addr_full': response.xpath('normalize-space(//meta[@name="address"]/@content)').extract_first(),
+            'phone': response.xpath('normalize-space(//meta[@name="phone"]/@content)').extract_first(),
+            'city': response.xpath('normalize-space(//meta[@name="city"]/@content)').extract_first(),
+            'state': response.xpath('normalize-space(//meta[@name="stateAbbreviation"]/@content)').extract_first(),
+            'postcode':response.xpath('normalize-space(//meta[@name="zip"]/@content)').extract_first(),
+            'ref': response.xpath('normalize-space(//meta[@name="id"]/@content)').extract_first(),
+            'website': response.url,
+            'lat':  response.xpath('normalize-space(//input[@id="storeLocation"]/@data-lat)').extract_first(),
+            'lon':  response.xpath('normalize-space(//input[@id="storeLocation"]/@data-long)').extract_first(),
+        }
+        
+        yield GeojsonPointItem(**properties)
+    def parse(self, response):
+        urls = response.xpath('//div[@class="cta-rte"]/p/a/@href').extract()
+        for path in urls:
+                yield scrapy.Request(response.urljoin(path), callback=self.parse_stores)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Wendy's
e.g. https://locations.wendys.com/jamestown-ny-3438


Discussion:

Resolving patch:
diff --git a/locations/spiders/wendys.py b/locations/spiders/wendys.py
--- a/locations/spiders/wendys.py
+++ b/locations/spiders/wendys.py
@@ -18,7 +18,7 @@ class WendysSpider(scrapy.Spider):
 
     name = "wendys"
     allowed_domains = ["locations.wendys.com"]
-    download_delay = 0
+    download_delay = 0.5
     download_timeout = 30
     start_urls = (
         'https://locations.wendys.com',


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Big Boy
http://www.bigboy.com/locations/


Discussion:

Resolving patch:
diff --git a/locations/spiders/bigboy.py b/locations/spiders/bigboy.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/bigboy.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+
+from locations.items import GeojsonPointItem
+
+
+class WalmartSpider(scrapy.Spider):
+    name = "bigboy"
+    allowed_domains = ["www.bigboy.com"]
+    start_urls = (
+        'http://www.bigboy.com/locations',
+    )
+
+    def parse(self, response):
+        stores = response.xpath('//div[@id="locations"]/div')
+
+        for store in stores:
+            url = store.xpath('@itemtype')[0].extract()
+            name = store.xpath('div[@class="thumbnail"]/strong[@itemprop="name"]/text()')[0].extract()
+            street = store.xpath('div[@class="thumbnail"]/div[@itemprop="address"]/span[@itemprop="streetAddress"]/text()')[0].extract()
+            locality = store.xpath('div[@class="thumbnail"]/div[@itemprop="address"]/span[@itemprop="addressLocality"]/text()')[0].extract()
+            region = store.xpath('div[@class="thumbnail"]/div[@itemprop="address"]/span[@itemprop="addressRegion"]/text()')[0].extract()
+            postalcode = store.xpath('div[@class="thumbnail"]/div[@itemprop="address"]/span[@itemprop="postalCode"]/text()')[0].extract()
+            phone = store.xpath('div[@class="thumbnail"]/div[@itemprop="address"]/span[@itemprop="telephone"]/text()')[0].extract()
+            lat = store.xpath('div[@class="thumbnail"]/div[@itemprop="geo"]/meta[@itemprop="latitude"]/@content')[0].extract()
+            lon = store.xpath('div[@class="thumbnail"]/div[@itemprop="geo"]/meta[@itemprop="longitude"]/@content')[0].extract()
+
+            yield GeojsonPointItem(
+                lat=lat,
+                lon=lon,
+                ref=name,
+                phone=phone,
+                name=name,
+                opening_hours='',
+                addr_full=street,
+                city=locality,
+                state=region,
+                postcode=postalcode,
+                website=url
+            )


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
AMC Movie Theaters
JSON endpoint:

```
curl 'https://www.amctheatres.com/api/locations' \
-H 'Content-Type: application/json' \
-H 'Accept: application/json, text/javascript, */*; q=0.01' \
-H 'Referer: https://www.amctheatres.com/movie-theatres' \
-H 'X-Requested-With: XMLHttpRequest' \
--data-binary '{"query":"90210"}' --compressed
```



Discussion:

Resolving patch:
diff --git a/locations/spiders/amctheatres.py b/locations/spiders/amctheatres.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/amctheatres.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+import scrapy
+
+from locations.items import GeojsonPointItem
+
+
+class AmcTheatresSpider(scrapy.Spider):
+    name = "amctheatres"
+    allowed_domains = ['amctheatres.com']
+    start_urls = (
+        'https://www.amctheatres.com/sitemaps/sitemap-theatres.xml',
+    )
+
+    def parse(self, response):
+        response.selector.remove_namespaces()
+        theater_elems = response.xpath('//url')
+        for theater_elem in theater_elems:
+            properties = {
+                'website': theater_elem.xpath('.//loc/text()').extract_first(),
+                'name': theater_elem.xpath('.//Attribute[@name="title"]/text()').extract_first(),
+                'ref': theater_elem.xpath('.//Attribute[@name="theatreId"]/text()').extract_first(),
+                'addr_full': theater_elem.xpath('.//Attribute[@name="addressLine1"]/text()').extract_first(),
+                'city': theater_elem.xpath('.//Attribute[@name="city"]/text()').extract_first(),
+                'state': theater_elem.xpath('.//Attribute[@name="state"]/text()').extract_first(),
+                'postcode': theater_elem.xpath('.//Attribute[@name="postalCode"]/text()').extract_first(),
+                'lat': float(theater_elem.xpath('.//Attribute[@name="latitude"]/text()').extract_first()),
+                'lon': float(theater_elem.xpath('.//Attribute[@name="longitude"]/text()').extract_first()),
+            }
+
+            yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Rubio's
https://www.rubios.com/store-locator


Discussion:

Resolving patch:
diff --git a/locations/spiders/rubios.py b/locations/spiders/rubios.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/rubios.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+
+from locations.items import GeojsonPointItem
+
+
+class RubiosSpider(scrapy.Spider):
+    name = "rubios"
+    allowed_domains = ['rubios.com']
+    start_urls = (
+        'https://www.rubios.com/sitemap.xml',
+    )
+
+    def parse(self, response):
+        response.selector.remove_namespaces()
+        city_urls = response.xpath('//url/loc/text()').extract()
+        regex = re.compile(r'http\S+rubios.com/store-locations/\S+/\S+/\S+')
+        for path in city_urls:
+            if re.search(regex, path):
+                yield scrapy.Request(
+                    path.strip(),
+                    callback=self.parse_store,
+                )
+
+    def parse_store(self, response):
+
+        properties = {
+            'name': response.xpath('//span[@itemprop="name"]/text()').extract_first(),
+            'ref': response.xpath('//span[@itemprop="name"]/text()').extract_first(),
+            'addr_full': response.xpath('//span[@itemprop="streetAddress"]/text()').extract_first(),
+            'city': response.xpath('//span[@itemprop="addressLocality"]/text()').extract_first(),
+            'state': response.xpath('//span[@itemprop="addressRegion"]/text()').extract_first(),
+            'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first(),
+            'phone': response.xpath('//span[@itemprop="telephone"]/a/text()').extract_first(),
+            'website': response.url,
+            'opening_hours': "".join(response.xpath('//div/div/div/span/span/span/text()').extract()).strip(),
+            'lon': float(response.xpath('//head/script[9]').extract_first().split('"coordinates":[')[1].split(']')[0].split(',')[0]),
+            'lat': float(response.xpath('//head/script[9]').extract_first().split('"coordinates":[')[1].split(']')[0].split(',')[1]),
+        }
+
+        yield GeojsonPointItem(**properties)


Test patch: