internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: alltheplaces/alltheplaces

Issue:
QuikTrip
https://www.quiktrip.com/Locations


Discussion:

Resolving patch:
diff --git a/locations/spiders/quiktrip.py b/locations/spiders/quiktrip.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/quiktrip.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+import scrapy
+
+from locations.items import GeojsonPointItem
+
+
+class QuiktripSpider(scrapy.Spider):
+    name = "quiktrip"
+    allowed_domains = ["quiktrip.com", "where2getit.com"]
+    start_urls = (
+        'https://hosted.where2getit.com/quiktrip/ajax?&xml_request=%3Crequest%3E%3Cappkey%3E82C11D38-0EC6-11E0-8AD9-4C59241F5146%3C%2Fappkey%3E%3Cformdata+id%3D%22locatorsearch%22%3E%3Cdataview%3Estore_default%3C%2Fdataview%3E%3Climit%3E1000%3C%2Flimit%3E%3Cgeolocs%3E%3Cgeoloc%3E%3Caddressline%3Ekansas%3C%2Faddressline%3E%3Clongitude%3E%3C%2Flongitude%3E%3Clatitude%3E%3C%2Flatitude%3E%3C%2Fgeoloc%3E%3C%2Fgeolocs%3E%3Csearchradius%3E1400%3C%2Fsearchradius%3E%3Cwhere%3E%3Ctravelcenter%3E%3Ceq%3E%3C%2Feq%3E%3C%2Ftravelcenter%3E%3Ctruckdiesel%3E%3Ceq%3E%3C%2Feq%3E%3C%2Ftruckdiesel%3E%3Ce15%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fe15%3E%3Cautodiesel%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fautodiesel%3E%3Cspecialtydrinks%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fspecialtydrinks%3E%3Cgen3%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fgen3%3E%3Chotandfreshpretzels%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fhotandfreshpretzels%3E%3Chotsandwiches%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fhotsandwiches%3E%3Cxlpizza%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fxlpizza%3E%3Cpersonalpizzas%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fpersonalpizzas%3E%3Cnoethanol%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fnoethanol%3E%3Ccertifiedscales%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fcertifiedscales%3E%3Cdef%3E%3Ceq%3E%3C%2Feq%3E%3C%2Fdef%3E%3Cfrozentreats%3E%3Ceq%3E%3C%2Feq%3E%3C%2Ffrozentreats%3E%3Cfreshbrewedtea%3E%3Ceq%3E%3C%2Feq%3E%3C%2Ffreshbrewedtea%3E%3Cfrozendrinks%3E%3Ceq%3E%3C%2Feq%3E%3C%2Ffrozendrinks%3E%3C%2Fwhere%3E%3C%2Fformdata%3E%3C%2Frequest%3E',
+    )
+
+    def parse(self, response):
+        shops = response.xpath('//collection/poi')
+
+        for shop in shops:
+            yield GeojsonPointItem(
+                ref=shop.xpath('.//clientkey/text()').extract_first(),
+                phone=shop.xpath('.//phone/text()').extract_first(),
+                addr_full=shop.xpath('.//address1/text()').extract_first(),
+                postcode=shop.xpath('.//postalcode/text()').extract_first(),
+                city=shop.xpath('.//city/text()').extract_first(),
+                state=shop.xpath('.//state/text()').extract_first(),
+                country=shop.xpath('.//country/text()').extract_first(),
+                lat=float(shop.xpath('.//latitude/text()').extract_first()),
+                lon=float(shop.xpath('.//longitude/text()').extract_first()),
+            )


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
GreatClips
Looks like location search only? https://www.greatclips.com/find-a-salon-online-check-in


Discussion:

Resolving patch:
diff --git a/locations/spiders/greatclips.py b/locations/spiders/greatclips.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/greatclips.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+
+from locations.items import GeojsonPointItem
+
+
+class GreatclipsSpider(scrapy.Spider):
+    name = "greatclips"
+    allowed_domains = ["greatclips.com", "stylewaretouch.net"]
+    start_urls = (
+        'https://www.stylewaretouch.net/checkin/wa/jsonMarkers?client=locator&lat=-34&lng=85&tzoffset=200&callback=onSuccess&failureCallback=onFailure&stores=',
+    )
+
+    def store_hours(self, store_hours):
+        if not store_hours or len(store_hours) < 5:
+            return
+
+        stri = ''
+        match = re.search(r'(\d{1,2})(:(\d{1,2}))?\s*(am|AM|mp|PM|pm)?\s*-\s*(\d{1,2})(:(\d{1,2}))?\s*(am|pm|mp|PM|AM)',store_hours)
+
+        if not match:
+            return store_hours
+
+        stri += str(int(match[1])+(12 if match[4] in ['pm','mp','PM'] else 0)) +match[2]+'-'
+        stri += str(int(match[5])+(12 if match[8] in ['pm','mp','PM'] else 0)) +match[6]+';'
+
+        return stri.rstrip(';')
+
+    def parse(self, response):
+        # all shops grouped in ther database by codes experimentally was
+        # uncovered all groups lies in 1..10000 range (checked till 100000)
+        for i in range(1, 10000):
+            yield scrapy.Request(
+                self.start_urls[0]+str(i),
+                callback=self.parse_shop,
+            )
+
+    def parse_shop(self, response):
+        if response.text.find('onFailure') != -1:
+            return
+
+        shops = json.loads(response.text.lstrip('onSuccess(').rstrip(')'))
+        if not shops:
+            return
+
+        for shop in shops:
+            props = {}
+
+            props['ref'] = shop['name']
+            props['phone'] = shop['phone']
+            props['addr_full'] = shop['Thoroughfare']
+            props['postcode'] = shop['ZIP']
+            props['state'] = shop['State']
+            props['city'] = shop['City']
+            props['lat'] = float(shop['latitude'])
+            props['lon'] = float(shop['longitude'])
+            props['website'] = 'https://www.greatclips.com/salons/'+shop['number']
+            props['country'] = shop['CountryCode']
+
+            props['opening_hours'] = 'Mo-Fr '+self.store_hours(shop['hours_mon'])+';Sa '+self.store_hours(shop['hours_sat'])+';Su '+self.store_hours(shop['hours_sun'])
+
+            yield GeojsonPointItem(**props)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Microsoft Store
https://www.microsoft.com/en-us/store/locations/all-locations


Discussion:
Is there a minimum number of locations necessary for us to be able to scrape a site?
Sure. Did you check the url? We can get the Microsoft stores

Resolving patch:
diff --git a/locations/spiders/microsoft.py b/locations/spiders/microsoft.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/microsoft.py
@@ -0,0 +1,33 @@
+import scrapy
+import re
+from locations.items import GeojsonPointItem
+
+class MicrosoftSpider(scrapy.Spider):
+
+    name = "microsoft"
+    allowed_domains = ["www.microsoft.com"]
+    download_delay = 0.5
+    start_urls = (
+        'https://www.microsoft.com/en-us/store/locations/all-locations',
+    )
+
+    def parse_stores(self, response):
+        properties = {
+            'addr_full': response.xpath('normalize-space(//div[@itemprop="streetAddress"]/span/text())').extract_first(),
+            'phone': response.xpath('normalize-space(//div[@itemprop="telephone"]/span/text())').extract_first(),
+            'city': response.xpath('normalize-space(//span[@itemprop="addressLocality"]/text())').extract_first(),
+            'state': response.xpath('normalize-space(//span[@itemprop="addressRegion"]/text())').extract_first(),
+            'postcode': response.xpath('normalize-space(//span[@itemprop="postalCode"]/text())').extract_first().replace(u'\xa0',''),
+            'ref': re.findall(r"[^\/]+$" ,response.url)[0],
+            'website': response.url,
+            'lat': float(response.xpath('normalize-space(//meta[@itemprop="latitude"]/@content)').extract_first()),
+            'lon': float(response.xpath('normalize-space(//meta[@itemprop="longitude"]/@content)').extract_first()),
+        }
+
+        yield GeojsonPointItem(**properties)
+
+
+    def parse(self, response):
+        urls = response.xpath('//ul[@instancename="Cities list"]/li/a/@href').extract()
+        for path in urls:
+            yield scrapy.Request(response.urljoin(path), callback=self.parse_stores)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
The Fresh Market
https://www.thefreshmarket.com


Discussion:

Resolving patch:
diff --git a/locations/spiders/freshmarket.py b/locations/spiders/freshmarket.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/freshmarket.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+
+from locations.items import GeojsonPointItem
+
+class FreshMarketSpider(scrapy.Spider):
+    name = "freshmarket"
+    allowed_domains = ['thefreshmarket.com']
+    start_urls = (
+        'https://www.thefreshmarket.com/your-market/store-locator/',
+    )
+
+    def parse(self, response):
+        json_data = response.xpath('//script[@data-reactid="39"]/text()').extract_first().rstrip(';').split('=')[-1]
+        data = json.loads(json_data)
+        allStores =  data['stores']['allStores']
+        for store in allStores:
+
+            properties = {
+            'name': store['storeName'],
+            'ref': store['storeNumber'],
+            'addr_full': store['address'],
+            'city': store['city'],
+            'state': store['state'],
+            'postcode': store['postalCode'],
+            'phone': store['phoneNumber'],
+            'website': "https://www.thefreshmarket.com/my-market/store/" + store['slug'],
+            'opening_hours': store['moreStoreHours'],
+            'lat': float(store['storeLocation']['lat']),
+            'lon': float(store['storeLocation']['lon']),
+            }
+
+            yield GeojsonPointItem(**properties)
\ No newline at end of file


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
PDQ
http://eatpdq.com/


Discussion:

Resolving patch:
diff --git a/locations/spiders/pdq.py b/locations/spiders/pdq.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/pdq.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+import time
+from scrapy.utils.gz import gunzip, is_gzipped
+
+from locations.items import GeojsonPointItem
+
+class PDQSpider(scrapy.Spider):
+    name = "pdq"
+    allowed_domains = ['eatpdq.qatserver.com']
+    start_urls = (
+        'http://eatpdq.qatserver.com/sitemap/sitemap.gz',
+    )
+
+    def parse(self, response):
+
+        sitemap = gunzip(response.body)
+        regex = re.compile(r'http://eatpdq.qatserver.com/locations/\S+(?=</loc>)')
+        city_urls = re.findall(regex, str(sitemap))
+
+        for path in city_urls:
+            if path.strip() == "http://eatpdq.qatserver.com/locations/find-a-location":
+                pass
+            else:
+                yield scrapy.Request(
+                    path.strip(),
+                    callback=self.parse_store,
+                )
+
+    def parse_store(self, response):
+        if "Sitefinity trial version" in response.xpath('//title/text()').extract():
+            yield scrapy.Request(
+                response.request.url,
+                callback=self.parse_store,
+            )
+        else:
+            if response.xpath('//div[@class="hours"]/div').extract():
+                storeHoursHTML = str(response.xpath('//div[@class="hours"]/div').extract()).replace("'",'').replace('[','').replace(']','')
+                p = re.compile(r'<.*?>')
+                storeHours = p.sub('', storeHoursHTML)
+                storeHours = storeHours.replace('\\n', ' - ')
+                storeHours = "".join(storeHours.strip())
+            else:
+                storeHours = response.xpath('//div[@class="hours"]/div').extract()
+
+            properties = {
+            'name': response.xpath('//div[@class="name"]/h1/text()').extract_first(),
+            'ref': response.xpath('//div[@class="name"]/h1/text()').extract_first(),
+            'addr_full': response.xpath('//div[@class="address"]/text()').extract_first().strip(),
+            'city': response.xpath('//div[@class="address"]/text()[2]').extract_first().split(',')[0],
+            'state': response.xpath('//div[@class="address"]/text()[2]').extract_first().split()[1],
+            'postcode': response.xpath('//div[@class="address"]/text()[2]').extract_first().split()[-1],
+            'phone': response.xpath('//tel/a[@href]/text()').extract_first(),
+            'website': response.request.url,
+            'opening_hours': storeHours,
+            'lat': float(response.xpath('//div[@class="address"]/a/@href').extract_first().split('@')[1].split(',')[0]),
+            'lon': float(response.xpath('//div[@class="address"]/a/@href').extract_first().split('@')[1].split(',')[1]),
+            }
+
+            yield GeojsonPointItem(**properties)
\ No newline at end of file


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
AT&T
https://www.att.com/stores/texas/
https://www.att.com/stores/Texas/Austin/


Discussion:

Resolving patch:
diff --git a/locations/spiders/att.py b/locations/spiders/att.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/att.py
@@ -0,0 +1,50 @@
+import scrapy
+from locations.items import GeojsonPointItem
+import json
+
+
+class ATTScraper(scrapy.Spider):
+    name = "att"
+
+    start_urls = ['https://www.att.com/sitemapfiles/stores-sitemap.xml']
+    api_base = "https://www.att.com/apis/maps/v2/locator/place/cpid/{}.json"
+
+    def process_hours(self, result):
+        ret = []
+        for day in ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]:
+            open, close = result[day.lower() + "_open"], result[day.lower() + "_close"]
+            if "-1" not in (open, close):
+                ret.append("{} {}-{}".format(day[:2], open, close))
+
+        return "; ".join(ret)
+
+    def parse(self, response):
+        response.selector.remove_namespaces()
+        locations = response.xpath('//url/loc/text()')
+        for loc in locations:
+            url = loc.extract()
+            store_id = url.rsplit('/', 1)[-1]
+            yield scrapy.Request(
+                self.api_base.format(store_id),
+                callback=self.parse_location,
+                meta={"url": loc.extract()}
+            )
+
+    def parse_location(self, response):
+        results = json.loads(response.body_as_unicode())["results"]
+        if results:
+            store_data = results[0]
+            yield GeojsonPointItem(
+                lat=store_data["lat"],
+                lon=store_data["lon"],
+                name=store_data["name"],
+                addr_full=(store_data["address1"] + " "
+                          + (store_data.get("address2") or "")).strip(),
+                city=store_data["city"],
+                state=store_data["region"],
+                postcode=store_data["postal"],
+                phone=store_data["phone"].strip(),
+                website=response.meta["url"],
+                opening_hours=self.process_hours(store_data),
+                ref=store_data["id"]
+            )


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Curves for Women
https://www.curves.com/curves-international
https://www.curves.com/locations/


Discussion:

Resolving patch:
diff --git a/locations/spiders/curves.py b/locations/spiders/curves.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/curves.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+
+from locations.items import GeojsonPointItem
+
+
+class CurvesSpider(scrapy.Spider):
+    name = "curves"
+    allowed_domains = ['curves.com']
+    start_urls = (
+        'https://www.curves.com/sitemap.xml',
+    )
+
+    def parse(self, response):
+        response.selector.remove_namespaces()
+        city_urls = response.xpath('//url/loc/text()').extract()
+        regex = re.compile(r'https://www.curves.com/locations/\S+')
+        for path in city_urls:
+            if re.search(regex,path):
+                yield scrapy.Request(
+                    path.strip(),
+                    callback=self.parse_store,
+                )
+            else:
+                pass
+
+    def parse_store(self, response):
+
+        if response.xpath('//div[@class="field field-name-field-franchise-club-hours field-type-text-long field-label-hidden"]/div/div').extract_first():
+            storeHoursHTML = str(response.xpath('//div[@class="field field-name-field-franchise-club-hours field-type-text-long field-label-hidden"]/div/div').extract_first())
+            p = re.compile(r'<.*?>')
+            storeHours = p.sub(' ', storeHoursHTML)
+            # storeHours = storeHours.replace('\\n', ' - ')
+            storeHours = storeHours.strip()
+        else:
+            storeHours = response.xpath('//div[@class="field field-name-field-franchise-club-hours field-type-text-long field-label-hidden"]/div/div').extract_first()
+
+        properties = {
+        'name': response.xpath('//h1/text()').extract_first(),
+        'ref': response.xpath('//h1/text()').extract_first(),
+        'addr_full': response.xpath('//div[@class="thoroughfare"]/text()').extract_first(),
+        'city': response.xpath('//div[@class]/span[@class="locality"]/text()').extract_first(),
+        'state': response.xpath('//div[@class]/span[@class="state"]/text()').extract_first(),
+        'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first(),
+        'country': response.xpath('//span[@class="country"]/text()').extract_first(),
+        'phone': response.xpath('//div[@class="field field-name-field-franchise-phone field-type-telephone field-label-hidden"]/div/div[@class="field-item even"]/text()').extract_first(),
+        'website': response.request.url,
+        'opening_hours': storeHours,
+        # 'lon': float(response.xpath('//head/script[9]').extract_first().split('"coordinates":[')[1].split(']')[0].split(',')[0]),
+        # 'lat': float(response.xpath('//head/script[9]').extract_first().split('"coordinates":[')[1].split(']')[0].split(',')[1]),
+        }
+
+        yield GeojsonPointItem(**properties)
\ No newline at end of file


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Jersey Mikes
https://www.jerseymikes.com


Discussion:

Resolving patch:
diff --git a/locations/spiders/jerseymikes.py b/locations/spiders/jerseymikes.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/jerseymikes.py
@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import re
+
+from locations.items import GeojsonPointItem
+
+
+class JerseyMikesSpider(scrapy.Spider):
+    name = "jerseymikes"
+    allowed_domains = ['jerseymikes.com']
+    start_urls = (
+        'https://www.jerseymikes.com/locations/all',
+    )
+
+    def parse(self, response):
+        response.selector.remove_namespaces()
+        city_urls = response.xpath('//div[@class="content"]/ul/li/a/@href').extract()
+        for path in city_urls:
+            yield scrapy.Request(
+                path.strip(),
+                callback=self.parse_location,
+            )
+
+    def parse_location(self, response):
+        response.selector.remove_namespaces()
+        location_urls = response.xpath('//div[@class="pure-g location-list location-list-striped"]/div/div/div/a/@href').extract()
+        for path in location_urls:
+            yield scrapy.Request(
+                "https://www.jerseymikes.com" + path.strip(),
+                callback=self.parse_store,
+            )
+
+    def parse_store(self, response):
+
+        properties = {
+        'name': response.xpath('//title/text()').extract_first(),
+        'ref': response.xpath('//title/text()').extract_first(),
+        'addr_full': response.xpath('//span[@class="addr1"]/text()').extract_first(),
+        'city': response.xpath('//strong[@class="city"]/text()').extract_first(),
+        'state': response.xpath('//strong[@class="state"]/text()').extract_first(),
+        'postcode': response.xpath('//span[@class="zip"]/text()').extract_first(),
+        'phone': response.xpath('//a[@class="tel"]/text()').extract_first(),
+        'website': response.request.url,
+        'opening_hours': response.xpath('//div[@class="location-hours"]/strong/text()').extract_first(),
+        'lat': float(response.xpath('//a[@class="pure-button full-width margin-bottom"]/@href').extract_first().split('/')[-1].split(',')[0]),
+        'lon': float(response.xpath('//a[@class="pure-button full-width margin-bottom"]/@href').extract_first().split('/')[-1].split(',')[1]),
+        }
+
+        yield GeojsonPointItem(**properties)
\ No newline at end of file


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
HEB
https://www.heb.com/store-locations


Discussion:
If this is approved, how do I assign it to myself? does someone have to add me as a contributor?
sorry, I posted this for me, can I complete this job?

Resolving patch:
diff --git a/locations/spiders/heb.py b/locations/spiders/heb.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/heb.py
@@ -0,0 +1,63 @@
+import scrapy
+from locations.items import GeojsonPointItem
+import re
+from xml.dom import minidom
+
+
+class HEBSpider(scrapy.Spider):
+    name = "heb"
+
+    start_urls = ["https://www.heb.com/sitemap/storeSitemap.xml"]
+
+    def parse(self, response):
+        document = minidom.parseString(response.body_as_unicode())
+        for url in (node.firstChild.nodeValue for node in document.getElementsByTagName("loc")):
+            yield scrapy.Request(url=url, callback=self.parse_store, meta={"url": url})
+
+    def get_hours(self, sidebar):
+        hours = []
+        ret = []
+        is_hours = False
+        for hour in sidebar:
+            if hour.strip().startswith("Store Hours"):
+                is_hours = True
+                continue
+            if is_hours:
+                if "AM" in hour or "PM" in hour:
+                    hours.append(hour)
+                else:
+                    break
+        for hour in hours:
+            hour = hour.strip()
+            range, start, _, _, _, end, _ = hour.split(" ")
+            split_end = end.split(":")
+            new_end = "%02d:%s" % (int(split_end[0]) + 12, split_end[1])
+            if "-" in range:
+                new_range = "{}-{}".format(*[x[:2] for x in range.split("-")])
+            else:
+                new_range = range[:2]
+            ret.append("{} {}-{}".format(new_range, start, new_end))
+        return "; ".join(ret)
+
+    def parse_store(self, response):
+        dat = re.findall(r"search_createMapPin\(.*\);", response.body_as_unicode())
+        if dat:
+            pin_call = dat[0]
+            store_str = pin_call[len("search_createMapPin("):-2]
+            if len(store_str.split(",")) == 17:
+                _, url, lat, lon, _, _, name, address, city, state, zip, _, phone, _, _, _, _ = store_str.split(",")
+                if lat and lon:
+                    sidebar = response.css(".hoursDetail span::text, .hoursDetail p:not(:contains('\t'))::text").extract()
+                    yield GeojsonPointItem(
+                        lat=float(lat),
+                        lon=float(lon),
+                        name=name.strip('"'),
+                        addr_full=address.strip('"'),
+                        city=city.strip('"'),
+                        state=state.strip('"'),
+                        postcode=zip.strip('"'),
+                        phone=phone.replace(" - ", "-").strip('"'),
+                        website=response.meta.get("url"),
+                        opening_hours=self.get_hours(sidebar),
+                        ref=response.meta.get("url")
+                    )


Test patch: