internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: alltheplaces/alltheplaces

Issue:
Jewel-Osco
https://local.jewelosco.com/index.html

Looks like it can probably just be added as a start url in the albertsons.py spider.


Discussion:

Resolving patch:
diff --git a/locations/spiders/albertsons.py b/locations/spiders/albertsons.py
--- a/locations/spiders/albertsons.py
+++ b/locations/spiders/albertsons.py
@@ -16,10 +16,14 @@
 class AlbertsonsSpider(scrapy.Spider):
 
     name = "albertsons"
-    allowed_domains = ["local.albertsons.com"]
     download_delay = 0.5
+    allowed_domains = [
+        "local.albertsons.com",
+        "local.jewelosco.com",
+    ]
     start_urls = (
         'https://local.albertsons.com/index.html',
+        'https://local.jewelosco.com/index.html',
     )
 
     def parse_stores(self, response):
@@ -41,7 +45,7 @@ def parse_stores(self, response):
         }
         hours = response.xpath('//div[@class="LocationInfo-right"]/div[1]/div[@class="LocationInfo-hoursTable"]/div[@class="c-location-hours-details-wrapper js-location-hours"]/table/tbody/tr/@content').extract()
         if hours:
-            properties['opening_hours'] = " ;".join(hours)
+            properties['opening_hours'] = "; ".join(hours)
         yield GeojsonPointItem(**properties)
 
     def parse_city_stores(self ,response):


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Burger Fi
https://order.burgerfi.com/locations


Discussion:
This issue hasn't been worked on in a while. I'm removing the assignment so others can work on it. If you're still working on it, please reassign yourself and reference a pull request or branch where you're doing your work.

Resolving patch:
diff --git a/locations/spiders/burger_fi.py b/locations/spiders/burger_fi.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/burger_fi.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import json
+import re
+
+from locations.items import GeojsonPointItem
+
+HEADERS = {
+    'Accept': 'application/json',
+}
+
+class BurgerFiSpider(scrapy.Spider):
+    name = "burger_fi"
+    allowed_domains = [ "order.burgerfi.com" ]
+    start_urls = (
+        "https://order.burgerfi.com/api/restaurants",
+    )
+
+    def start_requests(self):
+        url = self.start_urls[0]
+        yield scrapy.Request(url=url, headers=HEADERS, callback=self.parse)
+
+    def parse(self, response):
+        data = json.loads(response.body_as_unicode())
+
+        for store in data['restaurants']:
+            addr_full = "{}, {}, {} {}".format(store['streetaddress'], store['city'], store['state'], store['zip'])
+            properties = {
+                "ref": store['id'],
+                "name": store['name'],
+                "addr_full": addr_full,
+                "city": store['city'],
+                "state": store['state'],
+                "postcode": store['zip'],
+                "country": store['country'],
+                "lon": float(store['longitude']),
+                "lat": float(store['latitude']),
+                "phone": store['telephone'],
+            }
+
+            yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
anytime_fitness.py null values
The scraper currently includes address2 whether it is null or not, resulting in ", None" being appended to many (most?) of the addr:full fields.


Discussion:

Resolving patch:
diff --git a/locations/spiders/anytime_fitness.py b/locations/spiders/anytime_fitness.py
--- a/locations/spiders/anytime_fitness.py
+++ b/locations/spiders/anytime_fitness.py
@@ -2,6 +2,8 @@
 import scrapy
 from locations.items import GeojsonPointItem
 import json
+import html
+
 
 class AnytimeFitnessSpider(scrapy.Spider):
     name = 'anytime_fitness'
@@ -18,14 +20,13 @@ def parse(self, response):
             yield GeojsonPointItem(
                 lat = gym['latitude'],
                 lon = gym['longitude'],
-                addr_full = '{}, {}'.format(gym['content']['address'], gym['content']['address2']),
+                addr_full = ", ".join(filter(None, [gym['content']['address'], gym['content']['address2']])),
                 city = gym['content']['city'],
                 phone = gym['content']['phone'],
                 state = gym['content']['state_abbr'],
                 postcode = gym['content']['zip'],
                 ref = gym['content']['url'],
-                country = gym['content']['country']
+                country = gym['content']['country'],
+                name = html.unescape(gym['content']['title']),
+                extras = {"number": gym['content']['number']}
                 )
-                
-                
-    


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Simply Self Storage
https://www.simplyss.com/locations?a=map
Simply Self Storage
https://www.simplyss.com/locations?a=map


Discussion:

Resolving patch:
diff --git a/locations/spiders/simply_self_storage.py b/locations/spiders/simply_self_storage.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/simply_self_storage.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+import json
+import re
+from datetime import datetime
+
+import scrapy
+
+from locations.items import GeojsonPointItem
+
+
+class SimplySelfStorageSpider(scrapy.Spider):
+    name = "simply_self_storage"
+    allowed_domains = ["simplyss.com"]
+    start_urls = (
+        "https://www.simplyss.com/site_map/",
+    )
+
+    def parse(self, response):
+        store_urls = response.css(
+            "a.main-page-anchor[href*='/locations/'] + ul a.grey-anchor::attr(href)"
+        )
+        for store_url in store_urls:
+            yield scrapy.Request(
+                store_url.extract(),
+                callback=self.parse_store,
+            )
+
+    def parse_store(self, response):
+        props = json.loads(
+            response.css("script[type='application/ld+json']::text").extract_first()
+        )
+        yield GeojsonPointItem(
+            lat=float(props["geo"]["latitude"]),
+            lon=float(props["geo"]["longitude"]),
+            name=props["name"],
+            addr_full=props["address"]["streetAddress"],
+            city=props["address"]["addressLocality"],
+            state=props["address"]["addressRegion"],
+            postcode=props["address"]["postalCode"],
+            country=props["address"]["addressCountry"],
+            phone=props["contactPoint"]["telephone"],
+            website=response.url,
+            opening_hours=self.convert_opening_hours(props["openingHours"]),
+            ref=response.url,
+        )
+
+    def convert_opening_hours(self, hours_str):
+        """Convert schema.org hours to OSM opening_hours"""
+        hour_groups = []
+        for hour_group in re.split(r"(?<=M)\s", hours_str):
+            # Ignore closed statements
+            if "Closed" in hour_group:
+                continue
+            weekdays, hours = re.split(r"\s(?=\d)", hour_group)
+            weekday_list = weekdays.split(", ")
+            if len(weekday_list) > 1:
+                weekday_str = f"{weekday_list[0]}-{weekday_list[-1]}"
+            else:
+                weekday_str = weekday_list[0]
+            hour_times = [
+                datetime.strptime(hour_str, "%I:%M%p").strftime("%H:%M")
+                for hour_str in hours.split("-")
+            ]
+            hour_groups.append(f"{weekday_str} {'-'.join(hour_times)}")
+        return "; ".join(hour_groups)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
rue21
https://stores.rue21.com/index.html


Discussion:

Resolving patch:
diff --git a/locations/spiders/rue21.py b/locations/spiders/rue21.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/rue21.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+import datetime
+import re
+
+import scrapy
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+class Rue21Spider(scrapy.Spider):
+    download_delay = 0.2
+    name = "rue21"
+    allowed_domains = ["rue21.com"]
+    start_urls = (
+        'https://stores.rue21.com/index.html',
+    )
+
+    def parse_hours(self, elements):
+        opening_hours = OpeningHours()
+
+        hours = elements.xpath('.//tr[@itemprop="openingHours"]/@content').extract()
+
+        for hour in hours:
+
+            if hour.split()[-1].lower() == "closed":
+                continue
+            else:
+                day, open_time, close_time = re.search(
+                    r'([a-z]{2})\s([0-9:]+)-([0-9:]+)', hour, flags=re.IGNORECASE).groups()
+                opening_hours.add_range(day=day,
+                                        open_time=open_time,
+                                        close_time=close_time)
+        return opening_hours.as_opening_hours()
+
+    def parse_store(self, response):
+        ref = re.search(r'.+/(.+).html', response.url).group(1)
+
+        properties = {
+            'addr_full': response.xpath('//span[@class="c-address-street-1"]/text()').extract_first().strip(),
+            'city': response.xpath('//span[@itemprop="addressLocality"]/text()').extract_first(),
+            'state': response.xpath('//abbr[@itemprop="addressRegion"]/text()').extract_first(),
+            'postcode': response.xpath('//span[@itemprop="postalCode"]/text()').extract_first().strip(),
+            'country': response.xpath('//abbr[@itemprop="addressCountry"]/text()').extract_first(),
+            'ref': ref,
+            'website': response.url,
+            'lat': float(response.xpath('//meta[@itemprop="latitude"]/@content').extract_first()),
+            'lon': float(response.xpath('//meta[@itemprop="longitude"]/@content').extract_first()),
+            'name': response.xpath('//span[@class="location-geomodifier"]/text()').extract_first()
+        }
+
+        hours = self.parse_hours(response.xpath('//div[@class="c-location-hours"]//tbody'))
+
+        if hours:
+            properties['opening_hours'] = hours
+
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        urls = response.xpath('//a[@class="c-directory-list-content-item-link"]/@href').extract()
+        is_store_list = response.xpath('//div[@class="location-list-container"]').extract()
+
+        if not urls and is_store_list:
+            urls = response.xpath(
+                '//div[@class="location-list-container"]//a[text()="View Store Details"]/@href').extract()
+        for url in urls:
+            if url.count('/') >= 2:
+                yield scrapy.Request(response.urljoin(url), callback=self.parse_store)
+            else:
+                yield scrapy.Request(response.urljoin(url))


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Chili's
https://www.chilis.com/locations
Chili's
https://www.chilis.com/locations


Discussion:
This issue hasn't been worked on in a while. I'm removing the assignment so others can work on it. If you're still working on it, please reassign yourself and reference a pull request or branch where you're doing your work.
This issue hasn't been worked on in a while. I'm removing the assignment so others can work on it. If you're still working on it, please reassign yourself and reference a pull request or branch where you're doing your work.

Resolving patch:
diff --git a/locations/spiders/chilis.py b/locations/spiders/chilis.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/chilis.py
@@ -0,0 +1,70 @@
+import html
+import json
+import scrapy
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+DAY_MAPPING = {
+    'Monday': 'Mo',
+    'Tuesday': 'Tu',
+    'Wednesday': 'We',
+    'Thursday': 'Th',
+    'Friday': 'Fr',
+    'Saturday': 'Sa',
+    'Sunday': 'Su'
+}
+
+
+class ChilisSpider(scrapy.Spider):
+    name = "chilis"
+    allowed_domains = ["chilis.com"]
+    download_delay = 0.5
+    start_urls = (
+        'https://www.chilis.com/locations/us/all',
+    )
+
+    def parse_hours(self, hours):
+        opening_hours = OpeningHours()
+
+        for hour in hours:
+            opening_hours.add_range(day=DAY_MAPPING[hour["dayOfWeek"]],
+                                    open_time=hour["opens"],
+                                    close_time=hour["closes"])
+
+        return opening_hours.as_opening_hours()
+
+    def parse_store(self, response):
+        scripts = response.xpath('//script[@type="application/ld+json"]/text()').extract()
+        data = [json.loads(x) for x in scripts if json.loads(x)["@type"] == "Restaurant"][0]
+
+        properties = {
+            'addr_full': html.unescape(data["address"]["streetAddress"]),
+            'phone': data["telephone"],
+            'city': data["address"]["addressLocality"],
+            'state': data["address"]["addressRegion"],
+            'postcode': data["address"]["postalCode"],
+            'country': 'US',
+            'ref': data["branchCode"],
+            'website': response.url,
+            'lat': float(data["geo"]["latitude"]),
+            'lon': float(data["geo"]["longitude"]),
+            'name': html.unescape(data["name"])
+        }
+        hours = self.parse_hours(data["openingHoursSpecification"])
+
+        if hours:
+            properties['opening_hours'] = hours
+
+        yield GeojsonPointItem(**properties)
+
+    def parse_city(self, response):
+        urls = response.xpath('//a[text()="Details"]/@href').extract()
+        for url in urls:
+            yield scrapy.Request(response.urljoin(url), callback=self.parse_store)
+
+    def parse(self, response):
+        urls = response.xpath('//div[contains(@class, "city-locations")]//a[@class="city-link"]/@href').extract()
+        for url in urls:
+            yield scrapy.Request(response.urljoin(url), callback=self.parse_city)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Jason's Deli
https://www.jasonsdeli.com/restaurants


Discussion:

Resolving patch:
diff --git a/locations/spiders/jasonsdeli.py b/locations/spiders/jasonsdeli.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/jasonsdeli.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+import datetime
+import re
+
+import scrapy
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+class JasonsDeliSpider(scrapy.Spider):
+    download_delay = 0.2
+    name = "jasonsdeli"
+    allowed_domains = ["jasonsdeli.com"]
+    start_urls = (
+        'https://www.jasonsdeli.com/restaurants',
+    )
+
+    def parse_hours(self, elements):
+        opening_hours = OpeningHours()
+
+        for item in elements:
+            day, open_time, close_time = re.search(
+                r'([a-z]{3}):.([0-9:\sAPM]+)\s-\s([0-9:\sAPM]+)', item, flags=re.IGNORECASE).groups()
+            opening_hours.add_range(day=day[0:2],
+                                    open_time=datetime.datetime.strptime(open_time, '%I:%M %p').strftime('%H:%M'),
+                                    close_time=datetime.datetime.strptime(close_time, '%I:%M %p').strftime('%H:%M'))
+        return opening_hours.as_opening_hours()
+
+    def parse_store(self, response):
+        ref = re.search(r'.+/(.+)', response.url).group(1)
+
+        properties = {
+            'addr_full': response.xpath('//div[@class="address"]/text()').extract_first()[0].split('\n')[0],
+            'city': response.xpath('//div[@class="address"]/text()').extract()[-1].split(',')[0],
+            'state': response.xpath('//div[@class="address"]/text()').extract()[-1].split(', ')[1].split(' ')[-2],
+            'postcode': response.xpath('//div[@class="address"]/text()').extract()[-1].split(', ')[1].split(' ')[-1],
+            'ref': ref,
+            'website': response.url,
+            'phone': response.xpath('//a[@class="cnphone"]/text()').extract_first()
+        }
+
+        hours = self.parse_hours(response.xpath('//div[@class="loc-hours"]/p/text()').extract())
+
+        try:
+            bus_name = response.xpath('//div[@class="loc-title"]/text()').extract()[0].split(': ')[1]
+        except IndexError:
+            bus_name = response.xpath('//div[@class="loc-title"]/text()').extract_first()
+        properties['name'] = bus_name
+
+        if hours:
+            properties['opening_hours'] = hours
+
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        urls = response.xpath('//span[@class="field-content"]/a/@href').extract()
+
+        for url in urls:
+            yield scrapy.Request(response.urljoin(url), callback=self.parse_store)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Hyatt Hotels
https://www.hyatt.com/explore-hotels
Hyatt Hotels
https://www.hyatt.com/explore-hotels


Discussion:

Resolving patch:
diff --git a/locations/spiders/hyatt.py b/locations/spiders/hyatt.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/hyatt.py
@@ -0,0 +1,52 @@
+import json
+import scrapy
+import logging
+
+from locations.items import GeojsonPointItem
+
+
+class HyattSpider(scrapy.Spider):
+
+    name = "hyatt"
+    allowed_domains = ["hyatt.com"]
+    download_delay = 1.5  # delay required to avoid getting temporarily blocked from hyatt.com (403s)
+
+    base_url = 'https://www.hyatt.com/explore-hotels/partial?regionGroup={region}&categories=&brands='
+
+    regions = (
+        '1-NorthAmerica',
+        '2-CaribbeanAndLatinAmerica',
+        '3-Europe',
+        '4-AfricaAndMiddleEast',
+        '5-Asia',
+        '6-Australia'
+    )
+
+    def start_requests(self):
+        for region in self.regions:
+            yield scrapy.Request(url=self.base_url.format(region=region))
+
+    def parse_hotel(self, response):
+        data = json.loads(response.xpath('//script[@type="application/ld+json"]/text()').extract_first())[0]
+
+        properties = {
+            'ref': "_".join(response.url.split('/')[-3:]),
+            'name': data["name"],
+            'addr_full': data["address"]["streetAddress"],
+            'city': data["address"]["addressLocality"],
+            'state': data["address"]["addressRegion"],
+            'postcode': data["address"]["postalCode"],
+            'country': data["address"]["addressCountry"],
+            'phone': data.get("telephone", None),
+            'lat': float(data["geo"]["latitude"]),
+            'lon': float(data["geo"]["longitude"]),
+            'website': response.url,
+        }
+
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        urls = response.xpath('//li[contains(@class, "property")]/a/@href').extract()
+
+        for url in urls:
+            yield scrapy.Request(response.urljoin(url), headers={'Referer': None}, callback=self.parse_hotel)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
McCoy's Building Supply
https://www.mccoys.com/stores


Discussion:

Resolving patch:
diff --git a/locations/spiders/mccoys.py b/locations/spiders/mccoys.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/mccoys.py
@@ -0,0 +1,64 @@
+import scrapy
+import re
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+class McCoysSpider(scrapy.Spider):
+    name = "mccoys"
+    allowed_domains = ["www.mccoys.com"]
+    download_delay = 0.5
+    start_urls = (
+        'https://www.mccoys.com/stores',
+    )
+
+    def parse_hours(self, hours):
+
+        opening_hours = OpeningHours()
+        for i in range(0, len(hours), 2):
+            day = hours[i]
+            open_time, close_time = hours[i+1].split('-')
+            opening_hours.add_range(day=day[:2],
+                                    open_time=open_time.strip(),
+                                    close_time=close_time.strip(),
+                                    time_format='%I:%M %p')
+        return opening_hours.as_opening_hours()
+
+    def parse_store(self, response):
+        properties = response.meta["properties"]
+
+        address1, address2 = response.xpath('//a[contains(@href, "maps")]/text()').extract()
+        city, state, zipcode = re.search(r'^(.*),\s+([a-z]{2})\s+([0-9]{5})$', address2.strip(), re.IGNORECASE).groups()
+
+        properties.update({
+            'addr_full': address1.strip(),
+            'phone': response.xpath('//div[contains(text(), "Call")]/span/text()').extract_first(),
+            'city': city,
+            'state': state,
+            'postcode': zipcode,
+            'ref': re.search(r'.*stores\/(.+)\?*.*', response.url).groups()[0],
+            'website': response.url
+        })
+
+        hours = response.xpath('//div[contains(text(), "Store Hours")]/following-sibling::table//td/text()').extract()
+        opening_hours = self.parse_hours(hours)
+        if hours:
+            properties['opening_hours'] = opening_hours
+
+        yield GeojsonPointItem(**properties)
+
+    def parse(self, response):
+        stores = response.xpath('//div[@id="mc-filtered-stores-container"]/div/div')
+
+        for store in stores:
+            properties = {
+                "lat": float(store.xpath('./@data-lat').extract_first()),
+                "lon": float(store.xpath('./@data-long').extract_first()),
+                "name": store.xpath('.//div[@class="mc-bold-text"]/text()').extract_first().strip(),
+            }
+            url = store.xpath('.//a[contains(text(), "Store Information")]/@href').extract_first()
+
+            yield scrapy.Request(response.urljoin(url),
+                                 callback=self.parse_store,
+                                 meta={"properties": properties})


Test patch: