internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: alltheplaces/alltheplaces

Issue:
Cafe Zupas
https://cafezupas.com/locations


Discussion:

Resolving patch:
diff --git a/locations/spiders/cafe_zupas.py b/locations/spiders/cafe_zupas.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/cafe_zupas.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+import scrapy
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+class CafeZupasSpider(scrapy.Spider):
+    name = "cafe_zupas"
+    item_attributes = {"brand": "Cafe Zupas"}
+    allowed_domains = ["cafezupas.com"]
+    start_urls = [
+        "https://cafezupas.com/server.php?url=https://api.controlcenter.zupas.com/api/markets/listing"
+    ]
+
+    def parse_hours(self, location):
+        opening_hours = OpeningHours()
+        timings = [
+            ["mon_thurs_timings_open", "mon_thurs_timings_close"],
+            ["fri_sat_timings_open", "fri_sat_timings_close"],
+        ]
+        for i in timings:
+            if location[i[0]] in ["Closed", ""] or location[i[1]] in ["Closed", ""]:
+                continue
+            # Monday - Thursday
+            if i[0] == "mon_thurs_timings_open":
+                day_group_1 = ["MO", "TU", "WE", "TH"]
+                for day in day_group_1:
+                    opening_hours.add_range(
+                        day=day,
+                        open_time=location[i[0]],
+                        close_time=location[i[1]],
+                        time_format="%I:%M %p",
+                    )
+            # Friday - Saturday
+            if i[0] == "fri_sat_timings_open":
+                day_group_2 = ["FR", "SA"]
+                for day in day_group_2:
+                    opening_hours.add_range(
+                        day=day,
+                        open_time=location[i[0]],
+                        close_time=location[i[1]],
+                        time_format="%I:%M %p",
+                    )
+        return opening_hours.as_opening_hours()
+
+    def parse(self, response):
+        data = response.json()
+        for i in data["data"]["data"]:
+            for location in i["locations"]:
+
+                properties = {
+                    "ref": location["id"],
+                    "website": "https://cafezupas.com/locationcopy/info/"
+                    + location["name"].lower().replace(" ", "-"),
+                    "name": location["name"],
+                    "image": "https://cafezupas.com" + location["image"]
+                    if location["image"] is not None
+                    else None,
+                    "phone": location["phone"],
+                    "lat": location["lat"],
+                    "lon": location["long"],
+                    "addr_full": location["address"],
+                    "city": location["city"],
+                    "state": location["state"],
+                    "postcode": location["zip"],
+                    "facebook": location["facebook_url"],
+                    "opening_hours": self.parse_hours(location),
+                }
+                yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Cheesecake Factory
http://locations.thecheesecakefactory.com/


Discussion:

Resolving patch:
diff --git a/locations/spiders/thecheesecakefactory.py b/locations/spiders/thecheesecakefactory.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/thecheesecakefactory.py
@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from scrapy.spiders import SitemapSpider
+
+from locations.hours import OpeningHours
+from locations.structured_data_spider import StructuredDataSpider
+
+
+class TheCheesecakeFactorySpider(SitemapSpider, StructuredDataSpider):
+    download_delay = 0.1
+    name = "thecheesecakefactory"
+    item_attributes = {"brand": "The Cheesecake Factory", "brand_wikidata": "Q1045842"}
+    allowed_domains = [
+        "www.thecheesecakefactory.com",
+        "locations.thecheesecakefactory.com",
+    ]
+    sitemap_urls = ["https://locations.thecheesecakefactory.com/robots.txt"]
+    sitemap_rules = [("", "parse_sd")]


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
DAVIDsTEA
https://locations.davidstea.com


Discussion:

Resolving patch:
diff --git a/locations/spiders/davidstea.py b/locations/spiders/davidstea.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/davidstea.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from scrapy.spiders import SitemapSpider
+
+from locations.hours import OpeningHours
+from locations.structured_data_spider import StructuredDataSpider
+
+
+class UrbanOutfitters(SitemapSpider, StructuredDataSpider):
+    download_delay = 0.1
+    name = "davidstea"
+    item_attributes = {"brand": "David's Tea", "brand_wikidata": "Q3019129"}
+    allowed_domains = [
+        "locations.davidstea.com",
+    ]
+    sitemap_urls = ["https://locations.davidstea.com/robots.txt"]
+    sitemap_rules = [("", "parse_sd")]
+
+    def post_process_item(self, item, response, ld_data, **kwargs):
+        item["website"] = response.url
+        yield item


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Cooplands
https://cooplands.co.uk/shop-locations


Discussion:
I'll tackle this one. 👍 

Resolving patch:
diff --git a/locations/spiders/cooplands_doncaster.py b/locations/spiders/cooplands_doncaster.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/cooplands_doncaster.py
@@ -0,0 +1,36 @@
+import scrapy
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours, DAYS
+
+
+class CooplandsDoncasterSpider(scrapy.Spider):
+    name = "cooplands_doncaster"
+    allowed_domains = ["cooplands.co.uk"]
+    start_urls = ["https://cooplands.co.uk/shop-locations"]
+
+    hours = OpeningHours()
+    for DAY in DAYS[0:6]:
+        hours.add_range(day=DAY, open_time="08:00", close_time="17:00")
+
+    item_attributes = {
+        "brand": "Cooplands",
+        "brand_wikidata": "Q96622197",
+        "country": "GB",
+        "opening_hours": hours.as_opening_hours(),
+    }
+
+    def parse(self, response):
+        stores = response.xpath("//div[@class='box box-store']")
+
+        for index, store in enumerate(stores):
+            data = store.xpath("ul/li/text()").extract()
+            addr_full = data[0].strip() + data[1] if len(data) == 4 else ""
+
+            yield GeojsonPointItem(
+                ref=index,
+                name=store.xpath("h4/text()").extract_first(),
+                addr_full=addr_full,
+                postcode=data[-2].strip(),
+                phone=data[-1].strip(),
+            )


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Gelson's
https://www.gelsons.com/stores


Discussion:
Hi, can I work on this?
Thanks
Go for it :)

Resolving patch:
diff --git a/locations/spiders/gelsons.py b/locations/spiders/gelsons.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/gelsons.py
@@ -0,0 +1,63 @@
+import json
+import scrapy
+
+from locations.dict_parser import DictParser
+from locations.hours import OpeningHours, DAYS
+
+
+class GelsonsSpider(scrapy.spiders.SitemapSpider):
+    name = "gelsons"
+    item_attributes = {
+        "brand": "Gelson's Markets",
+        "brand_wikidata": "Q16993993",
+        "country": "US",
+    }
+
+    download_delay = 0.2
+    sitemap_urls = ["https://www.gelsons.com/sitemap.xml"]
+    sitemap_rules = [("/stores/", "parse_store")]
+
+    def sitemap_filter(self, entries):
+        for entry in entries:
+            if "virtual" in entry["loc"]:
+                continue
+            yield entry
+
+    def parse_store(self, response):
+
+        content = json.loads(
+            response.xpath('//script[@type="application/json"]/text()').extract_first()
+        )["props"]["pageProps"]
+        store_json = content["store"]
+        hours = content["pageComponents"][0]["headline"]
+
+        item = DictParser.parse(store_json)
+        item["phone"] = store_json["storePhone"]
+        item["website"] = response.url
+        item["opening_hours"] = self.parse_hours(hours)
+
+        yield item
+
+    def parse_hours(self, hour_string):
+        """Hours look like one of the following:
+
+        Hours: 7am - 9pm, 7 days a week
+        Hours: 7am - 9:30pm, 7 days a week
+
+        """
+        hours = OpeningHours()
+        open_time = hour_string.split(": ")[1].split(" - ")[0].replace(" ", "")
+        close_time = hour_string.split(" - ")[1].split(",")[0].replace(" ", "")
+
+        # Add minutes, if necessary
+        if ":" not in open_time:
+            open_time = f"{open_time[:-2]}:00am"
+
+        if ":" not in close_time:
+            close_time = f"{close_time[:-2]}:00pm"
+
+        if "7 days a week" in hour_string:
+            for day in DAYS:
+                hours.add_range(day, open_time, close_time, time_format="%H:%M%p")
+
+        return hours.as_opening_hours()


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Crate And Barrel
https://www.crateandbarrel.com/stores/list-state/retail-stores


Discussion:
This issue hasn't been worked on in a while. I'm removing the assignment so others can work on it. If you're still working on it, please reassign yourself and reference a pull request or branch where you're doing your work.
Hello, I've started working on this but I believe the website might be blocking crawlers. 

The location details are on a different page for each state, and I can crawl around 4 pages before getting redirected to a page with a captcha and getting a 503. 

I've tried changing the `User-Agent` in headers and setting a `download_delay = 10`. Are there other general strategies that could be good to try?
It looks like setting user-agent works for me. Your IP might be blocked for some amount of time now. I would just move on to another ticket.
I am interested in this

Resolving patch:
diff --git a/locations/spiders/crateandbarrel.py b/locations/spiders/crateandbarrel.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/crateandbarrel.py
@@ -0,0 +1,27 @@
+import json
+
+from scrapy.spiders import CrawlSpider, Rule
+from scrapy.linkextractors import LinkExtractor
+
+from locations.linked_data_parser import LinkedDataParser
+from locations.structured_data_spider import StructuredDataSpider
+
+
+class CrateAndBarrelSpider(CrawlSpider, StructuredDataSpider):
+    name = "crateandbarrel"
+    allowed_domains = ["www.crateandbarrel.com"]
+    item_attributes = {"brand": "crateandbarrel", "brand_wikidata": "Q5182604"}
+    start_urls = ["https://www.crateandbarrel.com/stores/list-state/retail-stores"]
+    rules = [
+        Rule(
+            LinkExtractor(allow=r"stores\/list-state\/retail-stores\/([a-zA-Z]{2})$"),
+        ),
+        Rule(
+            LinkExtractor(allow=r"stores\/.+\/.+$"),
+            callback="parse_sd",
+        ),
+    ]
+
+    def post_process_item(self, item, response, ld_data, **kwargs):
+        item["ref"] = item["website"] = response.url
+        yield item
diff --git a/locations/spiders/davidstea.py b/locations/spiders/davidstea.py
--- a/locations/spiders/davidstea.py
+++ b/locations/spiders/davidstea.py
@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 from scrapy.spiders import SitemapSpider
 
-from locations.hours import OpeningHours
 from locations.structured_data_spider import StructuredDataSpider
 
 


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Sprouts Farmers Markets
https://www.sprouts.com/stores/


Discussion:

Resolving patch:
diff --git a/locations/spiders/sprouts.py b/locations/spiders/sprouts.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/sprouts.py
@@ -0,0 +1,37 @@
+import re
+
+from scrapy.spiders import SitemapSpider
+
+from locations.hours import OpeningHours, sanitise_day, day_range
+from locations.structured_data_spider import StructuredDataSpider
+
+
+class SproutsFarmersMarket(SitemapSpider, StructuredDataSpider):
+    name = "sprouts_farmer_market"
+    allowed_domains = ["www.sprouts.com"]
+    item_attributes = {"brand": "Sprouts Farmer Market", "brand_wikidata": "Q7581369"}
+    sitemap_urls = ["https://www.sprouts.com/store-sitemap.xml"]
+    sitemap_rules = [(r"^https:\/\/www.sprouts.com\/store\/\w\w\/*\/*\/", "parse_sd")]
+
+    def post_process_item(self, item, response, ld_data, **kwargs):
+        item["website"] = response.url
+        item["opening_hours"] = ld_data["openingHours"].replace(",", ";")
+        yield item
+
+    def pre_process_data(self, ld_data, **kwargs):
+        oh = OpeningHours()
+        for days, open_time, open_time_ampm, close_time, close_time_ampm in re.findall(
+            r"(\w+-\w+) (\d+:\d+)\s?(AM|am|PM|pm) (\d+:\d+)\s?(AM|am|PM|pm)",
+            ld_data["openingHours"],
+        ):
+            start_day, end_day = days.split("-")
+            start_day = sanitise_day(start_day)
+            end_day = sanitise_day(end_day)
+            for day in day_range(start_day, end_day):
+                oh.add_range(
+                    day,
+                    open_time + open_time_ampm,
+                    close_time + close_time_ampm,
+                    time_format="%I:%M%p",
+                )
+        ld_data["openingHours"] = oh.as_opening_hours()


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Teds Montana Grill
https://www.tedsmontanagrill.com/locations.html


Discussion:
Hi, can I pick up this issue? 
Go for it

Resolving patch:
diff --git a/locations/spiders/teds_montana_grill.py b/locations/spiders/teds_montana_grill.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/teds_montana_grill.py
@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+import scrapy
+from locations.items import GeojsonPointItem
+
+
+class TedsMontanaGrillSpider(scrapy.Spider):
+    name = "teds_montana_grill"
+    allowed_domains = ["www.tedsmontanagrill.com"]
+    item_attributes = {"brand": "Ted's Montana Grill", "brand_wikidata": "Q16953170"}
+    start_urls = ("https://www.tedsmontanagrill.com/locations.html",)
+
+    def parse(self, response):
+        location_selectors = response.xpath('.//div[@class="location-list-item"]')
+        for location_selector in location_selectors[:]:
+            if location_selector.xpath(".//h3/text()").extract_first():
+                state = location_selector.xpath(".//h3/text()").extract_first()
+            yield from self.parse_location(location_selector, state)
+
+    def parse_location(self, location, state):
+        ref = location.xpath(".//a/@href").extract_first()
+        city = location.xpath(".//h5/text()").extract_first()
+        phone = (
+            location.xpath(".//p")[-1]
+            .xpath(".//span/a/@href")
+            .extract_first()
+            .replace("tel:", "")
+        )
+        address = location.xpath(".//p")[-1].xpath("text()").extract()
+        street_address = ", ".join([line.strip() for line in address[:-1]])
+        postcode = address[-1].strip()
+        addr_full = f"{street_address}, {postcode}"
+        properties = {
+            "ref": ref,
+            "state": state,
+            "city": city,
+            "phone": phone,
+            "street_address": street_address,
+            "postcode": postcode,
+            "addr_full": addr_full,
+        }
+        yield GeojsonPointItem(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Bashas'
https://www.bashas.com/locations/


Discussion:

Resolving patch:
diff --git a/locations/spiders/bashas.py b/locations/spiders/bashas.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/bashas.py
@@ -0,0 +1,57 @@
+import html
+import json
+import scrapy
+
+from locations.dict_parser import DictParser
+from locations.hours import OpeningHours
+
+
+class BashasSpider(scrapy.spiders.SitemapSpider):
+    name = "bashas"
+    item_attributes = {
+        "brand": "Bashas'",
+        "brand_wikidata": "Q4866786",
+    }
+
+    download_delay = 0.2
+    sitemap_urls = ["https://www.bashas.com/wpsl_stores-sitemap.xml"]
+    sitemap_rules = [("/stores/", "parse_store")]
+
+    def parse_store(self, response):
+        script = response.xpath(
+            "//script[@id='wpsl-js-js-extra']/text()"
+        ).extract_first()
+        open_brace = script.find('"locations":[{') + 13
+        close_brace = script.rfind("}]}") + 1
+        store_json = json.loads(script[open_brace:close_brace])
+        hours = response.xpath("//table[@class='wpsl-opening-hours']//text()").extract()
+
+        item = DictParser.parse(store_json)
+        item["street_address"] = item.pop("addr_full", None)
+        item["country"] = "US"
+        item["name"] = html.unescape(store_json["store"])
+        item["website"] = response.url
+        item["phone"] = response.xpath(
+            "//div[@class='wpsl-contact-details']//text()"
+        ).extract()[1]
+        item["opening_hours"] = self.parse_hours(hours)
+
+        yield item
+
+    @staticmethod
+    def parse_hours(hours):
+        opening_hours = OpeningHours()
+
+        days = hours[0::2]
+        times = hours[1::2]
+
+        for day, time_range in zip(days, times):
+            open_time, close_time = time_range.split(" - ")
+            opening_hours.add_range(
+                day=day[0:2],
+                open_time=open_time,
+                close_time=close_time,
+                time_format="%H:%M %p",
+            )
+
+        return opening_hours.as_opening_hours()
diff --git a/locations/spiders/gelsons.py b/locations/spiders/gelsons.py
--- a/locations/spiders/gelsons.py
+++ b/locations/spiders/gelsons.py
@@ -32,6 +32,7 @@ def parse_store(self, response):
         hours = content["pageComponents"][0]["headline"]
 
         item = DictParser.parse(store_json)
+        item["street_address"] = item.pop("addr_full", None)
         item["phone"] = store_json["storePhone"]
         item["website"] = response.url
         item["opening_hours"] = self.parse_hours(hours)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Pizza Hut Spider returns some closed outlets
It looks like the GB Pizza Hut spider "pizza_hut_gb" is returning a number of outlets that have closed. These are evident when the website either redirects to https://www.pizzahut.co.uk/restaurants/find or https://www.pizzahut.co.uk/restaurants/error/filenotfound . It seems that Pizza Hut are leaving up the https://www.pizzahut.co.uk/huts/uk-2/... web page after the outlet has closed, presumably for SEO reasons. These pages still contain the old location and web address, which the spider then picks up.

Examples include https://www.pizzahut.co.uk/huts/uk-2/437-ayr/ and https://www.pizzahut.co.uk/huts/uk-2/390-barrow/ .

I think these closed outlets can probably be removed from the dataset returned by looking at the openingHours LD field on the /huts/uk-2/ pages. The closed outlets seem to always have "openingHours":[]. The open branches have some sensible content there.


Discussion:
Yep, that all looks right. I checked 2022-11-12-13-31-47 and there are 23 items without opening hours, and they all redirect like you said. I'll do a PR now.

Resolving patch:
diff --git a/locations/spiders/pizza_hut_gb.py b/locations/spiders/pizza_hut_gb.py
--- a/locations/spiders/pizza_hut_gb.py
+++ b/locations/spiders/pizza_hut_gb.py
@@ -7,17 +7,19 @@
 class PizzaHutGB(SitemapSpider, StructuredDataSpider):
     name = "pizza_hut_gb"
     item_attributes = {"brand": "Pizza Hut", "brand_wikidata": "Q191615"}
+    PIZZA_HUT_DELIVERY = {"brand": "Pizza Hut Delivery", "brand_wikidata": "Q107293079"}
     sitemap_urls = ["https://www.pizzahut.co.uk/sitemap.xml"]
     sitemap_rules = [
         (r"https:\/\/www\.pizzahut\.co\.uk\/huts\/[-\w]+\/([-.\w]+)\/$", "parse_sd")
     ]
-    wanted_types = ["FastFoodRestaurant"]
 
-    def inspect_item(self, item, response):
+    def post_process_item(self, item, response, ld_data, **kwargs):
         item["street_address"] = clean_address(item["street_address"])
 
         if item["website"].startswith("https://www.pizzahut.co.uk/huts/"):
-            item["brand"] = "Pizza Hut Delivery"
-            item["brand_wikidata"] = "Q107293079"
+            item.update(self.PIZZA_HUT_DELIVERY)
+
+        if not item["opening_hours"]:
+            return
 
         yield item


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
nationwide_gb spider returning closed branches
The nationwide_gb spide is currently returning a number of branches that are described as "Permanently Closed" on their web pages. For example:

* https://www.nationwide.co.uk/branches/glasgow/1635-great-western-road
* https://www.nationwide.co.uk/branches/bournemouth/10-southbourne-grove
* https://www.nationwide.co.uk/branches/london/129-balham-high-road

They all have "- permanently closed" appended to their names, which might be robust enough to use to detect them. If not, they have their opening times for each day set to "closed".
nationwide_gb spider missing branches (regex not general enough)
According to e.g. https://www.altfi.com/article/9347_nationwide-pledges-to-keep-its-625-branches-open-until-2024 Nationwide should have 625 UK branches. The current nationwide_gb spider is only returning 549.

One issue that is probably behind most (if not all) of the missing branches is that the regex `r"https:\/\/www\.nationwide\.co\.uk\/branches\/[-()\w]+\/[-\w]+$"` used to detect branch page URLs is not sufficiently general. In addition to word characters and hypens the final (branch) part of the URL can also contain a forward slash (used to denote house-number ranges).

For example: https://www.nationwide.co.uk/branches/northampton/18/19-weston-favell-centre


Discussion:

I don't know about the 625 number, but I count [629](https://www.nationwide.co.uk/branches/) with 22 "permanently closed", so 607.

Resolving patch:
diff --git a/locations/spiders/nationwide_gb.py b/locations/spiders/nationwide_gb.py
--- a/locations/spiders/nationwide_gb.py
+++ b/locations/spiders/nationwide_gb.py
@@ -1,24 +1,20 @@
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
 
+from locations.categories import Categories
 from locations.structured_data_spider import StructuredDataSpider
 
 
 class NationwideGB(CrawlSpider, StructuredDataSpider):
     name = "nationwide_gb"
-    item_attributes = {"brand": "Nationwide", "brand_wikidata": "Q846735"}
+    item_attributes = {
+        "brand": "Nationwide",
+        "brand_wikidata": "Q846735",
+        "extras": Categories.BANK.value,
+    }
     start_urls = ["https://www.nationwide.co.uk/branches/index.html"]
-    rules = [
-        Rule(
-            LinkExtractor(
-                allow=r"https:\/\/www\.nationwide\.co\.uk\/branches\/[-()\w]+\/[-\w]+$"
-            ),
-            callback="parse_sd",
-        ),
-        Rule(
-            LinkExtractor(
-                allow=r"https:\/\/www\.nationwide\.co\.uk\/branches\/[-()\w]+$"
-            )
-        ),
-    ]
-    wanted_types = ["BankOrCreditUnion"]
+    rules = [Rule(LinkExtractor(allow=r"/branches/"), callback="parse_sd", follow=True)]
+
+    def post_process_item(self, item, response, ld_data, **kwargs):
+        if "permanently closed" not in item["name"].lower():
+            yield item


Test patch: