internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: alltheplaces/alltheplaces

Issue:
Rodda Paint (US) (Crawl, HTML Spider)
### Brand name

Rodda Paint

paint company in the USA

### Wikidata ID

Q7356461
https://www.wikidata.org/wiki/Q7356461
https://www.wikidata.org/wiki/Special:EntityData/Q7356461.json

### Store finder url(s)
https://www.roddapaint.com/location/
Official Url(s): https://roddapaint.com
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [ ] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q7356461 https://roddapaint.com`
```
```


Discussion:
Nothing in sitemap, so HTML parsing seems required. Note "dealers" vs stores

Resolving patch:
diff --git a/locations/spiders/rodda_paint_us.py b/locations/spiders/rodda_paint_us.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/rodda_paint_us.py
@@ -0,0 +1,38 @@
+from chompjs import parse_js_object
+from scrapy import Spider
+
+from locations.categories import Categories
+from locations.dict_parser import DictParser
+from locations.hours import OpeningHours
+from locations.pipelines.address_clean_up import merge_address_lines
+
+
+class RoddaPaintUSSpider(Spider):
+    name = "rodda_paint_us"
+    item_attributes = {"brand": "Rodda Paint", "brand_wikidata": "Q7356461", "extras": Categories.SHOP_PAINT.value}
+    allowed_domains = ["www.roddapaint.com"]
+    start_urls = ["https://www.roddapaint.com/location/"]
+
+    def parse(self, response):
+        js_blob = response.xpath('//script[@id="storelocator-script-js-before"]').get()
+        js_blob = js_blob.split("bhStoreLocatorLocations = ", 1)[1].split("}]}];", 1)[0] + "}]}]"
+        for location in parse_js_object(js_blob):
+            if location.get("bh_sl_loc_cat") == "Auth. Dealers":
+                continue
+            item = DictParser.parse(location)
+            item["name"] = item["name"].replace("Rodda Paint – ", "")
+            item["street_address"] = merge_address_lines([location.get("address"), location.get("address2")])
+            hours_string = " ".join(
+                [
+                    location.get("hours1"),
+                    location.get("hours2"),
+                    location.get("hours3"),
+                    location.get("hours4"),
+                    location.get("hours5"),
+                    location.get("hours6"),
+                    location.get("hours7"),
+                ]
+            )
+            item["opening_hours"] = OpeningHours()
+            item["opening_hours"].add_ranges_from_string(hours_string)
+            yield item


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
AEON BiG (MY)
### Brand name

AEON BiG

### Wikidata ID

Q8077280
https://www.wikidata.org/wiki/Q8077280
https://www.wikidata.org/wiki/Special:EntityData/Q8077280.json

### Store finder url(s)
https://aeonbig.com.my/store-locator
Official Url(s): https://aeonbig.com.my/
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [ ] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q8077280 https://aeonbig.com.my/`
```
```


Discussion:

Resolving patch:
diff --git a/locations/spiders/aeon_big_my.py b/locations/spiders/aeon_big_my.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/aeon_big_my.py
@@ -0,0 +1,35 @@
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import CrawlSpider, Rule
+
+from locations.categories import Categories
+from locations.google_url import extract_google_position
+from locations.hours import OpeningHours
+from locations.items import Feature
+from locations.pipelines.address_clean_up import merge_address_lines
+
+
+class AEONBiGMYSpider(CrawlSpider):
+    name = "aeon_big_my"
+    item_attributes = {"brand": "AEON BiG", "brand_wikidata": "Q8077280", "extras": Categories.SHOP_SUPERMARKET.value}
+    allowed_domains = ["aeonbig.com.my"]
+    start_urls = [
+        "https://aeonbig.com.my/wp-content/themes/twentyseventeen/ajax/portfolio-ajax-load-more-store.php?limit=1000&id=&p=1"
+    ]
+    rules = [Rule(LinkExtractor(allow=r"^https:\/\/aeonbig\.com\.my\/store\/\?[\w\-]+$"), callback="parse")]
+
+    def parse(self, response):
+        properties = {
+            "ref": response.url,
+            "name": response.xpath('//h4[contains(@class, "heading-primarys")]/text()')
+            .get("")
+            .replace("AEON BiG ", ""),
+            "addr_full": merge_address_lines(response.xpath('//div[@class="store_info"]/ul[1]/li[1]/text()').getall()),
+            "phone": response.xpath('//div[@class="store_info"]/ul[1]/li[2]/text()').get("").strip(),
+            "website": response.url,
+            "opening_hours": OpeningHours(),
+        }
+        extract_google_position(properties, response)
+        hours_string = " ".join(filter(None, map(str.strip, response.xpath('//li[@class="bs_date"]//text()').getall())))
+        hours_string = hours_string.replace("Daily", "Mo-Su").replace("Weekday", "Mo-Fr").replace("Weekend", "Sa-Su")
+        properties["opening_hours"].add_ranges_from_string(hours_string)
+        yield Feature(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Super C

### Brand name

Super C

Canadian supermarket chain

### Wikidata ID

Q3504127
https://www.wikidata.org/wiki/Q3504127
https://www.wikidata.org/wiki/Special:EntityData/Q3504127.json

### Store finder url(s)

Official Url(s): https://www.superc.ca/index.fr.html
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [x] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q3504127 https://www.superc.ca/index.fr.html`
```
```
Official Url(s): https://www.superc.ca/index.en.html
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [x] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q3504127 https://www.superc.ca/index.en.html`
```
```
https://www.superc.ca/en/find-a-grocery


Discussion:
Sitemap: https://www.superc.ca/sitemap-general-en.xml
Example store URL: https://www.superc.ca/en/find-a-grocery/742
HTML parsing needed. Lat/lon available in Google Maps URL.

Resolving patch:
diff --git a/locations/spiders/super_c_ca.py b/locations/spiders/super_c_ca.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/super_c_ca.py
@@ -0,0 +1,30 @@
+from chompjs import parse_js_object
+from scrapy.spiders import SitemapSpider
+
+from locations.categories import Categories
+from locations.linked_data_parser import LinkedDataParser
+
+
+class SuperCCASpider(SitemapSpider):
+    name = "super_c_ca"
+    item_attributes = {"brand": "Super C", "brand_wikidata": "Q3504127", "extras": Categories.SHOP_SUPERMARKET.value}
+    allowed_domains = ["www.superc.ca"]
+    sitemap_urls = ["https://www.superc.ca/sitemap-general-en.xml"]
+    sitemap_rules = [(r"^https:\/\/www\.superc\.ca\/en\/find-a-grocery\/\d+$", "parse")]
+    custom_settings = {"ROBOTSTXT_OBEY": False}  # HTTP 500 for robots.txt
+    download_delay = (
+        65  # HTTP 429 rate limiting applies with retry_after=65 (only need ~5 requests in a row to trigger)
+    )
+
+    def parse(self, response):
+        js_blob = (
+            response.xpath('//script[contains(text(), "var storeJsonLd = ")]/text()')
+            .get()
+            .replace("\\n", "")
+            .replace('\\"', '"')
+        )
+        json_ld_data = parse_js_object(js_blob.split('var storeJsonLd = "', 1)[1].split('";', 1)[0])
+        item = LinkedDataParser.parse_ld(json_ld_data)
+        item["ref"] = json_ld_data["id"].split("/find-a-grocery/", 1)[1]
+        item["branch"] = item["name"].removeprefix("Super C - ")
+        yield item


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Super del Norte
### Brand name

Super del Norte

Mexican supermarket chain

### Wikidata ID

Q88388513
https://www.wikidata.org/wiki/Q88388513
https://www.wikidata.org/wiki/Special:EntityData/Q88388513.json

### Store finder url(s)

Official Url(s): http://www.superdelnorte.com.mx/
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [x] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q88388513 http://www.superdelnorte.com.mx/`
```
```
https://www.superdelnorte.com.mx/sucursales



Discussion:
All 38 locations returned in JSON format via: https://superdelnorte.com.mx/api/tiendas

Note lat and lon are found in a Google Maps URL for each location within the JSON response.

Resolving patch:
diff --git a/locations/spiders/super_del_norte_mx.py b/locations/spiders/super_del_norte_mx.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/super_del_norte_mx.py
@@ -0,0 +1,24 @@
+from scrapy import Selector, Spider
+
+from locations.categories import Categories
+from locations.dict_parser import DictParser
+from locations.google_url import url_to_coords
+
+
+class SuperDelNorteMXSpider(Spider):
+    name = "super_del_norte_mx"
+    item_attributes = {
+        "brand": "Super del Norte",
+        "brand_wikidata": "Q88388513",
+        "extras": Categories.SHOP_SUPERMARKET.value,
+    }
+    allowed_domains = ["superdelnorte.com.mx"]
+    start_urls = ["https://superdelnorte.com.mx/api/tiendas"]
+
+    def parse(self, response):
+        for location in response.json()["result"]:
+            item = DictParser.parse(location)
+            item["street_address"] = location["descrp"].strip()
+            google_maps_url = Selector(text=location["map"]).xpath("//iframe/@src").get()
+            item["lat"], item["lon"] = url_to_coords(google_maps_url)
+            yield item


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Topaz
### Brand name

Topaz

Polish supermarket chain

### Wikidata ID

Q11837058
https://www.wikidata.org/wiki/Q11837058
https://www.wikidata.org/wiki/Special:EntityData/Q11837058.json

### Store finder url(s)
https://topaz24.pl/sklepy-topaz
Official Url(s): http://www.topaz24.pl/
- [x] I have run the below and if anything in generated, added it.
- [ ] I have manually checked for a storefinder
- [ ] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q11837058 http://www.topaz24.pl/`
```
```


Discussion:

Resolving patch:
diff --git a/locations/spiders/topaz_pl.py b/locations/spiders/topaz_pl.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/topaz_pl.py
@@ -0,0 +1,47 @@
+from chompjs import parse_js_object
+from scrapy import Request, Spider
+
+from locations.categories import Categories
+from locations.hours import DAYS_PL, OpeningHours
+from locations.items import Feature
+
+
+class TopazPLSpider(Spider):
+    name = "topaz_pl"
+    item_attributes = {"brand": "Topaz", "brand_wikidata": "Q11837058", "extras": Categories.SHOP_SUPERMARKET.value}
+    allowed_domains = ["topaz24.pl"]
+    start_urls = ["https://topaz24.pl/sklepy-topaz"]
+
+    def parse(self, response):
+        js_blob = (
+            "["
+            + response.xpath('//script[contains(text(), "var locations = [")]/text()')
+            .get()
+            .split("var locations = [", 1)[1]
+            .split("];", 1)[0]
+            + "]"
+        )
+        js_blob = js_blob.replace('\\"', "").replace("\\/", "/")
+        for location in parse_js_object(js_blob, unicode_escape=True):
+            properties = {
+                "ref": location[0].split("<a href=/sklep/", 1)[1].split(">", 1)[0],
+                "name": location[1],
+                "lat": location[2],
+                "lon": location[3],
+                "street_address": location[0].split("<br />", 2)[0],
+                "city": location[0].split("<br />", 2)[1],
+                "website": "https://topaz24.pl" + location[0].split("<a href=", 1)[1].split(">", 1)[0],
+            }
+            yield Request(url=properties["website"], meta={"properties": properties}, callback=self.parse_store)
+
+    def parse_store(self, response):
+        properties = response.meta["properties"]
+        properties["phone"] = list(
+            filter(None, map(str.strip, response.xpath('//div[@id="shop"]/div[1]/div[4]/div[1]//text()').getall()))
+        )[-1]
+        hours_string = " ".join(
+            filter(None, map(str.strip, response.xpath('//div[@id="shop"]/div[1]/div[4]/div[2]//text()').getall()))
+        )
+        properties["opening_hours"] = OpeningHours()
+        properties["opening_hours"].add_ranges_from_string(hours_string, days=DAYS_PL)
+        yield Feature(**properties)


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Todis
### Brand name

Todis

### Wikidata ID

Q3992174
https://www.wikidata.org/wiki/Q3992174
https://www.wikidata.org/wiki/Special:EntityData/Q3992174.json

### Store finder url(s)
https://www.todis.it/punti-vendita/scopri-i-negozi/
Official Url(s): http://www.todis.it
- [x] I have run the below and if anything in generated, added it.
- [ ] I have manually checked for a storefinder
- [ ] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q3992174 http://www.todis.it`
```
```


Discussion:

Resolving patch:
diff --git a/locations/dict_parser.py b/locations/dict_parser.py
--- a/locations/dict_parser.py
+++ b/locations/dict_parser.py
@@ -84,6 +84,8 @@ class DictParser:
         "miasto",
         # ES
         "ciudad",  # "city"
+        # IT
+        "comune",  # "comune"
     ]
 
     region_keys = [
@@ -99,6 +101,8 @@ class DictParser:
         "store-state",
         # JP
         "都道府県",  # "prefecture"
+        # IT
+        "regione",  # "region"
     ]
 
     country_keys = [
diff --git a/locations/spiders/todis_it.py b/locations/spiders/todis_it.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/todis_it.py
@@ -0,0 +1,46 @@
+from scrapy import Spider
+from scrapy.http import JsonRequest
+
+from locations.categories import Categories
+from locations.dict_parser import DictParser
+from locations.hours import DAYS_IT, OpeningHours
+
+
+class TodisITSpider(Spider):
+    name = "todis_it"
+    item_attributes = {"brand": "Todis", "brand_wikidata": "Q3992174", "extras": Categories.SHOP_SUPERMARKET.value}
+    allowed_domains = ["www.todis.it"]
+    start_urls = ["https://www.todis.it/wp-json/todis-stores/v1/stores"]
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield JsonRequest(url=url, method="POST", callback=self.parse_store_list)
+
+    def parse_store_list(self, response):
+        for location in response.json():
+            store_id = location["id"]
+            yield JsonRequest(
+                url=f"https://www.todis.it/wp-json/todis-stores/v1/store?id={store_id}", callback=self.parse_store
+            )
+
+    def parse_store(self, response):
+        location = response.json()
+        if location["store"]["active"] != "1":
+            return
+
+        item = DictParser.parse(location["store"])
+        item["name"] = location["store"]["store"]
+        item["street_address"] = item.pop("addr_full", None)
+        item["city"] = location["comune"]["comune"]
+        item["state"] = location["comune"]["regione"]
+        item.pop("website", None)
+
+        item["opening_hours"] = OpeningHours()
+        for day_hours in location["orari"]:
+            if day_hours["chiuso"] != "0":
+                continue
+            item["opening_hours"].add_range(DAYS_IT[day_hours["g"]], day_hours["a"], day_hours["c"])
+            if day_hours["a_2"] and day_hours["c_2"]:
+                item["opening_hours"].add_range(DAYS_IT[day_hours["g"]], day_hours["a_2"], day_hours["c_2"])
+
+        yield item


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Monoprix (FR) (on demand ware)

### Brand name

Monoprix

French retail chain

### Wikidata ID

Q3321241
https://www.wikidata.org/wiki/Q3321241
https://www.wikidata.org/wiki/Special:EntityData/Q3321241.json

### Store finder url(s)
https://www.monoprix.fr/on/demandware.store/Sites-TML-FR-Site/fr_FR/StoreLocatorWidget-Show
Official Url(s): https://www.monoprix.fr/
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [x] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q3321241 https://www.monoprix.fr/`
```
```
Official Url(s): http://www.monoprix.fr/
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [x] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q3321241 http://www.monoprix.fr/`
```
```


Discussion:

Resolving patch:
diff --git a/locations/spiders/monoprix_fr.py b/locations/spiders/monoprix_fr.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/monoprix_fr.py
@@ -0,0 +1,9 @@
+from locations.categories import Categories
+from locations.storefinders.woosmap import WoosmapSpider
+
+
+class MonoprixFRSpider(WoosmapSpider):
+    name = "monoprix_fr"
+    item_attributes = {"brand": "Monoprix", "brand_wikidata": "Q3321241", "extras": Categories.SHOP_SUPERMARKET.value}
+    key = "woos-ef21433b-45e1-3752-851f-6653279c035a"
+    origin = "https://www.monoprix.fr"


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
Metro (PE) (Custom API)
### Brand name

Metro

### Wikidata ID

Q16640217
https://www.wikidata.org/wiki/Q16640217
https://www.wikidata.org/wiki/Special:EntityData/Q16640217.json

### Store finder url(s)
https://www.metro.pe/stores

Official Url(s): http://www.metro.pe
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [x] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q16640217 http://www.metro.pe`
```
```


Discussion:
https://www.metro.pe/files/store.json


Resolving patch:
diff --git a/locations/spiders/metro_pe.py b/locations/spiders/metro_pe.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/metro_pe.py
@@ -0,0 +1,29 @@
+from scrapy import Spider
+
+from locations.categories import Categories
+from locations.dict_parser import DictParser
+from locations.hours import DAYS_ES, DELIMITERS_ES, OpeningHours
+
+
+class MetroPESpider(Spider):
+    name = "metro_pe"
+    item_attributes = {"brand": "Metro", "brand_wikidata": "Q16640217", "extras": Categories.SHOP_SUPERMARKET.value}
+    allowed_domains = ["www.metro.pe"]
+    start_urls = ["https://www.metro.pe/files/store.json"]
+
+    def parse(self, response):
+        locations = []
+        for region in response.json().values():
+            for office in region["offices"]:
+                office["id"] = region["name"] + str(office["id"])
+                locations.append(office)
+
+        for location in locations:
+            if not location["active"]:
+                continue
+            item = DictParser.parse(location)
+            item.pop("phone", None)
+            item["opening_hours"] = OpeningHours()
+            for schedule in location["schedule"]:
+                item["opening_hours"].add_ranges_from_string(schedule, days=DAYS_ES, delimiters=DELIMITERS_ES)
+            yield item


Test patch:
Repository: alltheplaces/alltheplaces

Issue:
puregym_gb spider broken
It looks like the PureGym spider (puregym_gb.py) is broken. For the last few weeks it was consistently returning 303 results, but then on 2024-05-04 the number dropped to 1. Looking at the Spider stats, I think it's found 401 branches, but then dropped all but one of them for having duplicate refs.

If I've understood what's going on correctly, the spider code is obtaining a ref from a "gymId" parameter on each individual branch page. However, I don't think this parameter exists any more on those pages. Hence each branch gets the same null ref value, and then all the duplicates are dropped.

Hopefully removing the custom ref code will fix the spider. I think there should be around 400 branches in total. (There's currently 399 unique https://www.puregym.com/gyms/*/ URLs in the sitemap.) So probably even the 303 results being returned consistently before was as a result of incorrectly dropping ~100 gyms.

I've just checked a sample of 15 of the URLs from the sitemap. I found one "temporarily closed" https://www.puregym.com/gyms/tonbridge/ and one "coming soon" https://www.puregym.com/gyms/washington/ . I don't know if it's worth trying to catch these cases or not.


Discussion:

Resolving patch:
diff --git a/locations/spiders/puregym_gb.py b/locations/spiders/puregym_gb.py
--- a/locations/spiders/puregym_gb.py
+++ b/locations/spiders/puregym_gb.py
@@ -1,28 +1,28 @@
+from urllib.parse import parse_qs, urlparse
+
 from scrapy.spiders import SitemapSpider
 
-from locations.google_url import extract_google_position
 from locations.structured_data_spider import StructuredDataSpider
 
 
 class PureGymGBSpider(SitemapSpider, StructuredDataSpider):
     name = "puregym_gb"
-    item_attributes = {
-        "brand": "PureGym",
-        "brand_wikidata": "Q18345898",
-        "country": "GB",
-    }
+    item_attributes = {"brand": "PureGym", "brand_wikidata": "Q18345898", "country": "GB"}
     allowed_domains = ["www.puregym.com"]
     sitemap_urls = ["https://www.puregym.com/sitemap.xml"]
-    sitemap_rules = [
-        (
-            r"https:\/\/www\.puregym\.com\/gyms\/([\w-]+)\/$",
-            "parse_sd",
-        ),
-    ]
+    sitemap_rules = [(r"/gyms/([^/]+)/$", "parse_sd")]
     wanted_types = ["HealthClub"]
 
-    def inspect_item(self, item, response):
-        item["ref"] = response.xpath('//meta[@itemprop="gymId"]/@content').get()
-        extract_google_position(item, response)
+    def pre_process_data(self, ld_data, **kwargs):
+        ld_data["address"] = ld_data.get("location", {}).get("address")
+
+    def post_process_item(self, item, response, ld_data, **kwargs):
+        item["branch"] = item.pop("name")
+        item["image"] = None
+
+        if img := response.xpath('//img[contains(@src, "tiles.stadiamaps.com")]/@src').get():
+            q = parse_qs(urlparse(img)[4])
+            if "center" in q:
+                item["lat"], item["lon"] = q["center"][0].split(",", 1)
 
         yield item


Test patch: