Repository: alltheplaces/alltheplaces

Issue:
CARPET ONE. (US, CA) (shop/carpet) (sitemap) (100+ locations)
### Brand name

CARPET ONE

carpet floor store

### Wikidata ID

Q121335910
https://www.wikidata.org/wiki/Q121335910
https://www.wikidata.org/wiki/Special:EntityData/Q121335910.json

### Store finder url(s)

https://www.carpetone.com/locations-sitemap.xml

Official Url(s): https://www.carpetone.com/locations
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [ ] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q121335910 https://www.carpetone.com/locations`
```
```
Official Url(s): https://www.carpetone.com
- [x] I have run the below and if anything in generated, added it.
- [x] I have manually checked for a storefinder
- [ ] This is not a manufacturer listing outlets operating under their own brands / this will not create duplicates
`pipenv run scrapy sf --brand-wikidata=Q121335910 https://www.carpetone.com`
```
```



Discussion:

Resolving patch:
diff --git a/locations/dict_parser.py b/locations/dict_parser.py
--- a/locations/dict_parser.py
+++ b/locations/dict_parser.py
@@ -2,9 +2,20 @@
 
 
 class DictParser:
-    ref_keys = ["ref", "id", "store-id", "store-number", "shop-number", "location-id", "slug", "store-code", "item-id"]
+    ref_keys = [
+        "ref",
+        "id",
+        "store-id",
+        "store-number",
+        "shop-number",
+        "location-id",
+        "location-number",
+        "slug",
+        "store-code",
+        "item-id",
+    ]
 
-    name_keys = ["name", "store-name", "display-name", "title", "business-name", "item-name"]
+    name_keys = ["name", "store-name", "display-name", "title", "business-name", "item-name", "location-name"]
 
     house_number_keys = ["house-number", "house-no", "street-number", "street-no", "address-street-no"]
 
diff --git a/locations/spiders/carpet_one_floor_and_home_ca.py b/locations/spiders/carpet_one_floor_and_home_ca.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/carpet_one_floor_and_home_ca.py
@@ -0,0 +1,18 @@
+from scrapy.spiders import SitemapSpider
+
+from locations.spiders.carpet_one_floor_and_home_us import CarpetOneFloorAndHomeUSSpider
+
+
+class CarpetOneFloorAndHomeCASpider(SitemapSpider):
+    name = "carpet_one_floor_and_home_ca"
+    item_attributes = CarpetOneFloorAndHomeUSSpider.item_attributes
+    allowed_domains = ["www.carpetone.ca"]
+    sitemap_urls = ["https://www.carpetone.ca/locations-sitemap.xml"]
+    sitemap_rules = [(r"^https:\/\/www\.carpetone\.ca\/locations\/[^/]+/[^/]+$", "parse")]
+    # Attempt crawling with a high delay to try and avoid receiving
+    # truncated binary responses (non-HTTP). Possible rate limiting
+    # mechanism to frustrate bots?
+    download_delay = 10
+
+    def parse(self, response):
+        yield from CarpetOneFloorAndHomeUSSpider.parse_store(response)
diff --git a/locations/spiders/carpet_one_floor_and_home_us.py b/locations/spiders/carpet_one_floor_and_home_us.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/carpet_one_floor_and_home_us.py
@@ -0,0 +1,46 @@
+from chompjs import parse_js_object
+from scrapy.spiders import SitemapSpider
+
+from locations.categories import Categories
+from locations.dict_parser import DictParser
+
+
+class CarpetOneFloorAndHomeUSSpider(SitemapSpider):
+    name = "carpet_one_floor_and_home_us"
+    item_attributes = {
+        "brand": "Carpet One Floor & Home",
+        "brand_wikidata": "Q121335910",
+        "extras": Categories.SHOP_FLOORING.value,
+    }
+    allowed_domains = ["www.carpetone.com"]
+    sitemap_urls = ["https://www.carpetone.com/locations-sitemap.xml"]
+    sitemap_rules = [(r"^https:\/\/www\.carpetone\.com\/locations\/[^/]+/[^/]+$", "parse")]
+    # Attempt crawling with a high delay to try and avoid receiving
+    # truncated binary responses (non-HTTP). Possible rate limiting
+    # mechanism to frustrate bots?
+    download_delay = 10
+
+    @staticmethod
+    def parse_store(response):
+        js_blob = response.xpath('//script[contains(text(), "var locationlist=[{")]/text()').get()
+        js_blob = "[{" + js_blob.split("var locationlist=[{", 1)[1].split("}];", 1)[0] + "}]"
+        for location in parse_js_object(js_blob):
+            item = DictParser.parse(location)
+            item["street_address"] = ", ".join(
+                filter(None, [location["address"].get("line1"), location["address"].get("line2")])
+            )
+            item["phone"] = location["address"].get("phone")
+            item["website"] = location.get("microSiteUrl")
+
+            for social_media_account in location.get("socialMedia", []):
+                if social_media_account.get("key") == "FacebookURL":
+                    item["facebook"] = social_media_account["value"]
+                elif social_media_account.get("key") == "InstagramURL":
+                    if not isinstance(item.get("extras"), dict):
+                        item["extras"] = {}
+                    item["extras"]["contact:instagram"] = social_media_account["value"]
+
+            yield item
+
+    def parse(self, response):
+        yield from self.parse_store(response)


Test patch: