Repository: alltheplaces/alltheplaces

Issue:
Williams-Sonoma
https://www.williams-sonoma.com/customer-service/store-locator.html?cm_type=fnav

Has a list of Williams Sonoma stores and another list of Williams Sonoma Home stores.


Discussion:
Single API call returning location JSON data for USA/Canada: `https://www.williams-sonoma.com/api/phygital/v1/storesbylocation.json?brands=WS&lat=40.71304703&lng=-74.00723267&radius=100000&includeOutlets=true`

The handful of international stores in Kuwait, Korea, Australia and Mexico (if anyone can be bothered collecting) need a CrawlSpider approach for the store locator page per search error message:
>  We're sorry, our 'Enter Location' is only available for U.S. & Canadian stores at this time. Please visit our 'Filter a Store' module for our international store locations. 

Resolving patch:
diff --git a/locations/spiders/williams_sonoma_us_ca.py b/locations/spiders/williams_sonoma_us_ca.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/williams_sonoma_us_ca.py
@@ -0,0 +1,43 @@
+from json import loads
+
+from scrapy import Spider
+from scrapy.http import JsonRequest
+
+from locations.dict_parser import DictParser
+from locations.hours import DAYS_FULL, OpeningHours
+from locations.settings import DEFAULT_PLAYWRIGHT_SETTINGS
+from locations.user_agents import BROWSER_DEFAULT
+
+
+class WilliamsSonomaUSCASpider(Spider):
+    name = "williams_sonoma_us_ca"
+    item_attributes = {"brand": "Williams Sonoma", "brand_wikidata": "Q96415240"}
+    allowed_domains = ["www.williams-sonoma.com"]
+    start_urls = [
+        "https://www.williams-sonoma.com/api/phygital/v1/storesbylocation.json?brands=WS&lat=40.71304703&lng=-74.00723267&radius=100000&includeOutlets=true"
+    ]
+    user_agent = BROWSER_DEFAULT
+    is_playwright_spider = True
+    custom_settings = DEFAULT_PLAYWRIGHT_SETTINGS
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield JsonRequest(url=url)
+
+    def parse(self, response):
+        # A quirk of scrapy-playwright is that the raw JSON response can't be
+        # obtained as Playwright's Response object is not exposed. Instead, a
+        # HTML wrapper is put around the raw JSON response, and thus we need
+        # to extract from this HTML wrapper.
+        for location in loads(response.xpath("//pre/text()").get()):
+            item = DictParser.parse(location)
+            item["street_address"] = location["address"].get("addrLine1")
+            item["website"] = "https://www.williams-sonoma.com/stores/{}-{}".format(
+                location["address"].get("countryCode").lower(), location["storeIdentifier"]
+            )
+            item["opening_hours"] = OpeningHours()
+            for day_name, day_hours in location.get("storeHoursMap", {}).items():
+                if day_name.title() not in DAYS_FULL:
+                    continue
+                item["opening_hours"].add_range(day_name, *day_hours.split(" - ", 1), "%I:%M %p")
+            yield item
diff --git a/locations/user_agents.py b/locations/user_agents.py
--- a/locations/user_agents.py
+++ b/locations/user_agents.py
@@ -1,14 +1,10 @@
-FIREFOX_ESR_102 = "Mozilla/5.0 (Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0"
-FIREFOX_ESR_LATEST = FIREFOX_ESR_102
+FIREFOX_ESR_115 = "Mozilla/5.0 (Linux x86_64; rv:115.0) Gecko/20100101 Firefox/115.0"
+FIREFOX_ESR_LATEST = FIREFOX_ESR_115
 
-FIREFOX_106 = "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0"
-FIREFOX_107 = "Mozilla/5.0 (X11; Linux x86_64; rv:107.0) Gecko/20100101 Firefox/107.0"
-FIREFOX_LATEST = FIREFOX_107
+FIREFOX_121 = "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/121.0"
+FIREFOX_LATEST = FIREFOX_121
 
-CHROME_106 = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36"
-CHROME_118 = (
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
-)
-CHROME_LATEST = CHROME_118
+CHROME_120 = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+CHROME_LATEST = CHROME_120
 
 BROWSER_DEFAULT = FIREFOX_ESR_LATEST


Test patch: