Repository: alltheplaces/alltheplaces

Issue:
Spider ikea_uk is broken
During the global build at 2021-08-11-14-42-19, spider **ikea_uk** failed with **0 features** and **0 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-08-11-14-42-19/logs/ikea_uk.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-08-11-14-42-19/output/ikea_uk.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-08-11-14-42-19/output/ikea_uk.geojson))


Discussion:
During the global build at 2021-08-18-14-42-26, spider **ikea_uk** failed with **0 features** and **0 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-08-18-14-42-26/logs/ikea_uk.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-08-18-14-42-26/output/ikea_uk.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-08-18-14-42-26/output/ikea_uk.geojson))

Resolving patch:
diff --git a/locations/spiders/ikea.py b/locations/spiders/ikea.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/ikea.py
@@ -0,0 +1,94 @@
+import json
+import scrapy
+
+from locations.hours import OpeningHours
+from locations.items import GeojsonPointItem
+
+
+class IkeaSpider(scrapy.Spider):
+    name = "ikea"
+    item_attributes = {"brand": "IKEA", "brand_wikidata": "Q54078"}
+    allowed_domains = ["ikea.com"]
+    start_urls = [
+        "https://www.ikea.com/ae/ar/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/bh/ar/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/eg/ar/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/jo/ar/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/kw/ar/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/ma/ar/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/qa/ar/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/sa/ar/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/cz/cs/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/dk/da/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/at/de/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/de/de/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/au/en/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/ca/en/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/gb/en/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/ie/en/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/in/en/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/ph/en/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/sg/en/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/us/en/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/cl/es/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/es/es/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/mx/es/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/fi/fi/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/be/fr/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/ch/fr/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/fr/fr/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/il/he/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/hr/hr/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/hu/hu/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/it/it/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/jp/ja/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/kr/ko/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/my/ms/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/nl/nl/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/no/no/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/pl/pl/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/pt/pt/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/ro/ro/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/ru/ru/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/sk/sk/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/si/sl/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/rs/sr/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/se/sv/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/th/th/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/ua/uk/meta-data/navigation/stores-detailed.json",
+        "https://www.ikea.com/cn/zh/meta-data/navigation/stores-detailed.json",
+    ]
+
+    def parse(self, response):
+        data = response.json()
+        for store in data:
+            if "storePageUrl" not in store:
+                continue
+            opening_hours = OpeningHours()
+            for day in store["hours"]["normal"]:
+                if day["open"] != "":
+                    opening_hours.add_range(
+                        day["day"][0:1].upper() + day["day"][1:2].lower(),
+                        day["open"],
+                        day["close"],
+                    )
+            properties = {
+                "lat": store["lat"],
+                "lon": store["lng"],
+                "name": store["displayName"],
+                "street": store["address"].get("street"),
+                "city": store["address"].get("city"),
+                "postcode": store["address"].get("zipCode"),
+                "country": response.request.url[21:23].upper(),
+                "website": store["storePageUrl"],
+                "ref": store["id"],
+                "opening_hours": opening_hours.as_opening_hours(),
+                "extras": {
+                    "store_type": store["buClassification"]["code"],
+                },
+            }
+
+            if properties["country"] == "US":
+                properties["state"] = store["address"].get("stateProvinceCode")[2:]
+
+            yield GeojsonPointItem(**properties)
diff --git a/locations/spiders/ikea_uk.py b/locations/spiders/ikea_uk.py
deleted file mode 100644
--- a/locations/spiders/ikea_uk.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# -*- coding: utf-8 -*-
-import scrapy
-import re
-
-from locations.items import GeojsonPointItem
-from locations.hours import OpeningHours
-
-DAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
-DAY_MAPPING = {
-    "Monday": "Mo",
-    "Tuesday": "Tu",
-    "Wednesday": "We",
-    "Thursday": "Th",
-    "Friday": "Fr",
-    "Saturday": "Sa",
-    "Sunday": "Su",
-}
-
-
-class IkeaUKSpider(scrapy.Spider):
-    name = "ikea_uk"
-    item_attributes = {"brand": "Ikea"}
-    allowed_domains = ["ikea.com"]
-    start_urls = ("http://www.ikea.com/gb/en/store/",)
-
-    def store_hours(self, store_hours):
-        opening_hours = OpeningHours()
-        store_hours = store_hours.xpath(
-            './/h3[contains(strong, "Store")]/following-sibling::p[1]'
-        )
-        weekdays = store_hours.xpath(".//strong/text()").extract()
-        weekday_hours = store_hours.xpath(
-            ".//strong/following-sibling::text()"
-        ).extract()
-        if "\u200b" in weekdays:
-            weekdays.remove("\u200b")
-        weekday_data = list(zip(weekdays, weekday_hours))
-        for weekday in weekday_data:
-            day_range, day_hours = weekday
-            day_hours = day_hours.replace("–", "-")
-            day_range = day_range.replace("–", "-")
-            day_range = (
-                day_range.strip().strip("\u200b").strip(":").strip(" -").split(" - ")
-            )
-            open_time, close_time = day_hours.split(" - ")
-            open_time = open_time.strip().strip("\u200b")
-            close_time = close_time.strip().strip("\u200b")
-
-            if len(day_range) > 1:
-                start = DAYS.index(day_range[0])
-                end = DAYS.index(day_range[1])
-                for day in DAYS[start:end]:
-                    opening_hours.add_range(
-                        day=DAY_MAPPING[day],
-                        open_time=open_time,
-                        close_time=close_time,
-                        time_format="%H:%M",
-                    )
-            else:
-                opening_hours.add_range(
-                    day=DAY_MAPPING[day_range[0]],
-                    open_time=open_time,
-                    close_time=close_time,
-                    time_format="%H:%M",
-                )
-
-        return opening_hours.as_opening_hours()
-
-    def parse(self, response):
-        shops_urls = response.xpath(
-            '//article//a[contains(@href, "stores")]/@href'
-        ).extract()
-        for shop_url in shops_urls:
-            yield scrapy.Request(url=shop_url, callback=self.parse_shops)
-
-    def parse_shops(self, response):
-        ref = re.search(r".+/(.+?)/?(?:\.html|$)", response.url).group(1)
-        address_full = response.xpath(
-            '//h3[contains(strong, "Address")]/following-sibling::p/text()'
-        ).extract_first()
-        name = response.xpath(
-            '//h1[@class="c1m1sl8e pub__h1 s1gshh7t"]/text()'
-        ).extract_first()
-
-        properties = {
-            "ref": ref,
-            "name": name,
-            "addr_full": address_full,
-            "country": "GB",
-            "website": response.url,
-        }
-        hours = response.xpath(
-            './/h2[contains(strong, "Opening hours")]/following::div[1]'
-        )
-
-        if hours:
-            properties["opening_hours"] = self.store_hours(hours)
-
-        yield GeojsonPointItem(**properties)
diff --git a/locations/spiders/ikea_us.py b/locations/spiders/ikea_us.py
deleted file mode 100644
--- a/locations/spiders/ikea_us.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# -*- coding: utf-8 -*-
-import scrapy
-import json
-
-from locations.items import GeojsonPointItem
-from locations.hours import OpeningHours
-
-
-DAY_MAPPING = {
-    "Monday": "Mo",
-    "Tuesday": "Tu",
-    "Wednesday": "We",
-    "Thursday": "Th",
-    "Friday": "Fr",
-    "Saturday": "Sa",
-    "Sunday": "Su",
-}
-
-
-class IkeaUSSpider(scrapy.Spider):
-    name = "ikea_us"
-    item_attributes = {"brand": "Ikea"}
-    allowed_domains = ["ikea.com"]
-    start_urls = (
-        "https://ww8.ikea.com/ext/iplugins/v2/en_US/data/localstorefinder/data.json",
-    )
-
-    def store_hours(self, store_hours):
-        opening_hours = OpeningHours()
-
-        for weekday in store_hours:
-            opening_hours.add_range(
-                day=DAY_MAPPING[weekday["day"]],
-                open_time=":".join([weekday["openHours"], weekday["openMinutes"]]),
-                close_time=":".join([weekday["closeHours"], weekday["closeMinutes"]]),
-                time_format="%I:%M",
-            )
-
-        return opening_hours.as_opening_hours()
-
-    def parse(self, response):
-        store_data = json.loads(response.text)
-
-        for store in store_data:
-
-            properties = {
-                "name": store["storeCity"],
-                "ref": store["storeNumber"] or store["storeId"],
-                "addr_full": store["storeAddress"],
-                "city": store["storeCity"],
-                "state": store["storeState"],
-                "postcode": store["storeZip"],
-                "country": "US",
-                "website": store.get("storeURL"),
-                "lat": float(store["geoLat"]),
-                "lon": float(store["geoLng"]),
-            }
-
-            hours = store["storeHours"]
-            if hours:
-                properties["opening_hours"] = self.store_hours(hours)
-
-            yield GeojsonPointItem(**properties)


Test patch: