Repository: alltheplaces/alltheplaces

Issue:
Spider mcdonalds_uk is broken
During the global build at 2021-09-15-14-42-44, spider **mcdonalds_uk** failed with **0 features** and **1 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-09-15-14-42-44/logs/mcdonalds_uk.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-09-15-14-42-44/output/mcdonalds_uk.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-09-15-14-42-44/output/mcdonalds_uk.geojson))


Discussion:

Resolving patch:
diff --git a/locations/spiders/mcdonalds_uk.py b/locations/spiders/mcdonalds_uk.py
--- a/locations/spiders/mcdonalds_uk.py
+++ b/locations/spiders/mcdonalds_uk.py
@@ -6,16 +6,16 @@
 from locations.items import GeojsonPointItem
 
 
-class McDonaldsNLSpider(scrapy.Spider):
+class McDonaldsUKSpider(scrapy.Spider):
     name = "mcdonalds_uk"
-    item_attributes = {"brand": "McDonald's"}
+    item_attributes = {"brand": "McDonald's", "brand_wikidata": "Q38076"}
     allowed_domains = ["mcdonalds.com"]
     download_delay = 2
 
     def start_requests(self):
-        url = "https://www.mcdonalds.com/googleapps/GoogleRestaurantLocAction.do?method=searchLocation&latitude={lat}&longitude={lng}&radius=20&maxResults=100&country=gb&language=en-gb"
+        url = "https://www.mcdonalds.com/googleappsv2/geolocation?latitude={lat}&longitude={lng}&radius=120&maxResults=300&country=gb&language=en-gb&showClosed=true"
         with open(
-            "./locations/searchable_points/eu_centroids_20km_radius_country.csv"
+            "./locations/searchable_points/eu_centroids_120km_radius_country.csv"
         ) as points:
             reader = csv.DictReader(points)
             for point in reader:
@@ -28,16 +28,107 @@ def start_requests(self):
     def parse(self, response):
         data = response.json()
         for store in data["features"]:
-            properties = {
-                "ref": store["properties"]["id"],
-                "name": store["properties"]["name"],
-                "addr_full": store["properties"]["addressLine1"],
-                "city": store["properties"]["addressLine3"],
-                "postcode": store["properties"]["postcode"],
-                "country": "UK",
-                "phone": store.get("telephone"),
-                "lat": float(store["geometry"]["coordinates"][1]),
-                "lon": float(store["geometry"]["coordinates"][0]),
-            }
-
-            yield GeojsonPointItem(**properties)
+            if "identifierValue" not in store["properties"].keys():
+                continue
+            else:
+                properties = {
+                    "website": "https://www.mcdonalds.com/gb/en-gb/location/"
+                    + self.urlify(store["properties"]["addressLine3"])
+                    + "/"
+                    + self.urlify(store["properties"]["longDescription"])
+                    + "/"
+                    + self.urlify(store["properties"]["addressLine1"])
+                    + "/"
+                    + store["properties"]["identifierValue"]
+                    + ".html",
+                    "ref": store["properties"]["id"],
+                    "name": store["properties"]["name"],
+                    "street": store["properties"]["addressLine1"],
+                    "city": store["properties"]["addressLine3"],
+                    "postcode": store["properties"]["postcode"],
+                    "country": "GB",
+                    "addr_full": ", ".join(
+                        (
+                            store["properties"]["addressLine1"],
+                            store["properties"]["addressLine3"],
+                            store["properties"]["postcode"],
+                            "United Kingdom",
+                        )
+                    ),
+                    "phone": "+44 " + store["properties"]["telephone"][1:],
+                    "lat": float(store["geometry"]["coordinates"][1]),
+                    "lon": float(store["geometry"]["coordinates"][0]),
+                    "opening_hours": "; ".join(
+                        (
+                            "Mo "
+                            + (
+                                store["properties"]["restauranthours"]["hoursMonday"]
+                                if "hoursMonday"
+                                in store["properties"]["restauranthours"]
+                                else "off"
+                            ),
+                            "Tu "
+                            + (
+                                store["properties"]["restauranthours"]["hoursTuesday"]
+                                if "hoursTuesday"
+                                in store["properties"]["restauranthours"]
+                                else "off"
+                            ),
+                            "We "
+                            + (
+                                store["properties"]["restauranthours"]["hoursWednesday"]
+                                if "hoursWednesday"
+                                in store["properties"]["restauranthours"]
+                                else "off"
+                            ),
+                            "Th "
+                            + (
+                                store["properties"]["restauranthours"]["hoursThursday"]
+                                if "hoursThursday"
+                                in store["properties"]["restauranthours"]
+                                else "off"
+                            ),
+                            "Fr "
+                            + (
+                                store["properties"]["restauranthours"]["hoursFriday"]
+                                if "hoursFriday"
+                                in store["properties"]["restauranthours"]
+                                else "off"
+                            ),
+                            "Sa "
+                            + (
+                                store["properties"]["restauranthours"]["hoursSaturday"]
+                                if "hoursSaturday"
+                                in store["properties"]["restauranthours"]
+                                else "off"
+                            ),
+                            "Su "
+                            + (
+                                store["properties"]["restauranthours"]["hoursSunday"]
+                                if "hoursSunday"
+                                in store["properties"]["restauranthours"]
+                                else "off"
+                            ),
+                        )
+                    )
+                    .replace(" - ", "-")
+                    .replace("-00:00", "-24:00")
+                    .replace("-23:59", "-24:00")
+                    if "restauranthours" in store["properties"].keys()
+                    else "",
+                }
+
+                yield GeojsonPointItem(**properties)
+
+    def urlify(self, str):
+        return (
+            str.lower()
+            .replace(" ", "-")
+            .replace("/", "")
+            .replace(",", "")
+            .replace("&", "")
+            .replace("'", "")
+            .replace("(", "")
+            .replace(")", "")
+            .replace(".", "")
+        )


Test patch: