Repository: alltheplaces/alltheplaces

Issue:
Spider tesco is broken
During the global build at 2021-10-06-14-42-44, spider **tesco** failed with **0 features** and **2 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-10-06-14-42-44/logs/tesco.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-10-06-14-42-44/output/tesco.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-10-06-14-42-44/output/tesco.geojson))


Discussion:

Resolving patch:
diff --git a/locations/spiders/tesco.py b/locations/spiders/tesco.py
--- a/locations/spiders/tesco.py
+++ b/locations/spiders/tesco.py
@@ -1,7 +1,8 @@
 import json
-import scrapy
+
 from locations.items import GeojsonPointItem
 from locations.hours import OpeningHours
+from scrapy.spiders import SitemapSpider
 
 DAY_MAPPING = {
     "MONDAY": "Mo",
@@ -13,35 +14,24 @@
     "SUNDAY": "Su",
 }
 
-COOKIES = {
-    "bm_sz": "04B124C1C96D68082A9F61BAAAF0B6D5~YAAQdjsvF22E8Xl6AQAACr1VfAxPEt+enarZyrOZrBaNvyuX71lK5QPuDR/FgDEWBZVMRhjiIf000W7Z1PiAjxobrz2Y5LcYMH3CvUNvpdS3MjVLUMGwMEBCf9L5nD5Gs9ho2YL8T7Tz7lYvpolvaOlJnKrHyhCFxxk/uyBZ2G/0QrGKLwSaCQShDsz7ink=",
-    "_abck": "440E40C406E69413DCCC08ABAA3E9022~-1~YAAQdjsvF26E8Xl6AQAACr1VfAYznoJdJhX7TNIZW1Rfh6qRhzquXg+L1TWoaL7nZUjXlNls2iPIKFQrCdrWqY/CNXW+mHyXibInMflIXJi5VVB/Swq53kABYJDuXYSlCunYvJAzMSr1q12NOYswz134Y8HRNzVWhkb2jMS5whmHxS/v0vniIvS1TQtKjEQlMGzQYmN41CmLX0JobipQhDtUB4VyNwztb2DCAZiqDX8BLwWg7h/DtPd4158qU69hNhayFTgWmD76/MiR8/T536tMmcoRyWLl4fEtP/XUmKOcksuZO7dbfNxXBffTxIXPYwf1eO77LNuZTCQq5kfsGZLJX8ODju2KSjnIF1vdnyHAe98FDIm+hw==~-1~-1~-1",
-}
-
-HEADERS = {
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
-    "accept-encoding": "gzip, deflate, br",
-    "cache-control": "max-age=0",
-    "referer": "https://www.tesco.com/store-locator/?q=london&qp=london&l=en",
-    "sec-ch-ua": '" Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"',
-    "sec-ch-ua-mobile": "?0",
-    "sec-fetch-dest": "document",
-    "sec-fetch-mode": "navigate",
-    "sec-fetch-site": "same-origin",
-    "sec-fetch-user": "?1",
-    "upgrade-insecure-requests": "1",
-    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36",
-}
-
 
-class TescoSpider(scrapy.Spider):
+class TescoSpider(SitemapSpider):
     name = "tesco"
-    item_attributes = {"brand": "Tesco"}
+    item_attributes = {"brand": "Tesco", "brand_wikidata": "Q487494"}
     allowed_domains = ["www.tesco.com"]
     download_delay = 0.3
     custom_settings = {
-        "USER_AGENT": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36",
+        "USER_AGENT": "Mozilla/5.0 (X11; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0",
     }
+    sitemap_urls = [
+        "https://www.tesco.com/store-locator/sitemap.xml",
+    ]
+    sitemap_rules = [
+        (
+            "https:\/\/www\.tesco\.com\/store-locator\/([\w\-\.]+)\/([\d]+\/)?([\w\-\.\(\)]+)$",
+            "parse_store",
+        )
+    ]
 
     def store_hours(self, store_hours):
         opening_hours = OpeningHours()
@@ -59,94 +49,64 @@ def store_hours(self, store_hours):
 
         return opening_hours.as_opening_hours()
 
-    def start_requests(self):
-        url = "https://www.tesco.com/store-locator/sitemap.xml"
-
-        yield scrapy.http.FormRequest(
-            url=url,
-            method="GET",
-            dont_filter=True,
-            cookies=COOKIES,
-            headers=HEADERS,
-            callback=self.parse,
-        )
-
-    def parse(self, response):
-        response.selector.remove_namespaces()
-        urls = response.xpath("//url/loc/text()").extract()
-
-        # Exclude location subpages for at-store services
-        service_paths = [
-            "petrol-filling-station",
-            "cafe",
-            "travel-money",
-            "f-f-clothing",
-            "pharmacy",
-        ]
-
-        for url in urls:
-            if not any(service_path in url for service_path in service_paths):
-
-                yield scrapy.http.FormRequest(
-                    url=url,
-                    method="GET",
-                    dont_filter=True,
-                    cookies=COOKIES,
-                    headers=HEADERS,
-                    callback=self.parse_store,
-                )
-
     def parse_store(self, response):
-        city_page = response.xpath(
-            '//span[@class="Hero-title Hero-title--top"]/text()'
+        store_details = response.xpath(
+            '//script[@type="application/json"][@id="storeData"]/text()'
         ).extract_first()
-
-        if not city_page:  # and is a store page
-            store_details = response.xpath(
-                '//script[@type="application/json" and contains(text(),"storeID")]/text()'
+        if store_details:
+            store_data = json.loads(store_details)
+            ref = store_data["store"]
+            addr_1 = response.xpath(
+                '//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-line1"]/text()'
             ).extract_first()
-            if store_details:
-                store_data = json.loads(store_details)
-                ref = store_data["storeID"]
-                name = store_data["pageName"]
-                addr_1 = response.xpath(
-                    '//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-line1"]/text()'
-                ).extract_first()
-                addr_2 = response.xpath(
-                    '//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-line2"]/text()'
-                ).extract_first()
-                if addr_2:
-                    addr_full = ", ".join([addr_1.strip(), addr_2.strip()])
-                else:
-                    addr_full = addr_1
-
-                properties = {
-                    "ref": ref,
-                    "name": name,
-                    "addr_full": addr_full,
-                    "city": response.xpath(
-                        '//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-city"]/text()'
-                    ).extract_first(),
-                    "postcode": response.xpath(
-                        '//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-postalCode"]/text()'
-                    ).extract_first(),
-                    "country": "GB",
-                    "lat": response.xpath(
-                        '//div[@class="Core-infoWrapper"]//span[@class="Address-coordinates"]/meta[@itemprop="latitude"]/@content'
-                    ).extract_first(),
-                    "lon": response.xpath(
-                        '//div[@class="Core-infoWrapper"]//span[@class="Address-coordinates"]/meta[@itemprop="longitude"]/@content'
-                    ).extract_first(),
-                    "phone": response.xpath(
-                        '//div[@class="Core-infoWrapper"]//span[@itemprop="telephone"]/text()'
-                    ).extract_first(),
-                    "website": response.url,
-                }
-
-                hours = response.xpath(
-                    '//div[@class="Core-infoWrapper"]//@data-days'
-                ).extract_first()
-                if hours:
-                    properties["opening_hours"] = self.store_hours(hours)
-
-                yield GeojsonPointItem(**properties)
+            addr_2 = response.xpath(
+                '//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-line2"]/text()'
+            ).extract_first()
+            if addr_2:
+                addr_full = ", ".join([addr_1.strip(), addr_2.strip()])
+            else:
+                addr_full = addr_1
+
+            properties = {
+                "ref": ref,
+                "name": response.xpath(
+                    '//h1[@itemprop="name"]/descendant-or-self::*/text()'
+                ).get(),
+                "street_address": addr_full,
+                "city": response.xpath(
+                    '//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-city"]/text()'
+                ).extract_first(),
+                "postcode": response.xpath(
+                    '//div[@class="Core-infoWrapper"]//span[@class="Address-field Address-postalCode"]/text()'
+                ).extract_first(),
+                "country": "GB",
+                "lat": response.xpath(
+                    '//div[@class="Core-infoWrapper"]//span[@class="Address-coordinates"]/meta[@itemprop="latitude"]/@content'
+                ).extract_first(),
+                "lon": response.xpath(
+                    '//div[@class="Core-infoWrapper"]//span[@class="Address-coordinates"]/meta[@itemprop="longitude"]/@content'
+                ).extract_first(),
+                "phone": "+44 "
+                + response.xpath(
+                    '//span[@itemprop="telephone"]/text()'
+                ).extract_first()[1:],
+                "website": response.url,
+            }
+
+            hours = response.xpath(
+                '//div[@class="Core-infoWrapper"]//@data-days'
+            ).extract_first()
+            if hours:
+                properties["opening_hours"] = self.store_hours(hours)
+
+            if store_data["storeformat"] == "Express":
+                properties["brand"] = "Tesco Express"
+                properties["brand_wikidata"] = "Q98456772"
+            elif store_data["storeformat"] == "Superstore":
+                properties["brand"] = "Tesco Superstore"
+                properties["brand_wikidata"] = "Q487494"
+            elif store_data["storeformat"] == "Extra":
+                properties["brand"] = "Tesco Extra"
+                properties["brand_wikidata"] = "Q25172225"
+
+            yield GeojsonPointItem(**properties)


Test patch: