Repository: alltheplaces/alltheplaces

Issue:
Spider kroger is broken
During the global build at 2021-10-27-14-42-46, spider **kroger** failed with **0 features** and **416 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-10-27-14-42-46/logs/kroger.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-10-27-14-42-46/output/kroger.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-10-27-14-42-46/output/kroger.geojson))


Discussion:
this is working. tested on 12th October 2022
Quick guide for how to fix this spider (which will be a bit of work, and I'm happy to have this assigned):

Extract city details from:

`https://www.kingsoopers.com/seo-store-files/link-hub/store-details-categories/grocery-stores.json`
`https://www.kingsoopers.com/seo-store-files/link-hub/store-details-categories/pharmacy-stores.json`

Convert the city name to lowercase, remove punctuation, convert spaces to dashes. Also convert the state abbreviation to lowercase.

Then get location IDs for each city from:

`https://www.kingsoopers.com/seo-store-files/link-hub/store-details-cities/lakewood-co-grocery.json`
`https://www.kingsoopers.com/seo-store-files/link-hub/store-details-cities/lakewood-co-pharmacy.json`

Then extract location details from:

`https://www.kingsoopers.com/atlas/v1/stores/v2/locator?filter.locationIds=62000060,62000061,62000062&projections=full`

Replace `www.kingsoopers.com` with the full list of domains of Kroger brands listed at [https://developer.kroger.com/reference/](https://developer.kroger.com/reference/)
You can also get the location ids from the sitemap.

```python
import re

from scrapy.spiders import SitemapSpider

from locations.categories import Categories, apply_category
from locations.items import Feature
from locations.spiders.vapestore_gb import clean_address
from locations.user_agents import BROWSER_DEFAULT

BRANDS = {
    "https://www.bakersplus.com/": {"brand": "Baker's", "brand_wikidata": "Q4849080"},
    "https://www.citymarket.com/": {"brand": "City Market", "brand_wikidata": "Q5123299"},
    "https://www.dillons.com/": {"brand": "Dillons", "brand_wikidata": "Q5276954"},
    "https://www.food4less.com/": {"brand": "Food 4 Less", "brand_wikidata": "Q5465282"},
    "https://www.foodsco.net/": {"brand": "Foods Co", "brand_wikidata": "Q5465282"},
    "https://www.fredmeyer.com/": {"brand": "Fred Meyer", "brand_wikidata": "Q5495932"},
    "https://www.frysfood.com/": {"brand": "Fry's Food Stores", "brand_wikidata": "Q5506547"},
    "https://www.gerbes.com/": {"brand": "Gerbes", "brand_wikidata": "Q5276954"},
    "https://www.jaycfoods.com/": {"brand": "JayC", "brand_wikidata": "Q6166302"},
    "https://www.kingsoopers.com/": {"brand": "King Soopers", "brand_wikidata": "Q6412065"},
    "https://www.kroger.com/": {"brand": "Kroger", "brand_wikidata": "Q153417"},
    "https://www.marianos.com/": {"brand": "Mariano's Fresh Market", "brand_wikidata": "Q55622168"},
    "https://www.metromarket.net/": {"brand": "Metro Market", "brand_wikidata": "Q7371288"},
    "https://www.pay-less.com/": {"brand": "Pay Less", "brand_wikidata": "Q7156587"},
    "https://www.picknsave.com/": {"brand": "Pick 'n Save", "brand_wikidata": "Q7371288"},
    "https://www.qfc.com/": {"brand": "QFC", "brand_wikidata": "Q7265425"},
    "https://www.ralphs.com/": {"brand": "Ralphs", "brand_wikidata": "Q3929820"},
    "https://www.smithsfoodanddrug.com/": {"brand": "Smith's", "brand_wikidata": "Q7544856"},
}


class KrogerSpider(SitemapSpider):
    name = "kroger"
    sitemap_urls = [f"{brand}storelocator-sitemap.xml" for brand in BRANDS.keys()]
    custom_settings = {"AUTOTHROTTLE_ENABLED": True, "USER_AGENT": BROWSER_DEFAULT}
    url_re = re.compile(r"/(\d{3})/(\d{5})$")

    def pre_process_data(self, ld_data, **kwargs):
        if phone := ld_data.get("telephone"):
            if isinstance(phone, dict):
                ld_data["telephone"] = phone.get("original")

    def post_process_item(self, item, response, ld_data, **kwargs):
        if m := re.search(r"\"lat\":(-?\d+\.\d+),\"lng\":(-?\d+\.\d+)", response.text):
            item["lat"], item["lon"] = m.groups()

        for url, brand in BRANDS.items():
            if response.url.startswith(url):
                item.update(brand)
                break

        apply_category(Categories.SHOP_SUPERMARKET, item)

        yield item

    def sitemap_filter(self, entries):
        for entry in entries:
            if m := re.search(self.url_re, entry["loc"]):
                entry["loc"] = "https://www.ralphs.com/atlas/v1/stores/v2/locator?filter.locationIds={}{}".format(
                    *m.groups()
                )
                yield entry

    def parse(self, response, **kwargs):
        for location in response.json()["data"]["stores"]:
            item = Feature()
            item["ref"] = location["locationId"]
            item["name"] = location["facilityName"]
            item["lat"] = location["locale"]["location"]["lat"]
            item["lon"] = location["locale"]["location"]["lng"]
            item["street_address"] = clean_address(location["locale"]["address"]["addressLines"])
            item["city"] = location["locale"]["address"]["cityTown"]
            item["postcode"] = location["locale"]["address"]["postalCode"]
            item["state"] = location["locale"]["address"]["stateProvince"]
            item["country"] = location["locale"]["address"]["countryCode"]
            item["phone"] = location["phoneNumber"].get("raw")

            item["extras"]["branch"] = location["vanityName"]
            item["extras"]["operator"] = location["legalName"]
            item["extras"]["storeType"] = location["storeType"]
            item["extras"]["banner"] = location["banner"]

            yield item
```

I can't remember what I was doing, or why I stopped. But that's my local file.

Resolving patch:
diff --git a/locations/spiders/harristeeter.py b/locations/spiders/harristeeter.py
deleted file mode 100644
--- a/locations/spiders/harristeeter.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import json
-import re
-
-import scrapy
-
-from locations.items import Feature
-
-DAYS = ["Mo", "Tu", "We", "Th", "Fr", "Sa", "Su"]
-
-
-class HarristeeterSpider(scrapy.Spider):
-    name = "harristeeter"
-    item_attributes = {"brand": "Harris Teeter"}
-    allowed_domains = ["harristeeter.com"]
-    start_urls = ("https://www.harristeeter.com/store/#/app/store-locator",)
-
-    handle_httpstatus_list = [401]
-    custom_settings = {
-        "DEFAULT_REQUEST_HEADERS": {
-            "Accept": "application/json, text/plain, */*",
-            "Accept-Encoding": "gzip, deflate, br",
-            "Connection": "keep-alive",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",
-        }
-    }
-
-    def store_hours(self, store_hours):
-        res = ""
-        for day in store_hours:
-            match = re.search(
-                r"(\w*)(\s*-\s*(\w*))?\s*(\d{1,2})(:(\d{1,2}))?\s*(am|pm|mp)?\s*-\s*(\d{1,2})(:(\d{1,2}))?\s*(am|pm|mp)",
-                day.replace("Midnight", "12:00pm"),
-            )
-
-            if not match:
-                continue
-            res += match[1][:2]
-
-            try:
-                res += match[2].replace(" ", "")[:3] + " "
-            except Exception:
-                res += " "
-
-            if match[5]:
-                first_minutes = match[5]
-            else:
-                first_minutes = ":00"
-
-            if match[9]:
-                second_minutes = match[9]
-            else:
-                second_minutes = ":00"
-
-            res += str(int(match[4]) + (12 if match[7] in ["pm", "mp"] else 0)) + first_minutes + "-"
-            res += str(int(match[8]) + (12 if match[10] in ["pm", "mp"] else 0)) + second_minutes + ";"
-
-        return res.rstrip(";").strip()
-
-    def parse(self, response):
-        yield scrapy.Request(
-            "https://www.harristeeter.com/api/checkLogin",
-            method="POST",
-            callback=self.check_login,
-        )
-
-    def check_login(self, response):
-        yield scrapy.Request(
-            "https://www.harristeeter.com/store/#/app/store-locator",
-            callback=self.get_store_locator,
-        )
-
-    def get_store_locator(self, response):
-        yield scrapy.Request(
-            "https://www.harristeeter.com/api/v1/stores/search?Address=98011&Radius=20000&AllStores=true",
-            callback=self.parse_shop,
-        )
-
-    def parse_shop(self, response):
-        shops = json.loads(response.text)["Data"]
-
-        for shop in shops:
-            props = {
-                "ref": shop["StoreNumber"],
-                "addr_full": shop["Street"],
-                "city": shop.get("City"),
-                "state": shop.get("State"),
-                "postcode": shop.get("PostalCode"),
-                "country": shop["Country"],
-                "name": shop["StoreName"],
-                "phone": shop["Telephone"],
-                "lat": float(shop["Latitude"]),
-                "lon": float(shop["Longitude"]),
-                "opening_hours": shop["StoreHours"].replace("Open 24 Hours", "Mo-Su 0:00-24:00"),
-            }
-
-            yield Feature(**props)
diff --git a/locations/spiders/kroger.py b/locations/spiders/kroger.py
deleted file mode 100644
--- a/locations/spiders/kroger.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import re
-
-from scrapy.spiders import SitemapSpider
-
-from locations.categories import Categories, apply_category
-from locations.structured_data_spider import StructuredDataSpider
-from locations.user_agents import BROWSER_DEFAULT
-
-BRANDS = {
-    "https://www.bakersplus.com/": {"brand": "Baker's", "brand_wikidata": "Q4849080"},
-    "https://www.citymarket.com/": {"brand": "City Market", "brand_wikidata": "Q5123299"},
-    "https://www.dillons.com/": {"brand": "Dillons", "brand_wikidata": "Q5276954"},
-    "https://www.food4less.com/": {"brand": "Food 4 Less", "brand_wikidata": "Q5465282"},
-    "https://www.foodsco.net/": {"brand": "Foods Co", "brand_wikidata": "Q5465282"},
-    "https://www.fredmeyer.com/": {"brand": "Fred Meyer", "brand_wikidata": "Q5495932"},
-    "https://www.frysfood.com/": {"brand": "Fry's Food Stores", "brand_wikidata": "Q5506547"},
-    "https://www.gerbes.com/": {"brand": "Gerbes", "brand_wikidata": "Q5276954"},
-    "https://www.jaycfoods.com/": {"brand": "JayC", "brand_wikidata": "Q6166302"},
-    "https://www.kingsoopers.com/": {"brand": "King Soopers", "brand_wikidata": "Q6412065"},
-    "https://www.kroger.com/": {"brand": "Kroger", "brand_wikidata": "Q153417"},
-    "https://www.marianos.com/": {"brand": "Mariano's Fresh Market", "brand_wikidata": "Q55622168"},
-    "https://www.metromarket.net/": {"brand": "Metro Market", "brand_wikidata": "Q7371288"},
-    "https://www.pay-less.com/": {"brand": "Pay Less", "brand_wikidata": "Q7156587"},
-    "https://www.picknsave.com/": {"brand": "Pick 'n Save", "brand_wikidata": "Q7371288"},
-    "https://www.qfc.com/": {"brand": "QFC", "brand_wikidata": "Q7265425"},
-    "https://www.ralphs.com/": {"brand": "Ralphs", "brand_wikidata": "Q3929820"},
-    "https://www.smithsfoodanddrug.com/": {"brand": "Smith's", "brand_wikidata": "Q7544856"},
-}
-
-
-class KrogerSpider(SitemapSpider, StructuredDataSpider):
-    name = "kroger"
-    sitemap_urls = [f"{brand}storelocator-sitemap.xml" for brand in BRANDS.keys()]
-    sitemap_rules = [("/stores/grocery/", "parse_sd")]
-    custom_settings = {"AUTOTHROTTLE_ENABLED": True, "USER_AGENT": BROWSER_DEFAULT}
-
-    def pre_process_data(self, ld_data, **kwargs):
-        if phone := ld_data.get("telephone"):
-            if isinstance(phone, dict):
-                ld_data["telephone"] = phone.get("original")
-
-    def post_process_item(self, item, response, ld_data, **kwargs):
-        if m := re.search(r"\"lat\":(-?\d+\.\d+),\"lng\":(-?\d+\.\d+)", response.text):
-            item["lat"], item["lon"] = m.groups()
-
-        for url, brand in BRANDS.items():
-            if response.url.startswith(url):
-                item.update(brand)
-                break
-
-        apply_category(Categories.SHOP_SUPERMARKET, item)
-
-        yield item
diff --git a/locations/spiders/kroger_us.py b/locations/spiders/kroger_us.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/kroger_us.py
@@ -0,0 +1,147 @@
+import re
+from copy import deepcopy
+from typing import Iterable
+from urllib.parse import urljoin, urlparse
+
+from scrapy import Request
+from scrapy.spiders import SitemapSpider
+from scrapy.spiders.sitemap import iterloc
+from scrapy.utils.sitemap import Sitemap
+
+from locations.categories import Categories, apply_category
+from locations.hours import OpeningHours
+from locations.items import Feature
+from locations.spiders.vapestore_gb import clean_address
+from locations.user_agents import BROWSER_DEFAULT
+
+BRANDS = {
+    "https://www.bakersplus.com/": {"brand": "Baker's", "brand_wikidata": "Q4849080"},
+    "https://www.citymarket.com/": {"brand": "City Market", "brand_wikidata": "Q5123299"},
+    "https://www.dillons.com/": {"brand": "Dillons", "brand_wikidata": "Q5276954"},
+    "https://www.food4less.com/": {"brand": "Food 4 Less", "brand_wikidata": "Q5465282"},
+    "https://www.foodsco.net/": {"brand": "Foods Co", "brand_wikidata": "Q5465282"},
+    "https://www.fredmeyer.com/": {"brand": "Fred Meyer", "brand_wikidata": "Q5495932"},
+    "https://www.frysfood.com/": {"brand": "Fry's Food Stores", "brand_wikidata": "Q5506547"},
+    "https://www.gerbes.com/": {"brand": "Gerbes", "brand_wikidata": "Q5276954"},
+    "https://www.harristeeter.com/": {"brand": "Harris Teeter", "brand_wikidata": "Q5665067"},
+    "https://www.jaycfoods.com/": {"brand": "JayC", "brand_wikidata": "Q6166302"},
+    "https://www.kingsoopers.com/": {"brand": "King Soopers", "brand_wikidata": "Q6412065"},
+    "https://www.kroger.com/": {"brand": "Kroger", "brand_wikidata": "Q153417"},
+    "https://www.marianos.com/": {"brand": "Mariano's Fresh Market", "brand_wikidata": "Q55622168"},
+    "https://www.metromarket.net/": {"brand": "Metro Market", "brand_wikidata": "Q7371288"},
+    "https://www.pay-less.com/": {"brand": "Pay Less", "brand_wikidata": "Q7156587"},
+    "https://www.picknsave.com/": {"brand": "Pick 'n Save", "brand_wikidata": "Q7371288"},
+    "https://www.qfc.com/": {"brand": "QFC", "brand_wikidata": "Q7265425"},
+    "https://www.ralphs.com/": {"brand": "Ralphs", "brand_wikidata": "Q3929820"},
+    "https://www.smithsfoodanddrug.com/": {"brand": "Smith's", "brand_wikidata": "Q7544856"},
+}
+
+
+class KrogerUSSpider(SitemapSpider):
+    name = "kroger_us"
+    sitemap_urls = [f"{brand}storelocator-sitemap.xml" for brand in BRANDS.keys()]
+    custom_settings = {"AUTOTHROTTLE_ENABLED": True, "USER_AGENT": BROWSER_DEFAULT}
+    url_re = re.compile(r"/(\d{3})/(\d{5})$")
+
+    departments = {
+        "09": Categories.PHARMACY,
+        "10": Categories.FUEL_STATION,
+    }
+
+    def _parse_sitemap(self, response):
+        brand_domain = urlparse(response.url).hostname
+
+        location_ids = []
+        for url in iterloc(Sitemap(self._get_sitemap_body(response))):
+            if m := re.search(self.url_re, url):
+                location_ids.append(("{}{}".format(*m.groups()), url))
+
+        for chunk in [location_ids[i : i + 25] for i in range(0, len(location_ids), 25)]:
+            url_map = {}
+            for ref, url in chunk:
+                url_map[ref] = url
+
+            yield Request(
+                url="https://{}/atlas/v1/stores/v2/locator?filter.locationIds={}".format(
+                    brand_domain,
+                    ",".join(url_map.keys()),
+                ),
+                meta={"url_map": url_map},
+                errback=self.parse_error,
+            )
+
+    def parse_error(self, failure, **kwargs):
+        for ref in failure.request.meta["url_map"].keys():
+            yield Request(
+                url=urljoin(failure.request.url, f"/atlas/v1/stores/v2/locator?filter.locationIds={ref}"),
+                meta={"url_map": failure.request.meta["url_map"]},
+            )
+
+    def parse(self, response, **kwargs):
+        for location in response.json()["data"]["stores"]:
+            properties = {
+                "ref": location["locationId"],
+                "name": location["facilityName"],
+                "lat": location["locale"]["location"]["lat"],
+                "lon": location["locale"]["location"]["lng"],
+                "street_address": clean_address(location["locale"]["address"]["addressLines"]),
+                "city": location["locale"]["address"]["cityTown"],
+                "postcode": location["locale"]["address"]["postalCode"],
+                "state": location["locale"]["address"]["stateProvince"],
+                "country": location["locale"]["address"]["countryCode"],
+                "phone": location["phoneNumber"].get("raw"),
+                "website": response.meta["url_map"][location["locationId"]],
+                "extras": {"operator": location["legalName"], "branch": location["vanityName"]},
+            }
+
+            for url, brand in BRANDS.items():
+                if response.url.startswith(url):
+                    properties.update(brand)
+                    break
+
+            properties["opening_hours"] = self.parse_hours(location.get("prettyHours", []))
+
+            if location["banner"] == "thelittleclinic":
+                properties["brand"] = "The Little Clinic"
+                properties["brand_wikidata"] = "Q64138262"
+                apply_category(Categories.PHARMACY, properties)
+            else:
+                yield from self.parse_departments(properties, location) or []
+                apply_category(Categories.SHOP_SUPERMARKET, properties)
+
+            yield Feature(**properties)
+
+    def parse_departments(self, supermarket_properties, location) -> Iterable[Feature]:
+        for department in location.get("departments"):
+            cat = self.departments.get(department["code"])
+            if not cat:
+                continue
+
+            properties = deepcopy(supermarket_properties)
+            properties["ref"] += department["code"]
+            properties["phone"] = department.get("phoneNumber", {}).get("raw")
+
+            if department.get("locale"):
+                properties["lat"] = department["locale"]["location"]["lat"]
+                properties["lon"] = department["locale"]["location"]["lng"]
+                properties["street_address"] = clean_address(department["locale"]["address"]["addressLines"])
+                properties["city"] = department["locale"]["address"]["cityTown"]
+                properties["postcode"] = department["locale"]["address"]["postalCode"]
+                properties["state"] = department["locale"]["address"]["stateProvince"]
+                properties["country"] = department["locale"]["address"]["countryCode"]
+
+            properties["opening_hours"] = self.parse_hours(department.get("prettyHours", []))
+
+            apply_category(cat, properties)
+
+            yield Feature(**properties)
+
+    @staticmethod
+    def parse_hours(rules: [dict]) -> OpeningHours:
+        hours_string = ""
+        for day_hours in rules:
+            hours_string = "{} {}: {}".format(hours_string, day_hours["displayName"], day_hours["displayHours"])
+
+        oh = OpeningHours()
+        oh.add_ranges_from_string(hours_string)
+        return oh


Test patch: