Repository: alltheplaces/alltheplaces

Issue:
Pret
I've previously scraped [this website](https://locations.pret.co.uk/), but the data is awful, they have a bunch of fake stores. This [API](https://api1.pret.com/v1/shops) is presumably better.


Discussion:
One of mine also: https://github.com/mjoe999/alltheplaces/commit/5dd43ca9fbbb366733476b8034bbdfdef9900eb1#diff-48c454b9ef6119dfba07475d1079ce7f4f7ad0291a092f5c8fd346410f9c1998
...... ho hum.

Yeah, unfortunately the website is completely shit. What's the difference between [7-8 St Martin's Place ](https://locations.pret.co.uk/london/7-8-st-martin's-place) and [7-8 St Martins Place](https://locations.pret.co.uk/london/7-8-st-martins-place)? There's dozens of ones like that on pret.co.uk alone.

The API only lists the latter.

Resolving patch:
diff --git a/locations/dict_parser.py b/locations/dict_parser.py
new file mode 100644
--- /dev/null
+++ b/locations/dict_parser.py
@@ -0,0 +1,68 @@
+from locations.items import GeojsonPointItem
+
+
+class DictParser(object):
+    @staticmethod
+    def parse(obj) -> GeojsonPointItem:
+        item = GeojsonPointItem()
+
+        location = DictParser.get_first_key(obj, ["location", "geolocation"])
+
+        # Check for a location object, if not use the parent
+        if location and not isinstance(location, dict):
+            location = obj
+
+        item["lat"] = DictParser.get_first_key(location, ["latitude", "lat"])
+        item["lon"] = DictParser.get_first_key(
+            location, ["longitude", "lon", "long", "lng"]
+        )
+
+        item["name"] = DictParser.get_first_key(
+            obj, ["name", "storeName", "displayName"]
+        )
+
+        address = DictParser.get_first_key(obj, ["address", "addr"])
+
+        if address and isinstance(address, str):
+            item["addr_full"] = address
+
+        if not address or not isinstance(address, dict):
+            address = obj
+
+        item["housenumber"] = DictParser.get_first_key(
+            address, ["houseNumber", "houseNo", "streetNumber"]
+        )
+        item["street"] = DictParser.get_first_key(address, ["street", "streetName"])
+        item["street_address"] = DictParser.get_first_key(
+            address, ["streetAddress", "street_address", "line1"]
+        )
+
+        item["city"] = DictParser.get_first_key(address, ["city", "town"])
+        item["state"] = DictParser.get_first_key(address, ["state", "region"])
+        item["postcode"] = DictParser.get_first_key(
+            address, ["postCode", "post_code", "postalCode"]
+        )
+        item["country"] = DictParser.get_first_key(address, ["country", "countryCode"])
+
+        contact = DictParser.get_first_key(obj, ["contact"])
+
+        if not contact or not isinstance(contact, dict):
+            contact = obj
+
+        item["phone"] = DictParser.get_first_key(contact, ["phone", "telephone", "tel"])
+
+        item["ref"] = DictParser.get_first_key(
+            obj, ["ref", "id", "store_id", "shopNumber", "slug"]
+        )
+
+        return item
+
+    @staticmethod
+    def get_first_key(obj, keys):
+        for key in keys:
+            if obj.get(key):
+                return obj[key]
+            elif obj.get(key.lower()):
+                return obj[key.lower()]
+            elif obj.get(key.upper()):
+                return obj[key.upper()]
diff --git a/locations/spiders/poundland.py b/locations/spiders/poundland.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/poundland.py
@@ -0,0 +1,63 @@
+import scrapy
+
+from locations.dict_parser import DictParser
+from locations.hours import OpeningHours
+
+
+class PoundlandSpider(scrapy.Spider):
+    name = "poundland"
+    item_attributes = {"brand": "Poundland", "brand_wikidata": "Q1434528"}
+    start_urls = [
+        "https://www.poundland.co.uk/rest/poundland/V1/locator/?searchCriteria[scope]=store-locator&searchCriteria[current_page]=1&searchCriteria[page_size]=10000"
+    ]
+    custom_settings = {"DEFAULT_REQUEST_HEADERS": {"Accept": "application/json"}}
+
+    def parse(self, response):
+        # We may have to handle pagination at some point
+        for store in response.json()["locations"]:
+            item = DictParser.parse(store)
+
+            item["street_address"] = ", ".join(
+                filter(None, store["address"].get("line"))
+            )
+
+            # "store_id" seems to be a better ref than "id"
+            item["ref"] = store.get("store_id")
+            item["website"] = (
+                "https://www.poundland.co.uk/store-finder/store_page/view/id/"
+                + item["ref"]
+                + "/"
+            )
+
+            oh = OpeningHours()
+            for rule in store["opening_hours"]:
+                if rule["hours"] == "Closed":
+                    continue
+                open_time, close_time = rule["hours"].split(" - ")
+                oh.add_range(rule["day"][:2], open_time, close_time)
+
+            item["opening_hours"] = oh.as_opening_hours()
+
+            item["extras"] = {}
+            item["extras"]["atm"] = "yes" if store.get("atm") == "1" else "no"
+            item["extras"]["icestore"] = "yes" if store.get("icestore") == "1" else "no"
+
+            if store["is_pep_co_only"] == "1":
+                item["brand"] = "Pep&Co"
+                item["brand_wikidata"] = "Q24908166"
+            else:
+                if store.get("pepshopinshop") == "1":
+                    # Pep and Poundland at this location
+                    pep = item.copy()
+
+                    pep["ref"] = pep["ref"] + "_pep"
+
+                    pep["brand"] = "Pep&Co"
+                    pep["brand_wikidata"] = "Q24908166"
+
+                    pep["located_in"] = self.item_attributes["brand"]
+                    pep["located_in_wikidata"] = self.item_attributes["brand_wikidata"]
+
+                    yield pep
+
+            yield item
diff --git a/locations/spiders/pret_a_manger.py b/locations/spiders/pret_a_manger.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/pret_a_manger.py
@@ -0,0 +1,55 @@
+import logging
+
+import scrapy
+
+from locations.dict_parser import DictParser
+from locations.hours import OpeningHours
+
+DAYS = ["Su", "Mo", "Tu", "We", "Th", "Fr", "Sa"]
+
+
+class PretAMangerSpider(scrapy.Spider):
+    name = "pret_a_manger"
+    veggie_pret = {"brand": "Veggie Pret", "brand_wikidata": "Q108118332"}
+    item_attributes = {"brand": "Pret A Manger", "brand_wikidata": "Q2109109"}
+    start_urls = ["https://api1.pret.com/v1/shops"]
+
+    def parse(self, response):
+        for store in response.json()["shops"]:
+            if not store["active"]:
+                continue
+
+            item = DictParser.parse(store)
+
+            item["street_address"] = ", ".join(
+                filter(None, [item["housenumber"], item["street"]])
+            )
+            item["housenumber"] = item["street"] = None
+
+            oh = OpeningHours()
+            for i in range(0, 7):
+                rule = store["tradingHours"][i]
+                if rule:
+                    if not rule == ["00:00", "00:00"] and len(rule) == 2:
+                        oh.add_range(DAYS[i], rule[0], rule[1])
+
+            item["opening_hours"] = oh.as_opening_hours()
+
+            item["extras"] = {}
+
+            item["extras"]["delivery"] = (
+                "yes" if store["features"]["delivery"] else "no"
+            )
+            item["extras"]["storeType"] = store["features"].get("storeType")
+            item["extras"]["wheelchair"] = (
+                "yes" if store["features"]["wheelchairAccess"] else "no"
+            )
+            item["extras"]["internet_access"] = (
+                "wlan" if store["features"]["wifi"] else "no"
+            )
+
+            if store["features"].get("storeType") == "veggie-pret":
+                item["brand"] = self.veggie_pret["brand"]
+                item["brand_wikidata"] = self.veggie_pret["brand_wikidata"]
+
+            yield item


Test patch:
diff --git a/tests/test_dict_parser.py b/tests/test_dict_parser.py
new file mode 100644
--- /dev/null
+++ b/tests/test_dict_parser.py
@@ -0,0 +1,124 @@
+from locations.dict_parser import DictParser
+
+
+def test_dict_parse():
+    i = DictParser.parse(
+        {
+            "id": "2107",
+            "name": "Kidderminster, Swan Centre",
+            "type": "store",
+            "distance": 71.53,
+            "address": {
+                "company": "Poundland",
+                "line": ["3-6 Coventry Street", "Swan Centre"],
+                "city": "Kidderminster",
+                "country": "UK",
+                "postcode": "DY10 2DG",
+            },
+            "geolocation": {"latitude": "52.38839100", "longitude": "-2.24784700"},
+            "store_id": "304",
+            "url_key": "2107",
+            "description": None,
+            "store_manager": "Steven Creighton",
+            "fax": "",
+            "tel": "01562 746695",
+            "email": "",
+            "store_features": '{"atm":"false","pepshopinshop":"true","icestore":"true","parking":"true","clickandcollect":"false","storetype":{"highstreet":"true","shoppingprecinct":"false","shoppingcentre":"false","retailpark":"false"}}',
+            "close_date": None,
+            "atm": "0",
+            "pepshopinshop": "1",
+            "icestore": "1",
+            "parking": "1",
+            "clickandcollect": "0",
+            "highstreet": "1",
+            "retailpark": "0",
+            "shoppingprecinct": "0",
+            "shoppingcentre": "0",
+            "is_pep_co_only": "0",
+            "route": "store-finder/2107",
+            "opening_hours": [
+                {"day": "Monday", "hours": "08:00 - 18:00"},
+                {"day": "Tuesday", "hours": "08:00 - 18:00"},
+                {"day": "Wednesday", "hours": "08:00 - 18:00"},
+                {"day": "Thursday", "hours": "08:00 - 18:00"},
+                {"day": "Friday", "hours": "08:00 - 18:00"},
+                {"day": "Saturday", "hours": "08:00 - 18:00"},
+                {"day": "Sunday", "hours": "10:00 - 16:00"},
+            ],
+            "open_today": "08:00 - 18:00",
+        }
+    )
+
+    assert i["ref"] == "2107"
+    assert i["name"] == "Kidderminster, Swan Centre"
+    assert i["city"] == "Kidderminster"
+    assert i["country"] == "UK"
+    assert i["postcode"] == "DY10 2DG"
+    assert i["lat"] == "52.38839100"
+    assert i["lon"] == "-2.24784700"
+    assert i["phone"] == "01562 746695"
+
+    i = DictParser.parse(
+        {
+            "id": "10288",
+            "channelId": "a136a947-d5c7-427d-b9c8-6e714b95b8dc",
+            "name": "100 Church Street",
+            "shopNumber": "43",
+            "location": {"lat": 40.713166, "lng": -74.009354},
+            "countryCode": "US",
+            "address": {
+                "streetName": "",
+                "streetNumber": "100 Church Street",
+                "city": "New York",
+                "postalCode": "10007",
+                "region": "NY",
+            },
+            "timezoneName": "America/New_York",
+            "active": True,
+            "contact": {"phone": "212 227 3108"},
+            "features": {
+                "availableForOrderAheadCollection": True,
+                "blenderpresent": False,
+                "collection": True,
+                "delivery": True,
+                "icemachinepresent": True,
+                "seating": "lots_of_seats",
+                "storeType": "pret",
+                "toastiemakerpresent": True,
+                "wheelchairAccess": True,
+                "wifi": True,
+            },
+            "tradingHours": [
+                ["00:00", "00:00"],
+                ["07:00", "19:30"],
+                ["07:00", "19:30"],
+                ["07:00", "19:30"],
+                ["07:00", "19:30"],
+                ["07:00", "19:30"],
+                ["00:00", "00:00"],
+            ],
+            "orderAheadTradingHours": [
+                ["", ""],
+                ["07:00", "18:00"],
+                ["07:00", "19:00"],
+                ["07:00", "19:00"],
+                ["07:00", "19:00"],
+                ["07:00", "18:00"],
+                ["", ""],
+            ],
+            "isCurrentlyAvailableForOA": True,
+            "isCurrentlyOpen": True,
+            "priceBand": "us_nyc",
+        }
+    )
+
+    assert i["ref"] == "10288"
+    assert i["name"] == "100 Church Street"
+    assert i["housenumber"] == "100 Church Street"
+    assert i["street"] is None
+    assert i["city"] == "New York"
+    assert i["country"] is None
+    assert i["postcode"] == "10007"
+    assert i["lat"] == 40.713166
+    assert i["lon"] == -74.009354
+    assert i["phone"] == "212 227 3108"