Repository: alltheplaces/alltheplaces

Issue:
Spider murphy_usa is broken
During the global build at 2021-10-13-14-42-23, spider **murphy_usa** failed with **0 features** and **102 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-10-13-14-42-23/logs/murphy_usa.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-10-13-14-42-23/output/murphy_usa.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-10-13-14-42-23/output/murphy_usa.geojson))


Discussion:

Resolving patch:
diff --git a/locations/spiders/murphy_usa.py b/locations/spiders/murphy_usa.py
deleted file mode 100644
--- a/locations/spiders/murphy_usa.py
+++ /dev/null
@@ -1,149 +0,0 @@
-import json
-
-import scrapy
-
-from locations.hours import OpeningHours
-from locations.items import Feature
-
-STATES = [
-    "AL",
-    "AK",
-    "AZ",
-    "AR",
-    "CA",
-    "CO",
-    "CT",
-    "DC",
-    "DE",
-    "FL",
-    "GA",
-    "HI",
-    "ID",
-    "IL",
-    "IN",
-    "IA",
-    "KS",
-    "KY",
-    "LA",
-    "ME",
-    "MD",
-    "MA",
-    "MI",
-    "MN",
-    "MS",
-    "MO",
-    "MT",
-    "NE",
-    "NV",
-    "NH",
-    "NJ",
-    "NM",
-    "NY",
-    "NC",
-    "ND",
-    "OH",
-    "OK",
-    "OR",
-    "PA",
-    "RI",
-    "SC",
-    "SD",
-    "TN",
-    "TX",
-    "UT",
-    "VT",
-    "VA",
-    "WA",
-    "WV",
-    "WI",
-    "WY",
-]
-
-
-class MurphyUSASpider(scrapy.Spider):
-    name = "murphy_usa"
-    item_attributes = {"brand": "Murphy USA", "brand_wikidata": "Q19604459"}
-    allowed_domains = ["murphyusa.com"]
-
-    def start_requests(self):
-        for state in STATES:
-            yield scrapy.http.JsonRequest(
-                "http://locator.murphyusa.com/MapServices.asmx/GetLocations",
-                method="POST",
-                data={"stateAbbr": state, "filter": None},
-                callback=self.fetch_diesel_stores,
-            )
-
-    def fetch_diesel_stores(self, response):
-        stores = response.json().get("d")
-
-        if len(stores) > 0:
-            query = json.loads(response.request.body)
-            query.update({"filter": "diesel"})
-
-            yield scrapy.http.JsonRequest(
-                "http://locator.murphyusa.com/MapServices.asmx/GetLocations",
-                method="POST",
-                meta={"stores": stores},
-                data=query,
-            )
-
-    def parse(self, response):
-        stores = response.meta["stores"]
-        diesel_stores = response.json().get("d")
-
-        for store in stores:
-            properties = {
-                "ref": store["StoreNum"],
-                "lon": store["Long"],
-                "lat": store["Lat"],
-                "name": f"Murphy Express #{store['StoreNum']}"
-                if store["IsExpress"]
-                else f"Murphy USA #{store['StoreNum']}",
-                "addr_full": store["Address"],
-                "city": store["City"],
-                "state": store["State"],
-                "postcode": store["Zip"],
-                "country": "US",
-                "extras": {
-                    "amenity:fuel": True,
-                    "fuel:diesel": any(d["StoreNum"] == store["StoreNum"] for d in diesel_stores),
-                },
-            }
-
-            yield scrapy.http.JsonRequest(
-                "http://locator.murphyusa.com/MapServices.asmx/StoreDetailHtml",
-                method="POST",
-                meta={"item_properties": properties},
-                data={"storeNum": int(store["StoreNum"]), "distance": 0},
-                callback=self.add_details,
-            )
-
-    def add_details(self, response):
-        properties = response.meta["item_properties"]
-        opening_hours = OpeningHours()
-
-        details_html = response.json().get("d")
-        details = scrapy.Selector(text=details_html)
-
-        days = details.css(".hours th::text").getall()
-        hours = details.css(".hours td::text").getall()
-
-        for day, hours in zip(days, hours):
-            (open_time, close_time) = hours.split()
-
-            opening_hours.add_range(
-                day=day[0:2],
-                open_time=f"{open_time}M",
-                close_time=f"{close_time}M",
-                time_format="%I:%M%p",
-            )
-
-        properties.update(
-            {
-                "phone": details.css(".phone::text").get().replace("Phone: ", ""),
-                "opening_hours": opening_hours.as_opening_hours(),
-            }
-        )
-
-        yield Feature(**properties)
diff --git a/locations/spiders/murphy_usa_us.py b/locations/spiders/murphy_usa_us.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/murphy_usa_us.py
@@ -0,0 +1,46 @@
+from scrapy import Spider
+from scrapy.http import JsonRequest
+
+from locations.categories import Extras, Fuel, apply_yes_no
+from locations.dict_parser import DictParser
+from locations.hours import DAYS_FULL, OpeningHours
+
+
+class MurphyUSAUSSpider(Spider):
+    name = "murphy_usa_us"
+    item_attributes = {"brand": "Murphy USA", "brand_wikidata": "Q19604459"}
+    allowed_domains = ["service.murphydriverewards.com"]
+    start_urls = ["https://service.murphydriverewards.com/api/store/list"]
+    custom_settings = {"ROBOTSTXT_OBEY": False}
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield JsonRequest(url=url, callback=self.parse_location_list)
+
+    def parse_location_list(self, response):
+        for location in response.json()["data"]:
+            store_id = location["id"]
+            yield JsonRequest(url=f"https://service.murphydriverewards.com/api/store/detail/{store_id}")
+
+    def parse(self, response):
+        location = response.json()["data"]
+        item = DictParser.parse(location)
+        item["ref"] = location["storeNumber"]
+        item["street_address"] = item.pop("addr_full")
+        item["opening_hours"] = OpeningHours()
+        for day in DAYS_FULL:
+            day_name = day.lower()
+            if location.get(f"{day_name}Open") and location.get(f"{day_name}Close"):
+                item["opening_hours"].add_range(
+                    day, location.get(f"{day_name}Open"), location.get(f"{day_name}Close"), "%I:%M%p"
+                )
+        apply_yes_no(Extras.TOILETS, item, location.get("hasPublicRestroom"), False)
+        apply_yes_no(Fuel.DIESEL, item, location.get("sellDiesel"), False)
+        apply_yes_no(Extras.ATM, item, location.get("hasAtm"), False)
+        apply_yes_no(Extras.CAR_WASH, item, location.get("hasCarWash"), False)
+        apply_yes_no(Fuel.PROPANE, item, location.get("sellsPropane"), False)
+        fuel_types = [gas_price["fuelType"] for gas_price in location["gasPrices"]]
+        apply_yes_no(Fuel.OCTANE_87, item, "Regular" in fuel_types, False)
+        apply_yes_no(Fuel.OCTANE_89, item, "Midgrade" in fuel_types, False)
+        apply_yes_no(Fuel.OCTANE_91, item, "Premium" in fuel_types, False)
+        yield item


Test patch: