Repository: alltheplaces/alltheplaces

Issue:
family_dollar: use new Where2GetIt spider
https://github.com/alltheplaces/alltheplaces/pull/4741 introduces a new Where2GetIt spider that Family Dollar appears to use alongside ld+json data on individual pages for each store.

The current FamilyDollarSpider (family_dollar.py) uses a CrawlSpider and extracts ld+json data from each location page. This seemingly could be replaced with a single request via the Where2GetIt spider using the application key D2F68B64-7E11-11E7-B734-190193322438 (example use: https://www.familydollar.com/locations/sc/hollywood/).

@Cj-Malone 


Discussion:

Resolving patch:
diff --git a/locations/spiders/family_dollar.py b/locations/spiders/family_dollar.py
deleted file mode 100644
--- a/locations/spiders/family_dollar.py
+++ /dev/null
@@ -1,28 +0,0 @@
-from scrapy.linkextractors import LinkExtractor
-from scrapy.settings.default_settings import RETRY_HTTP_CODES
-from scrapy.spiders import CrawlSpider, Rule
-
-from locations.structured_data_spider import StructuredDataSpider
-
-
-class FamilyDollarSpider(CrawlSpider, StructuredDataSpider):
-    name = "family_dollar"
-    item_attributes = {"brand": "Family Dollar", "brand_wikidata": "Q5433101"}
-    allowed_domains = ["familydollar.com"]
-    start_urls = ("https://locations.familydollar.com",)
-    time_format = "%I:%M %p"
-
-    # site apparently uses 301 to mean temporary server error, which causes
-    # scrapy to avoid repeating the request by default
-    custom_settings = {
-        "RETRY_HTTP_CODES": RETRY_HTTP_CODES + [301],
-        "REDIRECT_ENABLED": False,
-    }
-
-    rules = [
-        Rule(
-            LinkExtractor(allow=[r"/\d+/$"], restrict_css=".itemlist"),
-            callback="parse_sd",
-        ),
-        Rule(LinkExtractor(restrict_css=".itemlist")),
-    ]
diff --git a/locations/spiders/family_dollar_us.py b/locations/spiders/family_dollar_us.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/family_dollar_us.py
@@ -0,0 +1,50 @@
+from locations.hours import OpeningHours
+from locations.storefinders.where2getit import Where2GetItSpider
+
+
+class FamilyDollarUSSpider(Where2GetItSpider):
+    name = "family_dollar_us"
+    item_attributes = {"brand": "Family Dollar", "brand_wikidata": "Q5433101"}
+    w2gi_id = "D2F68B64-7E11-11E7-B734-190193322438"
+    w2gi_filter = {
+        "and": {
+            "distributioncenter": {"distinctfrom": "1"},
+            "bopis": {"eq": ""},
+            "tobacco": {"eq": ""},
+            "adult_beverages": {"eq": ""},
+            "propane": {"eq": ""},
+            "red_box": {"eq": ""},
+            "ebt": {"eq": ""},
+            "atm": {"eq": ""},
+            "ice": {"eq": ""},
+            "water_machine": {"eq": ""},
+            "refrigerated_frozen": {"eq": ""},
+            "helium": {"eq": ""},
+            "fresh_produce": {"eq": ""},
+            "billpay": {"eq": ""},
+        }
+    }
+    w2gi_country_code = "US"
+    w2gi_query = ""
+
+    def parse_item(self, item, location):
+        if location["coming_soon"] == "Y" or location["temp_closed"] == "Y" or location["temporarily_closed"] == "Y":
+            return
+        item["street_address"] = ", ".join(filter(None, [location["address1"], location["address2"]]))
+        item["website"] = (
+            "https://www.familydollar.com/locations/"
+            + item["state"].lower()
+            + "/"
+            + item["city"].lower().replace(" ", "-")
+            + "/"
+            + item["ref"]
+            + "/"
+        )
+        hours_string = ""
+        for day in ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]:
+            open_time = location.get(day.lower() + "open")
+            close_time = location.get(day.lower() + "close")
+            hours_string = hours_string + f" {day}: {open_time} - {close_time}"
+        item["opening_hours"] = OpeningHours()
+        item["opening_hours"].add_ranges_from_string(hours_string)
+        yield item


Test patch: