Repository: alltheplaces/alltheplaces

Issue:
Spider publix is broken
During the global build at 2021-05-26-14-42-23, spider **publix** failed with **0 features** and **0 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-05-26-14-42-23/logs/publix.log) and [the output](https://data.alltheplaces.xyz/runs/2021-05-26-14-42-23/output/publix.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-05-26-14-42-23/output/publix.geojson))


Discussion:
During the global build at 2021-08-18-14-42-26, spider **publix** failed with **0 features** and **0 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-08-18-14-42-26/logs/publix.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-08-18-14-42-26/output/publix.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-08-18-14-42-26/output/publix.geojson))

Resolving patch:
diff --git a/locations/spiders/publix.py b/locations/spiders/publix.py
--- a/locations/spiders/publix.py
+++ b/locations/spiders/publix.py
@@ -1,66 +1,50 @@
 # -*- coding: utf-8 -*-
-import scrapy
+import csv
+import json
 import re
 
-from locations.items import GeojsonPointItem
-
-
-class PublixSpider(scrapy.Spider):
-    name = "publix"
-    item_attributes = { 'brand': "Publix", 'brand_wikidata': "Q672170" }
-    allowed_domains = ['publix.com']
-    start_urls = (
-        'http://weeklyad.publix.com/Publix/Entry/Locations/',
-    )
-
-    def parse(self, response):
-        response.selector.remove_namespaces()
-        city_urls = response.xpath('//a[@class="stdLink action-tracking-nav"]/@href').extract()
-#        regex = re.compile(r'Publix\/BrowseByListing\/ByAllCategories\/\?StoreID=\d+')
-        for path in city_urls:
-#            if re.search(regex,path):
-            yield scrapy.Request(
-                str("http://weeklyad.publix.com") + path.strip(),
-                callback=self.parse_ad,
-            )
-
-
-    def parse_ad(self, response):
-        response.selector.remove_namespaces()
-        if response.xpath('//h3[@class="weeklyAdsLoc_text"]/span[@class="mainLocName"]').extract_first():
-            storeNumber = response.xpath('//h3[@class="weeklyAdsLoc_text"]/span[@class="mainLocName"]').extract_first().split('#')[1].split(')')[0]
-            yield scrapy.Request(
-                str("http://www.publix.com/locations/") + storeNumber.strip(),
-                callback=self.parse_store,
-            )
-
+import scrapy
 
-    def parse_store(self, response):
 
-        if "CLOSED" == response.xpath('//span[@class="store-status"]/text()').extract_first():
-            storeHours = 'STORE CLOSED'
+from locations.items import GeojsonPointItem
 
-        else:
-            storeHoursHTML = response.xpath('//div[@class="store-info-group"]').extract()[4]
-            p = re.compile(r'<.*?>')
-            storeHours = p.sub('', storeHoursHTML)
-            storeHours = storeHours.replace('\t', '').replace('\r', '').replace('\n', '').replace('       ', ' ')
-            storeHours = "".join(storeHours.strip())
 
+states = ["AL", "FL", "GA", "NC", "SC", "TN", "VA"]
 
-        properties = {
-        'name': response.xpath('//h1[@id="content_2_TitleTag"]/text()').extract_first().strip(),
-        'ref': response.xpath('//div[@class="store-info-group"]/text()').extract_first().strip(),
-        'addr_full': response.xpath('///div[@class="store-info-group"][2]/text()').extract_first().strip(),
-        'city': response.xpath('///div[@class="store-info-group"][2]/text()').extract()[1].split(',')[0].strip(),
-        'state': response.xpath('//div[@class="store-info-group"]/text()').extract()[2].split('\xa0')[0].split('\t')[-1].strip(),
-        'postcode': response.xpath('///div[@class="store-info-group"][2]/text()').extract()[1].split(',')[1].split()[1].strip(),
-        'phone': response.xpath('///div[@class="store-info-group"]/div[1]/text()').extract_first(),
-        'website': response.request.url,
-        'opening_hours': storeHours,
-        'lat': response.xpath('///div[@class="store-info-group"][4]/a/@href').extract_first().split('//')[2].split(',')[0],
-        'lon': response.xpath('///div[@class="store-info-group"][4]/a/@href').extract_first().split('//')[2].split(',')[1],
-        }
 
+class PublixSpider(scrapy.Spider):
+    name = "publix"
+    item_attributes = {"brand": "Publix", "brand_wikidata": "Q672170"}
+    allowed_domains = ["publix.com"]
+    download_delay = 0.2
+
+    def start_requests(self):
+        with open(
+            "./locations/searchable_points/us_centroids_10mile_radius_state.csv"
+        ) as points:
+            for row in csv.DictReader(points):
+                if row["state"] not in states:
+                    continue
+                latitude = row["latitude"]
+                longitude = row["longitude"]
+                url = f"https://services.publix.com/api/v1/storelocation?latitude={latitude}&longitude={longitude}"
+                yield scrapy.Request(url, headers={"Accept": "application/json"})
 
-        yield GeojsonPointItem(**properties)
+    def parse(self, response):
+        for row in json.loads(response.body)["Stores"]:
+            ref = row["KEY"]
+            properties = {
+                "lat": row["CLAT"],
+                "lon": row["CLON"],
+                "ref": ref,
+                "name": row["NAME"],
+                "addr_full": row["ADDR"],
+                "city": row["CITY"],
+                "state": row["STATE"],
+                "postcode": row["ZIP"],
+                "opening_hours": row["STRHOURS"],
+                "phone": row["PHONE"],
+                "extras": {"fax": row["FAX"]},
+                "website": f"https://www.publix.com/locations/{ref}",
+            }
+            yield GeojsonPointItem(**properties)


Test patch: