Repository: alltheplaces/alltheplaces

Issue:
The Fresh Grocer
https://www.thefreshgrocer.com/pd/stores/PA

The store finder lists DE but there aren't any locations there.


Discussion:

Resolving patch:
diff --git a/locations/spiders/pricerite.py b/locations/spiders/pricerite.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/pricerite.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+import json
+import re
+
+import scrapy
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+class PriceRiteSpider(scrapy.Spider):
+    name = "pricerite"
+    allowed_domains = ["priceritesupermarkets.com"]
+
+    start_urls = (
+        "https://www.priceritesupermarkets.com/locations/",
+    )
+
+    def parse(self, response):
+        script = response.xpath('//script[contains(text(), "var stores")]').extract_first()
+        stores = json.loads(re.search(r'var stores = (.*?);', script).groups()[0])
+
+        for store in stores:
+            properties = {
+                "ref": store["storeNumber"],
+                "name": store["name"],
+                "lat": store["latitude"],
+                "lon": store["longitude"],
+                "addr_full": store["address1"],
+                "city": store["city"],
+                "state": store["state"],
+                "postcode": store["zipCode"],
+            }
+
+            yield GeojsonPointItem(**properties)
+
diff --git a/locations/spiders/shoprite.py b/locations/spiders/shoprite.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/shoprite.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+import json
+import re
+
+import scrapy
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+class ShopriteSpider(scrapy.Spider):
+    name = "shoprite"
+    allowed_domains = ["shoprite.com"]
+
+    def start_requests(self):
+        url = "https://shoprite.com/StoreLocatorSearch"
+
+        for region in ["CT", "DE", "MD", "PA", "NJ", "NY"]:
+            payload = {
+                "Region": region,
+                "City": "",
+                "SearchTerm": "",
+                "FilterOptions%5B0%5D.IsActive": "false",
+                "FilterOptions%5B0%5D.Name": "Online%20Grocery%20Delivery",
+                "FilterOptions%5B0%5D.Value": "MwgService%3AShop2GroDelivery",
+                "FilterOptions%5B1%5D.IsActive": "false",
+                "FilterOptions%5B1%5D.Name": "Online%20Grocery%20Pickup",
+                "FilterOptions%5B1%5D.Value": "MwgService%3AShop2GroPickup",
+                "FilterOptions%5B2%5D.IsActive": "false",
+                "FilterOptions%5B2%5D.Name": "Platters%2C%20Cakes%20%26%20Catering",
+                "FilterOptions%5B2%5D.Value": "MwgService%3AOrderReady",
+                "FilterOptions%5B3%5D.IsActive": "false",
+                "FilterOptions%5B3%5D.Name": "Pharmacy",
+                "FilterOptions%5B3%5D.Value": "MwgService%3AUmaPharmacy",
+                "FilterOptions%5B4%5D.IsActive": "false",
+                "FilterOptions%5B4%5D.Name": "Retail%20Dietitian",
+                "FilterOptions%5B4%5D.Value": "ShoppingService%3ARetail%20Dietitian",
+                "Radius": "150",
+                "Take": "999"
+            }
+
+            yield scrapy.FormRequest(url=url, formdata=payload)
+
+    def parse(self, response):
+        stores = json.loads(re.search(r'stores: (\[{.*}\])', response.body_as_unicode()).groups()[0])
+        for store in stores:
+            store_id = store["PseudoStoreId"]
+            addr_1 = response.xpath('//li[@id="{}"]//div[@class="store__address"]/div[1]/text()'.format(store_id)).extract_first()
+            city_state_post = response.xpath('//li[@id="{}"]//div[@class="store__address"]/div[2]/text()'.format(store_id)).extract_first()
+            city, state, post = re.search(r'(.*?), (.*?) (\d+)', city_state_post).groups()
+
+            properties = {
+                "ref": store_id,
+                "name": store["StoreName"],
+                "lat": store["Coordinates"]["Latitude"],
+                "lon": store["Coordinates"]["Longitude"],
+                "addr_full": addr_1.strip(),
+                "city": city,
+                "state": state,
+                "postcode": post,
+                "website": "https://shoprite.com/store/" + store_id
+            }
+
+            yield GeojsonPointItem(**properties)
+
diff --git a/locations/spiders/thefreshgrocer.py b/locations/spiders/thefreshgrocer.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/thefreshgrocer.py
@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+import json
+import re
+
+import scrapy
+
+from locations.items import GeojsonPointItem
+from locations.hours import OpeningHours
+
+
+class FreshGrocerSpider(scrapy.Spider):
+    name = "thefreshgrocer"
+    allowed_domains = ["thefreshgrocer.com"]
+
+    def start_requests(self):
+        url = "https://www.thefreshgrocer.com/StoreLocatorSearch"
+
+        for region in ["DE", "PA"]:
+            payload = {
+                "Region": region,
+                "City": "",
+                "SearchTerm": "",
+                "FilterOptions%5B0%5D.IsActive": "false",
+                "FilterOptions%5B0%5D.Name": "Online%20Grocery%20Delivery",
+                "FilterOptions%5B0%5D.Value": "MwgService%3AShop2GroDelivery",
+                "FilterOptions%5B1%5D.IsActive": "false",
+                "FilterOptions%5B1%5D.Name": "Online%20Grocery%20Pickup",
+                "FilterOptions%5B1%5D.Value": "MwgService%3AShop2GroPickup",
+                "FilterOptions%5B2%5D.IsActive": "false",
+                "FilterOptions%5B2%5D.Name": "Platters%2C%20Cakes%20%26%20Catering",
+                "FilterOptions%5B2%5D.Value": "MwgService%3AOrderReady",
+                "FilterOptions%5B3%5D.IsActive": "false",
+                "FilterOptions%5B3%5D.Name": "Pharmacy",
+                "FilterOptions%5B3%5D.Value": "MwgService%3AUmaPharmacy",
+                "FilterOptions%5B4%5D.IsActive": "false",
+                "FilterOptions%5B4%5D.Name": "Retail%20Dietitian",
+                "FilterOptions%5B4%5D.Value": "ShoppingService%3ARetail%20Dietitian",
+                "Radius": "150",
+                "Take": "999"
+            }
+
+            yield scrapy.FormRequest(url=url, formdata=payload)
+
+    def parse(self, response):
+        stores = json.loads(re.search(r'stores: (\[.*\])', response.body_as_unicode()).groups()[0])
+        for store in stores:
+            store_id = store["PseudoStoreId"]
+            addr_1 = response.xpath('//li[@id="{}"]//div[@class="store__address"]/div[1]/text()'.format(store_id)).extract_first()
+            city_state_post = response.xpath('//li[@id="{}"]//div[@class="store__address"]/div[2]/text()'.format(store_id)).extract_first()
+            city, state, post = re.search(r'(.*?), (.*?) (\d+)', city_state_post).groups()
+
+            properties = {
+                "ref": store_id,
+                "name": store["StoreName"],
+                "lat": store["Coordinates"]["Latitude"],
+                "lon": store["Coordinates"]["Longitude"],
+                "addr_full": addr_1.strip(),
+                "city": city,
+                "state": state,
+                "postcode": post,
+                "website": "https://shoprite.com/store/" + store_id
+            }
+
+            yield GeojsonPointItem(**properties)
+


Test patch: