Repository: alltheplaces/alltheplaces

Issue:
Dollar General
The results from the Dollar General spider #344 currently only includes 8,146 locations out of approximately 15,000 locations.


Discussion:

Resolving patch:
diff --git a/locations/spiders/dollar_general.py b/locations/spiders/dollar_general.py
--- a/locations/spiders/dollar_general.py
+++ b/locations/spiders/dollar_general.py
@@ -4,11 +4,6 @@
 import re
 from locations.items import GeojsonPointItem
 
-STATES = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
-          'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD',
-          'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ',
-          'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC',
-          'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY']
 
 WEEKDAYS = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
 
@@ -19,44 +14,79 @@ class DollarGeneralSpider(scrapy.Spider):
 
     def start_requests(self):
         url = 'https://hosted.where2getit.com/dollargeneral/rest/locatorsearch?lang=fr_FR'
-        for state in STATES:
-            formdata = {
-                'request': {
-                    'appkey': '9E9DE426-8151-11E4-AEAC-765055A65BB0',
-                    'formdata': {
-                        'geoip': 'false',
-                        'dataview': 'store_default',
-                        'geolocs': {
-                            'geoloc': [
-                                {
-                                    'addressline': state,
-                                    'country': 'US'
+
+        with open('./locations/searchable_points/us_centroids_50mile_radius.csv') as points:
+            next(points) # Ignore the header
+            for point in points:
+                row = point.split(',')
+                lat = row[1]
+                lon = row[2]
+
+                formdata = {
+                    "request": {
+                        "appkey":"9E9DE426-8151-11E4-AEAC-765055A65BB0",
+                        "formdata": {
+                            "geoip": "false",
+                            "dataview":"store_default",
+                            "geolocs": {
+                                "geoloc": [
+                                    {
+                                        "addressline":"",
+                                        "country":"US",
+                                        "latitude":"{}".format(lat),
+                                        "longitude":"{}".format(lon)
+                                    }
+                                ]
+                            },
+                            "searchradius":"50",
+                            "where": {
+                                "nci": {
+                                    "eq":""
+                                },
+                                "or": {
+                                    "PROPANE": {
+                                        "eq":""
+                                    },
+                                    "REDBOX": {
+                                        "eq":""
+                                    },
+                                    "RUGDR": {
+                                        "eq":""
+                                    },
+                                    "MULTICULTURAL_HAIR": {
+                                        "eq":""
+                                    },
+                                    "icon": {
+                                        "eq":""
+                                    },
+                                    "DG_GO": {
+                                        "eq":""
+                                    }
                                 }
-                            ]
-                        },
-                        'searchradius': '100'
+                            },
+                            "false":"0"
+                        }
                     }
                 }
-            }
 
-            headers = {
-                'Accept': 'application/json, text/javascript, */*; q=0.01',
-                'Accept-Encoding': 'gzip, deflate, br',
-                'Accept-Language': 'en-US,en;q=0.9',
-                'Content-Type': 'application/json',
-                'Origin': 'https://hosted.where2getit.com',
-                'Host': 'hosted.where2getit.com',
-                'Referer': 'https://hosted.where2getit.com/dollargeneral/',
-                'X-Requested-With': 'XMLHttpRequest'
-            }
+                headers = {
+                    'Accept': 'application/json, text/javascript, */*; q=0.01',
+                    'Accept-Encoding': 'gzip, deflate, br',
+                    'Accept-Language': 'en-US,en;q=0.9',
+                    'Content-Type': 'application/json',
+                    'Origin': 'https://hosted.where2getit.com',
+                    'Host': 'hosted.where2getit.com',
+                    'Referer': 'https://hosted.where2getit.com/dollargeneral/',
+                    'X-Requested-With': 'XMLHttpRequest'
+                }
 
-            yield scrapy.http.Request(
-                url,
-                self.parse,
-                method='POST',
-                body=json.dumps(formdata),
-                headers=headers,
-            )
+                yield scrapy.http.Request(
+                    url,
+                    self.parse,
+                    method='POST',
+                    body=json.dumps(formdata),
+                    headers=headers,
+                )
 
     def parse(self, response):
         result = json.loads(response.body_as_unicode())


Test patch: