Repository: alltheplaces/alltheplaces

Issue:
GEICO
https://www.geico.com/insurance-agents/


Discussion:

Resolving patch:
diff --git a/locations/spiders/geico.py b/locations/spiders/geico.py
new file mode 100644
--- /dev/null
+++ b/locations/spiders/geico.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+import json
+import re
+
+import scrapy
+
+from locations.items import GeojsonPointItem
+
+MAP_URL = 'https://www.geico.com/public/php/geo_map.php?'
+
+
+class GeicoSpider(scrapy.Spider):
+    name = "geico"
+    item_attributes = {'brand': 'GEICO', 'brand_wikidata': 'Q1498689'}
+    allowed_domains = ['www.geico.com']
+    start_urls = (
+        'https://www.geico.com/sitemap.xml',
+    )
+    download_delay = 1.5
+
+    def parse(self, response):
+        response.selector.remove_namespaces()
+
+        urls = response.xpath('//loc[contains(text(), "insurance-agents")]/text()').extract()
+
+        for url in urls:
+            if len(url.split('/')) > 7:  # location page
+                yield scrapy.Request(url, callback=self.parse_location)
+
+    def parse_location(self, response):
+        script_data = response.xpath(
+            '//script[@type="application/ld+json" and contains(text(), "streetAddress")]/text()').extract_first()
+        if script_data:
+            data = json.loads(script_data)
+            ref = "_".join(re.search(r".+/(.+?)/(.+?)/?(?:\.html|$)", response.url).groups())
+
+            metadata = {
+                'name': data["name"],
+                'ref': ref,
+                'addr_full': data["address"]["streetAddress"],
+                'city': data["address"]["addressLocality"],
+                'state': data["address"]["addressRegion"],
+                'postcode': data["address"]["postalCode"],
+                'phone': data.get("telephone"),
+                'website': data.get("url") or response.url,
+            }
+            address = " ".join([data["address"]["streetAddress"],
+                                data["address"]["addressLocality"] + ",",
+                                data["address"]["addressRegion"],
+                               data["address"]["postalCode"]])
+
+            form_data = {'address': address,
+                         'langauge': 'en-US',
+                         'type': 'Sales',
+                         'captcha': 'false'}
+
+            yield scrapy.http.FormRequest(url=MAP_URL, method='POST', formdata=form_data, meta=metadata,
+                                          callback=self.parse_location_map)
+
+    def parse_location_map(self, response):
+        data = json.loads(response.text)
+        map_data = data[1]  # skip header element
+        lat = map_data['latitude']
+        lon = map_data['longitude']
+
+        properties = {
+            'ref': response.meta['ref'],
+            'name': response.meta['name'],
+            'addr_full': response.meta['addr_full'],
+            'city': response.meta['city'],
+            'state': response.meta['state'],
+            'postcode': response.meta['postcode'],
+            'lat': lat,
+            'lon': lon,
+            'phone': response.meta['phone'],
+            'website': response.meta['website']
+        }
+
+        yield GeojsonPointItem(**properties)


Test patch: