Repository: alltheplaces/alltheplaces

Issue:
Spider statefarm is broken
During the global build at 2021-09-22-14-42-27, spider **statefarm** failed with **0 features** and **6575 errors**.

Here's [the log](https://data.alltheplaces.xyz/runs/2021-09-22-14-42-27/logs/statefarm.txt) and [the output](https://data.alltheplaces.xyz/runs/2021-09-22-14-42-27/output/statefarm.geojson) ([on a map](https://data.alltheplaces.xyz/map.html?show=https://data.alltheplaces.xyz/runs/2021-09-22-14-42-27/output/statefarm.geojson))


Discussion:

Resolving patch:
diff --git a/locations/spiders/statefarm.py b/locations/spiders/statefarm.py
--- a/locations/spiders/statefarm.py
+++ b/locations/spiders/statefarm.py
@@ -9,7 +9,7 @@ class StateFarmSpider(scrapy.Spider):
     name = "statefarm"
     item_attributes = { 'brand': "State Farm" }
     allowed_domains = ["statefarm.com"]
-    download_delay = 0.2
+    download_delay = 0.1
 
     start_urls = [
         'https://www.statefarm.com/agent/us',
@@ -17,23 +17,20 @@ class StateFarmSpider(scrapy.Spider):
 
     def parse_location(self, response):
 
-        name = response.xpath('//*[@id="AgentNameLabelId"]//span[@itemprop="name"]/text()').extract_first()
+        name = response.xpath('//span[@itemprop="name"]/text()').extract_first()
         if name:
             name += ' - State Farm Insurance Agent'
 
-        lat = response.xpath('//*[@id="agentOfficePrimaryLocLat"]/@value').extract_first()
-        lon = response.xpath('//*[@id="agentOfficePrimaryLocLong"]/@value').extract_first()
-
         properties = {
             'ref': "_".join(response.url.split('/')[-3:]),
             'name': name,
-            'addr_full':  response.xpath('normalize-space(//div[@itemtype="http://schema.org/PostalAddress"]//span[@id="locStreetContent_mainLocContent"]/text())').extract_first(),
-            'city':  response.xpath('//div[@itemtype="http://schema.org/PostalAddress"]/div[2]/span/span[1]/text()').extract_first().strip(', '),
-            'state':  response.xpath('//div[@itemtype="http://schema.org/PostalAddress"]/div[2]/span/span[2]/text()').extract_first(),
-            'postcode':  response.xpath('//div[@itemtype="http://schema.org/PostalAddress"]/div[2]/span/span[3]/text()').extract_first(),
-            'phone': response.xpath('normalize-space(//span[@id="offNumber_mainLocContent"]/span/text())').extract_first(),
-            'lat': float(lat) if lat else None,
-            'lon': float(lon) if lon else None,
+            'addr_full': response.xpath('normalize-space(//div[@itemtype="http://schema.org/PostalAddress"]//*[@itemprop="streetAddress"]/text())').extract_first(),
+            'city': response.xpath('//div[@itemtype="http://schema.org/PostalAddress"]//*[@itemprop="addressLocality"]/text()').extract_first(),
+            'state': response.xpath('//div[@itemtype="http://schema.org/PostalAddress"]//*[@itemprop="addressRegion"]/text()').extract_first(),
+            'postcode': response.xpath('//div[@itemtype="http://schema.org/PostalAddress"]//*[@itemprop="postalCode"]/text()').extract_first(),
+            'phone': response.xpath('//*[@itemprop="telephone"]/a/text()').extract_first(),
+            'lat': float(response.xpath('//@data-latitude').extract_first()),
+            'lon': float(response.xpath('//@data-longitude').extract_first()),
             'website': response.url,
         }
 
@@ -45,7 +42,7 @@ def parse(self, response):
 
         if agents:
             for agent in agents:
-                agent_site = agent.xpath('.//a[contains(text(), "Visit agent site")]/@href').extract_first()
+                agent_site = agent.xpath('.//a[contains(text(), "Agent Website")]/@href').extract_first()
                 if not agent_site:
                     raise Exception('no agent site found')
                 yield scrapy.Request(response.urljoin(agent_site), callback=self.parse_location)


Test patch: