Repository: openstates/openstates-scrapers

Issue:
ID upper + lower: missing legislator phone numbers
Upper: https://legislature.idaho.gov/senate/membership/
Lower: https://legislature.idaho.gov/house/membership/



Discussion:
I'll take a look at this.

Resolving patch:
diff --git a/openstates/id/legislators.py b/openstates/id/legislators.py
--- a/openstates/id/legislators.py
+++ b/openstates/id/legislators.py
@@ -1,18 +1,48 @@
-from billy.scrape.legislators import LegislatorScraper, Legislator
 import re
-import datetime
 import lxml.html
+from billy.scrape.legislators import LegislatorScraper, Legislator
 
-_BASE_URL = 'https://legislature.idaho.gov/%s/membership/'
-_CHAMBERS = {'upper':'senate', 'lower':'house'}
-_PARTY = {
-        '(R)': 'Republican',
-        '(D)': 'Democratic',
-    }
-_PHONE_NUMBERS = {'hom':'phone_number',
-                  'bus':'business_phone',
-                  'fax':'fax_number'}
-
+BASE_URL = 'https://legislature.idaho.gov/%s/membership/'
+CHAMBERS = {'upper': 'senate', 'lower': 'house'}
+PARTY = {
+    '(R)': 'Republican',
+    '(D)': 'Democratic',
+}
+
+phone_patterns = {
+    'office': re.compile(r'Statehouse'),
+    'business': re.compile(r'Bus'),
+    'home': re.compile(r'Home'),
+}
+def get_phones(el):
+    phones = {}
+    for link in el.xpath('p/a[@class = "mob-tel"]'):
+        prefix = link.getprevious().tail
+        for label, pattern in phone_patterns.items():
+            if pattern.search(prefix) is not None:
+                phones[label] = parse_phone(link.get('href'))
+    return phones
+
+parse_phone_pattern = re.compile(r'tel:(?:\+1)?(\d{10}$)')
+def parse_phone(phone):
+    res = parse_phone_pattern.search(phone)
+    if res is not None:
+        return res.groups()[0]
+
+fax_pattern = re.compile(r'fax\s+\((\d{3})\)\s+(\d{3})-(\d{4})', re.IGNORECASE)
+def get_fax(el):
+    res = fax_pattern.search(el.text_content())
+    if res is not None:
+        return ''.join(res.groups())
+
+address_pattern = re.compile(r', \d{5}')
+address_replace_pattern = re.compile(r'(\d{5})')
+def get_address(el):
+    for br in el.xpath('p/br'):
+        piece = (br.tail or '').strip()
+        res = address_pattern.search(piece)
+        if res is not None:
+            return address_replace_pattern.sub(r'ID \1', piece).strip()
 
 class IDLegislatorScraper(LegislatorScraper):
     """Legislator data seems to be available for the current term only."""
@@ -23,7 +53,7 @@ def scrape(self, chamber, term):
         Scrapes legislators for the current term only
         """
         self.validate_term(term, latest_only=True)
-        url = _BASE_URL % _CHAMBERS[chamber].lower()
+        url = BASE_URL % CHAMBERS[chamber].lower()
         index = self.get(url).text
         html = lxml.html.fromstring(index)
         html.make_links_absolute(url)
@@ -37,35 +67,19 @@ def scrape(self, chamber, term):
 
             name = inner.xpath('p/strong')[0].text.replace(u'\xa0', ' ').strip()
             name = re.sub('\s+', ' ', name)
-            party = _PARTY[inner.xpath('p/strong')[0].tail.strip()]
+            party = PARTY[inner.xpath('p/strong')[0].tail.strip()]
             email = inner.xpath('p/strong/a')[0].text
             district = inner.xpath('p/a')[0].text.replace('District ', '')
             leg_url = inner.xpath('p/a/@href')[0]
 
-            address = home_phone = office_phone = fax = None
-
-            for br in inner.xpath('p/br'):
-                piece = br.tail or ''
-                piece = piece.strip()
-
-                if re.findall(', \d{5}', piece):
-                    address = re.sub(r'(\d{5})', r'ID \1', piece).strip()
-                elif piece.startswith('Home '):
-                    home_phone = piece[5:]
-                elif piece.startswith('Bus '):
-                    office_phone = piece[4:]
-                elif piece.startswith('FAX '):
-                    fax = piece[4:]
-                print(piece)
-
-
-
             leg = Legislator(term, chamber, district, name, party=party,
                              email=email)
 
-            phone = home_phone or office_phone
+            phones = get_phones(inner)
             leg.add_office('district', 'District Office',
-                           address=address, fax=fax, phone=phone)
+                           address=get_address(inner), fax=get_fax(inner),
+                           phone=phones.get('home') or phones.get('business'))
+            leg.add_office('capitol', 'Capitol Office', phone=phones.get('office'))
 
             leg.add_source(url)
             leg['photo_url'] = img_url


Test patch: