Repository: openstates/openstates-scrapers

Issue:
DC failing since at least 2018-10-07
DC has been failing since 2018-10-07

Based on automated runs it appears that DC has not run successfully in 2 days (2018-10-07).


```
  /opt/openstates/venv-pupa/lib/python3.6/site-packages/psycopg2/__init__.py:144: UserWarning: The psycopg2 wheel package will be renamed from release 2.8; in order to keep installing from binary please use "pip install psycopg2-binary" instead. For details see: <http://initd.org/psycopg/docs/install.html#binary-install-from-pypi>.
  """)
00:02:15 INFO pupa: save jurisdiction District of Columbia as jurisdiction_ocd-jurisdiction-country:us-district:dc-government.json
00:02:15 INFO pupa: save organization Executive Office of the Mayor as organization_52cbefa8-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save organization Council of the District of Columbia as organization_52cbec42-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post Chairman as post_52cbf192-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post At-Large as post_52cbf30e-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post Ward 1 as post_52cbf4d0-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post Ward 2 as post_52cbf62e-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post Ward 3 as post_52cbf76e-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post Ward 4 as post_52cbf8ae-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post Ward 5 as post_52cbf9ee-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post Ward 6 as post_52cbfb38-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post Ward 7 as post_52cbfc78-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO pupa: save post Ward 8 as post_52cbfdb8-cab7-11e8-9f29-027a76602a4a.json
00:02:15 INFO scrapelib: GET - http://dccouncil.us/council
loaded Open States pupa settings...
dc (scrape, import)
  people: {}
  committees: {}
  bills: {}
Traceback (most recent call last):
  File "/opt/openstates/venv-pupa//bin/pupa", line 11, in <module>
    load_entry_point('pupa', 'console_scripts', 'pupa')()
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/__main__.py", line 68, in main
    subcommands[args.subcommand].handle(args, other)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 260, in handle
    return self.do_handle(args, other, juris)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
    report['scrape'] = self.do_scrape(juris, args, scrapers)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
    report[scraper_name] = scraper.do_scrape(**scrape_args)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/scrape/base.py", line 120, in do_scrape
    raise ScrapeError('no objects returned from {} scrape'.format(self.__class__.__name__))
pupa.exceptions.ScrapeError: no objects returned from DCPersonScraper scrape
```

Visit http://bobsled.openstates.org for more info.



Discussion:
Looks like they've launched a new site for people. I'll try to get started on a new one this week, it doesn't look super bad to scrape.

Resolving patch:
diff --git a/openstates/dc/people.py b/openstates/dc/people.py
--- a/openstates/dc/people.py
+++ b/openstates/dc/people.py
@@ -5,9 +5,10 @@
 
 def get_field(doc, key):
     # get text_content of parent of the element containing the key
-    elem = doc.xpath('//div[@id="member-info"]/p/strong[text()="%s"]/..' % key)
+    # elem = doc.xpath('//div[@id="member-info"]/p/strong[text()="%s"]/..' % key)
+    elem = doc.xpath('//li[contains(@class,"column")]/h4[text()="%s"]/../p')
     if elem:
-        return elem[0].text_content().replace(key, '').strip()
+        return elem[0].text_content().strip()
     else:
         return ''
 
@@ -15,12 +16,13 @@ def get_field(doc, key):
 class DCPersonScraper(Scraper):
 
     def scrape(self):
-        council_url = 'http://dccouncil.us/council'
+        council_url = 'http://dccouncil.us/councilmembers/'
         data = self.get(council_url).text
         doc = lxml.html.fromstring(data)
         doc.make_links_absolute(council_url)
         # page should have 13 unique council URLs
-        urls = set(doc.xpath('//a[contains(@href, "dccouncil.us/council/")]/@href'))
+        urls = set(
+            doc.xpath('//a[contains(@href, "dccouncil.us/council/")]/@href'))
         # print '\n'.join(urls)
         assert len(urls) <= 13, "should have 13 unique councilmember URLs"
 
@@ -29,42 +31,52 @@ def scrape(self):
             doc = lxml.html.fromstring(data)
             doc.make_links_absolute(url)
 
-            descriptor = doc.xpath('//p[@class="head-descriptor"]/text()')[0]
-            title_name = doc.xpath('//h2/text()')[0]
+            descriptor = doc.xpath(
+                '//div[contains(@class,"media-object-section")]/'
+                'p[contains(@class,"h4")]/text()')[0]
+            title_name = doc.xpath(
+                '//div[contains(@class,"media-object-section")]/h1/text()')[0]
 
             # removes the title that is prepended to the name
             name = re.sub(r'^Councilmember ', '', title_name)
 
-            if 'Chairman' in descriptor:
+            if 'chairman' in descriptor.lower():
                 district = 'Chairman'
-            else:
-                district = get_field(doc, 'Represents: ')
-
-            if not district:
+            elif 'at-large' in descriptor.lower():
                 district = 'At-Large'
+            else:
+                district = descriptor.split("&bullet;")[1].strip()
 
             # party
-            party = get_field(doc, "Political Affiliation:")
+            party = get_field(doc, "Political Affiliation")
             if 'Democratic' in party:
                 party = 'Democratic'
+            elif 'Republican' in party:
+                party = 'Republican'
             else:
                 party = 'Independent'
 
-            photo_url = doc.xpath('//div[@id="member-thumb"]/img/@src')
+            photo_url = doc.xpath('//figure/a/img/@src')
             if photo_url:
                 photo_url = photo_url[0]
             else:
                 photo_url = ''
 
-            office_address = get_field(doc, "Office:")
-            phone = get_field(doc, "Tel:")
-            if phone.endswith('| Fax:'):
-                fax = None
-                phone = phone.strip('| Fax:') or None
-            else:
-                phone, fax = phone.split(' | Fax: ')
+            office_address = get_field(doc, "Office")
+
+            faxes = doc.xpath('//p[@class="byline"]/text()')
+            fax = faxes[-1].strip()
+
+            email = doc.xpath(
+                '//p[@class="byline"]/a[@class="contact-link"]')[0].text_content().strip()
+            phone = doc.xpath(
+                '//p[@class="byline"]/a[@class="contact-link"]')[1].text_content().strip()
 
-            email = doc.xpath('//a[starts-with(text(), "Send an email")]/@href')[0].split(':')[1]
+            bio = "\n".join(
+                doc.xpath('//div[contains(@class,"js-hide")]/p/text()')).strip()
+            if doc.xpath('//p[contains(@class,"page-summary")]'):
+                short_bio = doc.xpath(
+                    '//p[contains(@class,"page-summary")]')[0].text_content().strip()
 
             person = Person(
                 name=name,
@@ -72,6 +84,8 @@ def scrape(self):
                 image=photo_url,
                 primary_org='legislature',
                 district=str(district),
+                biography=bio,
+                summary=short_bio
             )
 
             person.add_source(url)


Test patch: