Repository: openstates/openstates-scrapers

Issue:
TX failing since at least 2018-06-30
TX has been failing since 2018-06-30

Based on automated runs it appears that TX has not run successfully in 2 days (2018-06-30).


```
  During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/adapters.py", line 445, in send
    timeout=timeout
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
    _stacktrace=sys.exc_info()[2])
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/urllib3/util/retry.py", line 398, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='house.texas.gov', port=443): Max retries exceeded with url: /members/ (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')],)",),))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/opt/openstates/venv-pupa//bin/pupa", line 11, in <module>
    load_entry_point('pupa', 'console_scripts', 'pupa')()
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/__main__.py", line 68, in main
    subcommands[args.subcommand].handle(args, other)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 260, in handle
    return self.do_handle(args, other, juris)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
    report['scrape'] = self.do_scrape(juris, args, scrapers)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
    report[scraper_name] = scraper.do_scrape(**scrape_args)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/scrape/base.py", line 111, in do_scrape
    for obj in self.scrape(**kwargs) or []:
  File "/opt/openstates/openstates/openstates/tx/people.py", line 87, in scrape
    yield from self.scrape_chamber('lower')
  File "/opt/openstates/openstates/openstates/tx/people.py", line 96, in scrape_chamber
    response = self.get(roster_url)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 525, in get
    return self.request('GET', url, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/scrapelib/__init__.py", line 286, in request
    **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/scrapelib/cache.py", line 66, in request
    resp = super(CachingSession, self).request(method, url, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/scrapelib/__init__.py", line 88, in request
    return super(ThrottledSession, self).request(method, url, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/scrapelib/__init__.py", line 153, in request
    resp = super(RetrySession, self).request(method, url, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 512, in request
    resp = self.send(prep, **send_kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 644, in send
    history = [resp for resp in gen] if allow_redirects else []
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 644, in <listcomp>
    history = [resp for resp in gen] if allow_redirects else []
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 222, in resolve_redirects
    **adapter_kwargs
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 622, in send
    r = adapter.send(request, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/adapters.py", line 511, in send
    raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='house.texas.gov', port=443): Max retries exceeded with url: /members/ (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')],)",),))
```

Visit http://bobsled.openstates.org for more info.



Discussion:

Resolving patch:
diff --git a/openstates/tx/people.py b/openstates/tx/people.py
--- a/openstates/tx/people.py
+++ b/openstates/tx/people.py
@@ -12,6 +12,9 @@
 class TXPersonScraper(Scraper, LXMLMixin):
     jurisdiction = 'tx'
 
+    # bad SSL as of July 2018
+    verify = False
+
     def __init__(self, *args, **kwargs):
         super(TXPersonScraper, self).__init__(*args, **kwargs)
 
@@ -88,12 +91,12 @@ def scrape(self, chamber=None):
 
     def scrape_chamber(self, chamber):
         rosters = {
-            'lower': 'http://www.house.state.tx.us/members/',
+            'lower': 'https://www.house.state.tx.us/members/',
             'upper': 'http://www.senate.texas.gov/directory.php'
         }
 
         roster_url = rosters[chamber]
-        response = self.get(roster_url)
+        response = self.get(roster_url, verify=False)
         # auto detect encoding
         response.encoding = response.apparent_encoding
         roster_page = lxml.html.fromstring(response.text)


Test patch: