Repository: openstates/openstates-scrapers

Issue:
SC failing since at least 2018-10-28
SC has been failing since 2018-10-28

Based on automated runs it appears that SC has not run successfully in 5 days (2018-10-28).


```
      self._validate_conn(conn)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/urllib3/connectionpool.py", line 839, in _validate_conn
    conn.connect()
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/urllib3/connection.py", line 344, in connect
    ssl_context=context)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/urllib3/util/ssl_.py", line 342, in ssl_wrap_socket
    return context.wrap_socket(sock, server_hostname=server_hostname)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py", line 459, in wrap_socket
    raise ssl.SSLError('bad handshake: %r' % e)
ssl.SSLError: ("bad handshake: Error([('SSL routines', 'CONNECT_CR_CERT', 'certificate verify failed')],)",)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
    timeout=timeout
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
    _stacktrace=sys.exc_info()[2])
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/urllib3/util/retry.py", line 398, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='www.scstatehouse.gov', port=443): Max retries exceeded with url: /billsearch.php (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'CONNECT_CR_CERT', 'certificate verify failed')],)",),))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/opt/openstates/venv-pupa//bin/pupa", line 11, in <module>
    load_entry_point('pupa', 'console_scripts', 'pupa')()
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/__main__.py", line 68, in main
    subcommands[args.subcommand].handle(args, other)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 260, in handle
    return self.do_handle(args, other, juris)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 302, in do_handle
    self.check_session_list(juris)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 228, in check_session_list
    scraped_sessions = juris.get_session_list()
  File "/opt/openstates/openstates/openstates/sc/__init__.py", line 112, in get_session_list
    doc = lxml.html.fromstring(requests.get(url).text)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/api.py", line 60, in request
    return session.request(method=method, url=url, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 524, in request
    resp = self.send(prep, **send_kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 659, in send
    history = [resp for resp in gen] if allow_redirects else []
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 659, in <listcomp>
    history = [resp for resp in gen] if allow_redirects else []
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 238, in resolve_redirects
    **adapter_kwargs
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 637, in send
    r = adapter.send(request, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.6/site-packages/requests/adapters.py", line 514, in send
    raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='www.scstatehouse.gov', port=443): Max retries exceeded with url: /billsearch.php (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'CONNECT_CR_CERT', 'certificate verify failed')],)",),))
```

Visit http://bobsled.openstates.org for more info.



Discussion:

Resolving patch:
diff --git a/openstates/sc/__init__.py b/openstates/sc/__init__.py
--- a/openstates/sc/__init__.py
+++ b/openstates/sc/__init__.py
@@ -109,5 +109,5 @@ def get_session_list(self):
         url = 'http://www.scstatehouse.gov/billsearch.php'
         path = "//select[@id='session']/option/text()"
 
-        doc = lxml.html.fromstring(requests.get(url).text)
+        doc = lxml.html.fromstring(requests.get(url, verify=False).text)
         return doc.xpath(path)
diff --git a/openstates/sc/bills.py b/openstates/sc/bills.py
--- a/openstates/sc/bills.py
+++ b/openstates/sc/bills.py
@@ -147,7 +147,7 @@ def scrape_subjects(self, session):
                              ('INDEXCODE', '0'),
                              ('INDEXTEXT', ''),
                              ('AORB', 'B'),
-                             ('PAGETYPE', '0')))).text
+                             ('PAGETYPE', '0'))), verify=False).text
         doc = lxml.html.fromstring(data)
         # skip first two subjects, filler options
         for option in doc.xpath('//option')[2:]:
@@ -155,7 +155,7 @@ def scrape_subjects(self, session):
             code = option.get('value')
             url = '%s?AORB=B&session=%s&indexcode=%s' % (subject_search_url,
                                                          session_code, code)
-            data = self.get(url).text
+            data = self.get(url, verify=False).text
             doc = lxml.html.fromstring(data)
             for bill in doc.xpath('//span[@style="font-weight:bold;"]'):
                 match = re.match(r'(?:H|S) \d{4}', bill.text)
@@ -172,7 +172,7 @@ def scrape_vote_history(self, bill, vurl):
         :param vurl: source for the voteEvent information.
         :return: voteEvent object
         """
-        html = self.get(vurl).text
+        html = self.get(vurl, verify=False).text
         doc = lxml.html.fromstring(html)
         doc.make_links_absolute(vurl)
 
@@ -297,7 +297,7 @@ def scrape_details(self, bill_detail_url, session, chamber, bill_id):
         :param bill_id:
         :return:
         """
-        page = self.get(bill_detail_url).text
+        page = self.get(bill_detail_url, verify=False).text
 
         if 'INVALID BILL NUMBER' in page:
             self.warning('INVALID BILL %s' % bill_detail_url)
@@ -357,7 +357,7 @@ def scrape_details(self, bill_detail_url, session, chamber, bill_id):
 
         # find versions
         version_url = doc.xpath('//a[text()="View full text"]/@href')[0]
-        version_html = self.get(version_url).text
+        version_html = self.get(version_url, verify=False).text
         version_doc = lxml.html.fromstring(version_html)
         version_doc.make_links_absolute(version_url)
         for version in version_doc.xpath('//a[contains(@href, "/prever/")]'):
@@ -425,7 +425,7 @@ def scrape(self, chamber=None, session=None):
             index_url = self.urls[chamber]['daily-bill-index']
             chamber_letter = 'S' if chamber == 'upper' else 'H'
 
-            page = self.get(index_url).text
+            page = self.get(index_url, verify=False).text
             doc = lxml.html.fromstring(page)
             doc.make_links_absolute(index_url)
 


Test patch: