Repository: openstates/openstates-scrapers

Issue:
MA failing since at least 2019-05-19
MA has been failing since 2019-05-19

Based on automated runs it appears that MA has not run successfully in 2 days (2019-05-19).


```
  OpenSSL.SSL.Error: [('SSL routines', 'CONNECT_CR_CERT', 'certificate verify failed')]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/urllib3/connectionpool.py", line 603, in urlopen
    chunked=chunked)
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/urllib3/connectionpool.py", line 344, in _make_request
    self._validate_conn(conn)
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/urllib3/connectionpool.py", line 843, in _validate_conn
    conn.connect()
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/urllib3/connection.py", line 350, in connect
    ssl_context=context)
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/urllib3/util/ssl_.py", line 355, in ssl_wrap_socket
    return context.wrap_socket(sock, server_hostname=server_hostname)
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py", line 478, in wrap_socket
    raise ssl.SSLError('bad handshake: %r' % e)
ssl.SSLError: ("bad handshake: Error([('SSL routines', 'CONNECT_CR_CERT', 'certificate verify failed')],)",)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
    timeout=timeout
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/urllib3/connectionpool.py", line 641, in urlopen
    _stacktrace=sys.exc_info()[2])
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/urllib3/util/retry.py", line 399, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='malegislature.gov', port=443): Max retries exceeded with url: /Bills/Search (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'CONNECT_CR_CERT', 'certificate verify failed')],)",),))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/opt/**PGUSER**/venv-pupa//bin/pupa", line 11, in <module>
    load_entry_point('pupa', 'console_scripts', 'pupa')()
  File "/opt/**PGUSER**/venv-pupa/src/pupa/pupa/cli/__main__.py", line 68, in main
    subcommands[args.subcommand].handle(args, other)
  File "/opt/**PGUSER**/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 278, in handle
    return self.do_handle(args, other, juris)
  File "/opt/**PGUSER**/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 323, in do_handle
    self.check_session_list(juris)
  File "/opt/**PGUSER**/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 246, in check_session_list
    scraped_sessions = juris.get_session_list()
  File "/opt/**PGUSER**/**PGUSER**/**PGUSER**/ma/__init__.py", line 81, in get_session_list
    'https://malegislature.gov/Bills/Search').text)
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/requests/api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/requests/api.py", line 60, in request
    return session.request(method=method, url=url, **kwargs)
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 533, in request
    resp = self.send(prep, **send_kwargs)
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/requests/sessions.py", line 646, in send
    r = adapter.send(request, **kwargs)
  File "/opt/**PGUSER**/venv-pupa/lib/python3.6/site-packages/requests/adapters.py", line 514, in send
    raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='malegislature.gov', port=443): Max retries exceeded with url: /Bills/Search (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'CONNECT_CR_CERT', 'certificate verify failed')],)",),))
```

Visit http://bobsled.openstates.org for more info.



Discussion:

Resolving patch:
diff --git a/openstates/ma/__init__.py b/openstates/ma/__init__.py
--- a/openstates/ma/__init__.py
+++ b/openstates/ma/__init__.py
@@ -78,7 +78,7 @@ def get_organizations(self):
 
     def get_session_list(self):
         doc = lxml.html.fromstring(requests.get(
-            'https://malegislature.gov/Bills/Search').text)
+            'https://malegislature.gov/Bills/Search', verify=False).text)
         sessions = doc.xpath("//div[@data-refinername='lawsgeneralcourt']/div/label/text()")
 
         # Remove all text between parens, like (Current) (7364)
diff --git a/openstates/ma/bills.py b/openstates/ma/bills.py
--- a/openstates/ma/bills.py
+++ b/openstates/ma/bills.py
@@ -10,6 +10,8 @@
 
 
 class MABillScraper(Scraper):
+    verify = False
+
     categorizer = Categorizer()
     session_filters = {}
     chamber_filters = {}
@@ -23,6 +25,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         # forcing these values so that 500s come back as skipped bills
         self.raise_errors = False
+        self.verify = False
 
     def format_bill_number(self, raw):
         return raw.replace('Bill ', '').replace('.', ' ').strip()
@@ -85,7 +88,7 @@ def list_bills(self, session, chamber, pageNumber):
                           pageNumber, session_filter, chamber_filter)
                       )
 
-        page = lxml.html.fromstring(requests.get(search_url).text)
+        page = lxml.html.fromstring(self.get(search_url).text)
         resultRows = page.xpath('//table[@id="searchTable"]/tbody/tr/td[2]/a/text()')
         return resultRows
 
@@ -102,7 +105,7 @@ def get_max_pages(self, session, chamber):
                       'Refinements%5Blawsbranchname%5D={}'.format(
                           session_filter, chamber_filter)
                       )
-        page = lxml.html.fromstring(requests.get(search_url).text)
+        page = lxml.html.fromstring(self.get(search_url).text)
 
         if page.xpath('//ul[contains(@class,"pagination-sm")]/li[last()]/a/@onclick'):
             maxPage = page.xpath('//ul[contains(@class,"pagination-sm")]/li[last()]/a/@onclick')[0]
@@ -118,7 +121,7 @@ def scrape_bill(self, session, bill_id, chamber):
         bill_url = 'https://malegislature.gov/Bills/{}/{}'.format(session_for_url, bill_id)
 
         try:
-            response = requests.get(bill_url)
+            response = self.get(bill_url)
             self.info("GET (with `requests`) - {}".format(bill_url))
         except requests.exceptions.RequestException:
             self.warning(u'Server Error on {}'.format(bill_url))
@@ -419,6 +422,7 @@ def scrape_senate_vote(self, vote, vurl):
     def get_as_ajax(self, url):
         # set the X-Requested-With:XMLHttpRequest so the server only sends along the bits we want
         s = requests.Session()
+        s.verify = False
         s.headers.update({'X-Requested-With': 'XMLHttpRequest'})
         return s.get(url)
 


Test patch: