Repository: openstates/openstates-scrapers

Issue:
SC failing since at least 2017-10-29
SC has been failing since 2017-10-29

Based on automated runs it appears that SC has not run successfully in 2 days (2017-10-29).


```
    File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/requests/models.py", line 745, in generate
    for chunk in self.raw.stream(chunk_size, decode_content=True):
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/urllib3/response.py", line 432, in stream
    for line in self.read_chunked(amt, decode_content=decode_content):
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/urllib3/response.py", line 626, in read_chunked
    self._original_response.close()
  File "/usr/lib/python3.5/contextlib.py", line 77, in __exit__
    self.gen.throw(type, value, traceback)
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/urllib3/response.py", line 320, in _error_catcher
    raise ProtocolError('Connection broken: %r' % e, e)
urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(0 bytes read)', IncompleteRead(0 bytes read))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/opt/openstates/venv-pupa//bin/pupa", line 11, in <module>
    load_entry_point('pupa', 'console_scripts', 'pupa')()
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/__main__.py", line 67, in main
    subcommands[args.subcommand].handle(args, other)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 260, in handle
    return self.do_handle(args, other, juris)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 305, in do_handle
    report['scrape'] = self.do_scrape(juris, args, scrapers)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/cli/commands/update.py", line 173, in do_scrape
    report[scraper_name] = scraper.do_scrape(**scrape_args)
  File "/opt/openstates/venv-pupa/src/pupa/pupa/scrape/base.py", line 111, in do_scrape
    for obj in self.scrape(**kwargs) or []:
  File "/opt/openstates/openstates/openstates/sc/bills.py", line 375, in scrape
    self.scrape_subjects(session)
  File "/opt/openstates/openstates/openstates/sc/bills.py", line 116, in scrape_subjects
    data = self.get(url).text
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/requests/sessions.py", line 521, in get
    return self.request('GET', url, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/scrapelib/__init__.py", line 286, in request
    **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/scrapelib/cache.py", line 66, in request
    resp = super(CachingSession, self).request(method, url, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/scrapelib/__init__.py", line 92, in request
    return super(ThrottledSession, self).request(method, url, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/scrapelib/__init__.py", line 182, in request
    raise exception_raised
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/scrapelib/__init__.py", line 157, in request
    resp = super(RetrySession, self).request(method, url, **kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/requests/sessions.py", line 508, in request
    resp = self.send(prep, **send_kwargs)
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/requests/sessions.py", line 658, in send
    r.content
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/requests/models.py", line 823, in content
    self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes()
  File "/opt/openstates/venv-pupa/lib/python3.5/site-packages/requests/models.py", line 748, in generate
    raise ChunkedEncodingError(e)
requests.exceptions.ChunkedEncodingError: ('Connection broken: IncompleteRead(0 bytes read)', IncompleteRead(0 bytes read))
```

Visit http://bobsled.openstates.org for more info.



Discussion:

Resolving patch:
diff --git a/openstates/sc/bills.py b/openstates/sc/bills.py
--- a/openstates/sc/bills.py
+++ b/openstates/sc/bills.py
@@ -9,6 +9,14 @@
 from pupa.utils.generic import convert_pdf
 import lxml.html
 
+# Workaround to prevent chunking error (thanks @showerst)
+#
+# @see https://stackoverflow.com/a/37818792/1858091
+import http.client
+
+http.client.HTTPConnection._http_vsn = 10
+http.client.HTTPConnection._http_vsn_str = 'HTTP/1.0'
+
 
 def action_type(action):
     """
@@ -110,12 +118,6 @@ def scrape_subjects(self, session):
         for option in doc.xpath('//option')[2:]:
             subject = option.text
             code = option.get('value')
-
-            if code == '1130':
-                # added Nov 2017, this code broke the scrape for a few weeks
-                self.warning('skipping 1130 - broken page')
-                continue
-
             url = '%s?AORB=B&session=%s&indexcode=%s' % (subject_search_url,
                                                          session_code, code)
             data = self.get(url).text


Test patch: