Repository: unt-libraries/py-wasapi-client

Issue:
Handle  RemoteDisconnected
Trying to download a collection of ~7,000 WARCs will error and hang after downloading ~100 WARCs. Command executed:
`wasapi-client --collection 418 -u 'xxxxxxxx' -vv --log /somedir/mi_418_out.log.14 -p 1 -d /somedir/mi_418/`

```
Process Downloader-2:
Traceback (most recent call last):
  File "/home/user/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
    chunked=chunked)
  File "/home/user/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 384, in _make_request
    six.raise_from(e, None)
  File "<string>", line 2, in raise_from
  File "/home/user/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 380, in _make_request
    httplib_response = conn.getresponse()
  File "/usr/lib64/python3.6/http/client.py", line 1331, in getresponse
    response.begin()
  File "/usr/lib64/python3.6/http/client.py", line 297, in begin
    version, status, reason = self._read_status()
  File "/usr/lib64/python3.6/http/client.py", line 266, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
 
During handling of the above exception, another exception occurred:
 
Traceback (most recent call last):
  File "/home/user/.local/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
    timeout=timeout
  File "/home/user/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 638, in urlopen
    _stacktrace=sys.exc_info()[2])
  File "/home/user/.local/lib/python3.6/site-packages/urllib3/util/retry.py", line 368, in increment
    raise six.reraise(type(error), error, _stacktrace)
  File "/home/user/.local/lib/python3.6/site-packages/urllib3/packages/six.py", line 685, in reraise
    raise value.with_traceback(tb)
  File "/home/user/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 600, in urlopen
    chunked=chunked)
  File "/home/user/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 384, in _make_request
    six.raise_from(e, None)
  File "<string>", line 2, in raise_from
  File "/home/user/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 380, in _make_request
    httplib_response = conn.getresponse()
  File "/usr/lib64/python3.6/http/client.py", line 1331, in getresponse
    response.begin()
  File "/usr/lib64/python3.6/http/client.py", line 297, in begin
    version, status, reason = self._read_status()
  File "/usr/lib64/python3.6/http/client.py", line 266, in _read_status
    raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
 
During handling of the above exception, another exception occurred:
 
Traceback (most recent call last):
  File "/usr/lib64/python3.6/multiprocessing/process.py", line 258, in _bootstrap
    self.run()
  File "/home/user/.local/bin/wasapi_client.py", line 352, in run
    data_file = download_file(data_file, self.session, output_path)
  File "/home/user/.local/bin/wasapi_client.py", line 219, in download_file
    response = session.get(location, stream=True)
  File "/home/user/.local/lib/python3.6/site-packages/requests/sessions.py", line 546, in get
    return self.request('GET', url, **kwargs)
  File "/home/user/.local/lib/python3.6/site-packages/requests/sessions.py", line 533, in request
    resp = self.send(prep, **send_kwargs)
  File "/home/user/.local/lib/python3.6/site-packages/requests/sessions.py", line 668, in send
    history = [resp for resp in gen] if allow_redirects else []
  File "/home/user/.local/lib/python3.6/site-packages/requests/sessions.py", line 668, in <listcomp>
    history = [resp for resp in gen] if allow_redirects else []
  File "/home/user/.local/lib/python3.6/site-packages/requests/sessions.py", line 247, in resolve_redirects
    **adapter_kwargs
  File "/home/user/.local/lib/python3.6/site-packages/requests/sessions.py", line 646, in send
    r = adapter.send(request, **kwargs)
  File "/home/user/.local/lib/python3.6/site-packages/requests/adapters.py", line 498, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response',))
```

Comments:

Patch:
diff --git a/.travis.yml b/.travis.yml
index 5339b5e..d856516 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,9 +1,11 @@
+dist: xenial
 language: python
 sudo: false
 python:
   - "3.4"
   - "3.5"
   - "3.6"
+  - "3.7"
 install:
   - pip install -r requirements-test.txt
   - pip install flake8
diff --git a/tox.ini b/tox.ini
index f9a6310..0ab10bc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -2,7 +2,7 @@
 max-line-length = 99
 
 [tox]
-envlist = py34,py35,py36,flake8
+envlist = py34,py35,py36,py37,flake8
 
 [testenv]
 usedevelop=True
diff --git a/wasapi_client.py b/wasapi_client.py
index 29b7167..dbb7509 100755
--- a/wasapi_client.py
+++ b/wasapi_client.py
@@ -97,7 +97,7 @@ def get_webdata(webdata_uri, session):
     try:
         return response.json()
     except (JSONDecodeError, ValueError) as err:
-        sys.exit('Non-JSON response from {}'.format(webdata_uri))
+        sys.exit('Non-JSON response from {}:\n{}'.format(webdata_uri, err))
 
 
 def get_files_count(webdata_uri, auth=None):
@@ -216,7 +216,13 @@ def download_file(data_file, session, output_path):
         data_file.verified = True
         return data_file
     for location in data_file.locations:
-        response = session.get(location, stream=True)
+        try:
+            response = session.get(location, stream=True)
+        except requests.exceptions.RequestException as err:
+            # This could be a remote disconnect, read timeout, connection timeout,
+            # temporary name resolution issue...
+            LOGGER.error('Error downloading {}:\n{}'.format(location, err))
+            continue
         msg = '{}: {} {}'.format(location,
                                  response.status_code,
                                  response.reason)


Test patch:
diff --git a/requirements-test.txt b/requirements-test.txt
index 45fc928..4cdb0e8 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,2 +1,2 @@
 requests>=2.18.1
-pytest==3.2.1
+pytest>=4.6.4
diff --git a/tests/test_wasapi_client.py b/tests/test_wasapi_client.py
index f152c89..bc1e3e9 100644
--- a/tests/test_wasapi_client.py
+++ b/tests/test_wasapi_client.py
@@ -7,6 +7,7 @@ import multiprocessing
 import os
 import sys
 from collections import OrderedDict
+from logging import INFO
 from unittest.mock import call, mock_open, patch
 
 import pytest
@@ -331,14 +332,36 @@ class Test_download_file:
         with patch.object(session, 'get', return_value=mock_403) as mock_get, \
                 pytest.raises(wc.WASAPIDownloadError) as err:
             wc.download_file(self.data_file, session, self.filename)
-
         for item in (str(self.locations), self.filename):
-            assert item in str(err)
+            assert item in err.value.args[0]
         # Check all locations were tried.
         calls = [call(self.locations[0], stream=True),
                  call(self.locations[1], stream=True)]
         mock_get.assert_has_calls(calls)
 
+    def test_download_get_raises_some_RequestException(self, caplog):
+        caplog.set_level(INFO)
+        session = requests.Session()
+        mock_200 = MockResponse200('')
+
+        with patch.object(session, 'get') as mock_get, \
+                patch('wasapi_client.write_file') as mock_write_file:
+            # Raise a subclass of RequestException on first download attempt;
+            # mock a successful response on the second attempt
+            mock_get.side_effect = [requests.exceptions.ConnectionError(),
+                                    mock_200]
+            wc.download_file(self.data_file, session, self.filename)
+
+        # Check all locations were tried.
+        calls = [call(self.locations[0], stream=True),
+                 call(self.locations[1], stream=True)]
+        mock_get.assert_has_calls(calls)
+        mock_write_file.assert_called_once_with(mock_200, self.filename)
+        # Verify requests exception was caught and logged.
+        for msg in ('Error downloading http://loc1/blah.warc.gz:',
+                    'http://loc2/blah.warc.gz: 200 OK'):
+            assert msg in caplog.text
+
     def test_download_file_OSError(self):
         session = requests.Session()
         mock_200 = MockResponse200('')
@@ -350,7 +373,7 @@ class Test_download_file:
                 wc.download_file(self.data_file, session, self.filename)
 
         for item in (str(self.locations), self.filename):
-            assert item in str(err)
+            assert item in err.value.args[0]
         # Check we only tried downloading files until successful download.
         mock_get.assert_called_once_with(self.locations[0], stream=True)
         mock_write_file.assert_called_once_with(mock_200, self.filename)