Repository: Duke-GCB/DukeDSClient

Issue:
User received TooLargeChunkDownloadError exception
User was downloading a project via `ddsclient download -p <projectname> --include <filepath>`.

Error:
```
  File ".../conda/envs/bespin/lib/python3.7/site-packages/ddsc/core/download.py", line 455, 
in _write_response_to_file
    self._on_bytes_read(len(chunk))
  File ".../conda/envs/bespin/lib/python3.7/site-packages/ddsc/core/download.py", line 464, 
in _on_bytes_read
    raise TooLargeChunkDownloadError(self.actual_bytes_read, self.bytes_to_read, 
  self.local_path)

ddsc.core.download.TooLargeChunkDownloadError: 
Received too many bytes downloading part of a file. 
Actual: 44159228 Expected: 32963486 File: <filename>
```

I was unable to reproduce the issue the same file downloaded fine for me.
In this place in the code we are reading from a GET request with `Range` headers delimited to the part of the file we are downloading. So we should only receive the amount we are expecting.
This exception was added to catch the unlikely case that we received more than we asked for. 



Discussion:

Patch:
diff --git a/ddsc/core/download.py b/ddsc/core/download.py
index f9bf81b..545d438 100644
--- a/ddsc/core/download.py
+++ b/ddsc/core/download.py
@@ -439,6 +439,7 @@ class RetryChunkDownloader(object):
         if response.status_code == 401:
             raise DownloadInconsistentError(response.text)
         response.raise_for_status()
+        self.actual_bytes_read = 0
         self._write_response_to_file(response)
         self._verify_download_complete()
 


Test patch:
diff --git a/ddsc/core/tests/test_download.py b/ddsc/core/tests/test_download.py
index 4fa9275..7535a22 100644
--- a/ddsc/core/tests/test_download.py
+++ b/ddsc/core/tests/test_download.py
@@ -541,3 +541,41 @@ class TestRetryChunkDownloader(TestCase):
 
         with self.assertRaises(PartialChunkDownloadError):
             downloader._verify_download_complete()
+
+    @patch('ddsc.core.download.RemoteFileUrl')
+    @patch('ddsc.core.download.requests')
+    def test_retry_download_loop_too_few_actual_bytes_read(self, mock_requests, mock_remote_file_url):
+        mock_requests.exceptions.ConnectionError = ValueError
+        mock_project_file = Mock()
+        mock_context = Mock()
+        downloader = RetryChunkDownloader(project_file=mock_project_file, local_path=None, seek_amt=0,
+                                          bytes_to_read=100, download_context=mock_context)
+        downloader.max_retry_times = 2
+        downloader.get_url_and_headers_for_range = Mock()
+        downloader.get_url_and_headers_for_range.return_value = ('someurl', ['headers'])
+        mock_requests.get.return_value.iter_content.return_value = ['X' * 10]
+        fake_open = mock_open()
+        with patch('ddsc.core.download.open', fake_open, create=True):
+            with self.assertRaises(PartialChunkDownloadError):
+                downloader.retry_download_loop()
+        self.assertEqual(downloader.actual_bytes_read, 10)
+
+    @patch('ddsc.core.download.RemoteFileUrl')
+    @patch('ddsc.core.download.requests')
+    def test_retry_download_loop_too_few_actual_bytes_read_then_works(self, mock_requests, mock_remote_file_url):
+        mock_requests.exceptions.ConnectionError = ValueError
+        mock_project_file = Mock()
+        mock_context = Mock()
+        downloader = RetryChunkDownloader(project_file=mock_project_file, local_path=None, seek_amt=0,
+                                          bytes_to_read=100, download_context=mock_context)
+        downloader.max_retry_times = 2
+        downloader.get_url_and_headers_for_range = Mock()
+        downloader.get_url_and_headers_for_range.return_value = ('someurl', ['headers'])
+        mock_requests.get.return_value.iter_content.side_effect = [
+            ['X' * 10],  # only 10 bytes the first time
+            ['X' * 100],  # all 100 bytes the second time
+        ]
+        fake_open = mock_open()
+        with patch('ddsc.core.download.open', fake_open, create=True):
+            downloader.retry_download_loop()
+        self.assertEqual(downloader.actual_bytes_read, 100)