Repository: gtsystem/python-remotezip

Issue:
Unhandled HTTPErrors When Negative Offset Not Supported
When trying to use remotezip with PyPi, I discovered that while their server supports range requests, it does not support using a negative offset to get {content-length - bytes}-{content-length}.

When raise_for_status is called, an HTTP 501 error is returned (it could theoretically also be a 405) and remotezip aborts.
 https://github.com/gtsystem/python-remotezip/blob/2afca9a2df6c302f61eb612683bd4e4ee74e9cb5/remotezip.py#L184-L189

Reproducer:
```
from remotezip import RemoteZip

url = "https://files.pythonhosted.org/packages/71/6d/95777fd66507106d2f8f81d005255c237187951644f85a5bd0baeec8a88f/paramiko-2.12.0-py2.py3-none-any.whl"
  with RemoteZip(url) as wzip:
      wzip.extract('METADATA')
```

I was going to do a pull request, but not being super good with Python myself, I found it was not obviously fixible (to me at least) with a simple try/catch since it's going through constructors, etc. 

Checking for a 501 or 405 error before raise_for_status and falling back to getting self.__file_size by a separate http request for content-length should fix this.

Something like:
```
def _request(self, kwargs): 
     if self._session: 
         res = self._session.get(self._url, stream=True, **kwargs) 
     else: 
         res = requests.get(self._url, stream=True, **kwargs) 
if res._status_code == 501 or 405:
    (do whatever needs to be done so that
    self._file_size = requests.get(self._url, stream=True).headers['Content-Length']
    )
res.raise_for_status()
```

Edit: I also submitted a bug/feature request to PyPI/warehoise about this on their server; I don't anticipate they will implement it quickly, but if they do, the given reproducer may not work. 

Discussion:

Patch:
diff --git a/README.md b/README.md
index f359611..d570447 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,8 @@ To download the content, this library rely on the `requests` module. The constru
 * ... Please look at the [requests](http://docs.python-requests.org/en/master/user/quickstart/#make-a-request) documentation for futher usage details.
 * **initial\_buffer\_size**: How much data (in bytes) to fetch during the first connection to download the zip file central directory. If your zip file conteins a lot of files, would be a good idea to increase this parameter in order to avoid the need for further remote requests. *Default: 64kb*.
 * **session**: a custom session object to use for the request.
+* **support_suffix_range**: You can set this attribute to `False` if the remote server doesn't support suffix range
+  (negative offset). Notice that this option will use one more HEAD request to fetch the content length.
 
 ### Class Interface
 
diff --git a/remotezip.py b/remotezip.py
index cfe6924..27366c2 100644
--- a/remotezip.py
+++ b/remotezip.py
@@ -162,10 +162,11 @@ class RemoteIO(io.IOBase):
 
 class RemoteFetcher:
     """Represent a remote file to be fetched in parts"""
-    def __init__(self, url, session=None, **kwargs):
+    def __init__(self, url, session=None, support_suffix_range=True, **kwargs):
         self._kwargs = kwargs
         self._url = url
         self._session = session
+        self._support_suffix_range = support_suffix_range
 
     @staticmethod
     def parse_range_header(content_range_header):
@@ -191,15 +192,33 @@ class RemoteFetcher:
             raise RangeNotSupported("The server doesn't support range requests")
         return res.raw, res.headers['Content-Range']
 
-    def prepare_request(self, data_range):
-        range_header = self.build_range_header(*data_range)
+    def prepare_request(self, data_range=None):
         kwargs = dict(self._kwargs)
         kwargs['headers'] = headers = dict(kwargs.get('headers', {}))
-        headers['Range'] = range_header
+        if data_range is not None:
+            headers['Range'] = self.build_range_header(*data_range)
         return kwargs
 
+    def get_file_size(self):
+        if self._session:
+            res = self._session.head(self._url, **self.prepare_request())
+        else:
+            res = requests.head(self._url, **self.prepare_request())
+        try:
+            res.raise_for_status()
+            return int(res.headers['Content-Length'])
+        except IOError as e:
+            raise RemoteIOError(str(e))
+        except KeyError:
+            raise RemoteZipError("Cannot get file size: Content-Length header missing")
+
     def fetch(self, data_range, stream=False):
         """Fetch a part of a remote file"""
+        # Handle the case suffix range request is not supported. Fixes #15
+        if data_range[0] < 0 and data_range[1] is None and not self._support_suffix_range:
+            size = self.get_file_size()
+            data_range = (max(0, size + data_range[0]), size - 1)
+
         kwargs = self.prepare_request(data_range)
         try:
             res, range_header = self._request(kwargs)
@@ -217,8 +236,9 @@ def pairwise(iterable):
 
 
 class RemoteZip(zipfile.ZipFile):
-    def __init__(self, url, initial_buffer_size=64*1024, session=None, fetcher=RemoteFetcher, **kwargs):
-        fetcher = fetcher(url, session, **kwargs)
+    def __init__(self, url, initial_buffer_size=64*1024, session=None, fetcher=RemoteFetcher, support_suffix_range=True,
+                 **kwargs):
+        fetcher = fetcher(url, session, support_suffix_range=support_suffix_range, **kwargs)
         rio = RemoteIO(fetcher.fetch, initial_buffer_size)
         super(RemoteZip, self).__init__(rio)
         rio.set_position_to_size(self._get_position_to_size())
diff --git a/setup.py b/setup.py
index 1aea6e4..4abadc9 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@ with open("README.md") as f:
 
 setup(
     name='remotezip',
-    version='0.11.1',
+    version='0.12.0',
     author='Giuseppe Tribulato',
     author_email='gtsystem@gmail.com',
     py_modules=['remotezip'],


Test patch:
diff --git a/test_remotezip.py b/test_remotezip.py
index c45f41c..03e006d 100644
--- a/test_remotezip.py
+++ b/test_remotezip.py
@@ -42,6 +42,7 @@ class ServerSimulator:
         context.headers['Content-Range'] = rz.RemoteFetcher.build_range_header(init_pos, init_pos + len(content))
         return content
 
+
 class LocalFetcher(rz.RemoteFetcher):
     def fetch(self, data_range, stream=False):
         with open(self._url, 'rb') as f:
@@ -60,9 +61,9 @@ class LocalFetcher(rz.RemoteFetcher):
 
             f = io.BytesIO(f.read(range_max - range_min + 1))
             buff = rz.PartialBuffer(f, range_min, range_max - range_min + 1, stream=stream)
-            #buff = self._make_buffer(f, content_range, stream=stream)
         return buff
 
+
 class TestPartialBuffer(unittest.TestCase):
     def setUp(self):
         if not hasattr(self, 'assertRaisesRegex'):
@@ -288,6 +289,19 @@ class TestRemoteZip(unittest.TestCase):
             self.assertEqual(buffer.tell(), 10)
             self.assertEqual(buffer.read(3), b"abc")
 
+    def test_fetch_ending_unsupported_suffix(self):
+        # fetch file ending
+        expected_headers = {'Range': 'bytes=900-999'}
+        headers = {'Content-Range': 'Bytes 900-999/1000'}
+        with requests_mock.Mocker() as m:
+            m.head("http://test.com/file.zip", status_code=200, headers={'Content-Length': '1000'})
+            m.get("http://test.com/file.zip", content=b"abc", status_code=200, headers=headers,
+                  request_headers=expected_headers)
+            fetcher = rz.RemoteFetcher("http://test.com/file.zip", support_suffix_range=False)
+            buffer = fetcher.fetch((-100, None), stream=True)
+            self.assertEqual(buffer.tell(), 900)
+            self.assertEqual(buffer.read(3), b"abc")
+
     @staticmethod
     def make_zip_file(fname):
         with zipfile.ZipFile(fname, 'w', compression=zipfile.ZIP_DEFLATED) as zip: