Repository: gtsystem/python-remotezip

Issue:
Get 416 Client Error on some zip files
remotezip raising `remotezip.RemoteIOError` on some files. For example, http://0x0.st/o5Pa.apk (it's unavailable for now, so I re-uploaded it to https://transfer.sh/get/rndzvr/test.apk)
Also uploaded it to GitHub as zip archive: [test.zip](https://github.com/gtsystem/python-remotezip/files/10267095/test.zip)
```
$ remotezip http://0x0.st/o5Pa.apk AndroidManifest.xml
Extracting AndroidManifest.xml...
Traceback (most recent call last):
  File "/.../venv/lib/python3.10/site-packages/remotezip.py", line 193, in fetch_fun
    res, headers = self.request(self.url, range_header, kwargs, self.session)
  File "/.../venv/lib/python3.10/site-packages/remotezip.py", line 184, in request
    res.raise_for_status()
  File "/.../venv/lib/python3.10/site-packages/requests/models.py", line 1021, in raise_for_status
    raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 416 Client Error: Requested Range Not Satisfiable for url: http://0x0.st/o5Pa.apk

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/.../venv/bin/remotezip", line 43, in <module>
    extract_files(args.url, args.filename, args.dir)
  File "/.../venv/bin/remotezip", line 26, in extract_files
    zip.extract(fname, path=path)
  File "/usr/lib/python3.10/zipfile.py", line 1628, in extract
    return self._extract_member(member, path, pwd)
  File "/usr/lib/python3.10/zipfile.py", line 1698, in _extract_member
    with self.open(member, pwd=pwd) as source, \
  File "/usr/lib/python3.10/zipfile.py", line 1530, in open
    fheader = zef_file.read(sizeFileHeader)
  File "/usr/lib/python3.10/zipfile.py", line 745, in read
    data = self._file.read(n)
  File "/.../venv/lib/python3.10/site-packages/remotezip.py", line 112, in read
    self.buffer = self.fetch_fun((self.buffer.position, self.buffer.position + fetch_size -1), stream=stream)
  File "/.../venv/lib/python3.10/site-packages/remotezip.py", line 196, in fetch_fun
    raise RemoteIOError(str(e))
remotezip.RemoteIOError: 416 Client Error: Requested Range Not Satisfiable for url: http://0x0.st/o5Pa.apk
```
I also tested this file with [PartialZipBrowser](https://github.com/tihmstar/partialZipBrowser), and it works fine:
```
$ pzb -g AndroidManifest.xml https://0x0.st/o5Pa.apk
Version: 9bfdde2b2456181045f74631683fba491d8bf4f2 - 38
libfragmentzip version: 0.64-aaf6fae83a0aa6f7aae1c94721857076d04a14e8-RELEASE
init pzb: https://0x0.st/o5Pa.apk
init done
getting: AndroidManifest.xml
100% [===================================================================================================>]
download succeeded
```

It looks like byte range calculation bug in remotezip.
I also sniffed traffic, generated by remotezip and PartialZibBrowser while downloading this file, in mitmproxy, this may help you.
1. remotezip
![image](https://user-images.githubusercontent.com/43933400/208459898-d62dd2ad-1fd3-40d8-85c5-df2416a06144.png)
2. PartialZipBrowser:
![image](https://user-images.githubusercontent.com/43933400/208460225-12c31661-f2ed-44c7-b412-de6ba7bc2aca.png)


Discussion:

Patch:
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index cd0191b..c56f2c7 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -12,7 +12,7 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python-version: ["2.7", "3.6", "3.9", "3.10"]
diff --git a/remotezip.py b/remotezip.py
index cf82595..cfe6924 100644
--- a/remotezip.py
+++ b/remotezip.py
@@ -1,5 +1,6 @@
 import io
 import zipfile
+from itertools import tee
 
 import requests
 
@@ -208,6 +209,13 @@ class RemoteFetcher:
             raise RemoteIOError(str(e))
 
 
+def pairwise(iterable):
+    # pairwise('ABCDEFG') --> AB BC CD DE EF FG
+    a, b = tee(iterable)
+    next(b, None)
+    return zip(a, b)
+
+
 class RemoteZip(zipfile.ZipFile):
     def __init__(self, url, initial_buffer_size=64*1024, session=None, fetcher=RemoteFetcher, **kwargs):
         fetcher = fetcher(url, session, **kwargs)
@@ -216,15 +224,12 @@ class RemoteZip(zipfile.ZipFile):
         rio.set_position_to_size(self._get_position_to_size())
 
     def _get_position_to_size(self):
-        ilist = self.infolist()
+        ilist = [info.header_offset for info in self.infolist()]
         if len(ilist) == 0:
             return {}
+        ilist.sort()
+        ilist.append(self.start_dir)
+        return {a: b-a for a, b in pairwise(ilist)}
 
-        position_to_size = {ilist[-1].header_offset: self.start_dir - ilist[-1].header_offset}
-        for i in range(len(ilist) - 1):
-            m1, m2 = ilist[i: i+2]
-            position_to_size[m1.header_offset] = m2.header_offset - m1.header_offset
-
-        return position_to_size
 
 
diff --git a/setup.py b/setup.py
index dea5194..1aea6e4 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@ with open("README.md") as f:
 
 setup(
     name='remotezip',
-    version='0.11.0',
+    version='0.11.1',
     author='Giuseppe Tribulato',
     author_email='gtsystem@gmail.com',
     py_modules=['remotezip'],


Test patch:
diff --git a/test_remotezip.py b/test_remotezip.py
index 317a57c..c45f41c 100644
--- a/test_remotezip.py
+++ b/test_remotezip.py
@@ -238,6 +238,32 @@ class TestRemoteZip(unittest.TestCase):
 
                 self.assertIsNone(zfile.testzip())
 
+    @staticmethod
+    def make_unordered_zip_file(fname):
+        with zipfile.ZipFile(fname, 'w') as zip:
+            zip.writestr("fileA", "A" * 300000 + 'Z')
+            zip.writestr("fileB", "B" * 10000 + 'Z')
+            zip.writestr("fileC", "C" * 100000 + 'Z')
+            info_list = zip.infolist()
+            info_list[0], info_list[1] = info_list[1], info_list[0]
+
+    def test_unordered_fileinfo(self):
+        """Test that zip file with unordered fileinfo records works as well. Fix #13."""
+        with TmpDir() as dire:
+            fname = os.path.join(dire, 'test.zip')
+            self.make_unordered_zip_file(fname)
+
+            with rz.RemoteZip(fname, fetcher=LocalFetcher) as zfile:
+                names = zfile.namelist()
+                self.assertEqual(names, ['fileB', 'fileA', 'fileC'])
+                with zfile.open('fileB', 'r') as f:
+                    self.assertEqual(f.read(), b"B" * 10000 + b'Z')
+                with zfile.open('fileA', 'r') as f:
+                    self.assertEqual(f.read(), b"A" * 300000 + b'Z')
+                with zfile.open('fileC', 'r') as f:
+                    self.assertEqual(f.read(), b"C" * 100000 + b'Z')
+                self.assertIsNone(zfile.testzip())
+
     def test_fetch_part(self):
         # fetch a range
         expected_headers = {'Range': 'bytes=10-20'}