Repository: scikit-hep/uproot5

Issue:
Cannot open ROOT over HTTPS
I cannot open a ROOT file over HTTPS with uproot4 while it works with uproot

```python
import uproot4
_ = uproot4.open('https://cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root')
```
It fails with
```
OSError: remote server does not support HTTP range requests
for URL https://cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root
```

Do you have any ideas what may goes wrong?

Full StackTrace:
```
OSError                                   Traceback (most recent call last)
<ipython-input-3-2fc8249c41b5> in <module>
----> 1 _ = uproot4.open('https://cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root')

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/reading.py in open(path, object_cache, array_cache, custom_classes, **options)
    134         )
    135 
--> 136     file = ReadOnlyFile(
    137         file_path,
    138         object_cache=object_cache,

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/reading.py in __init__(self, file_path, object_cache, array_cache, custom_classes, **options)
    565             self._fUUID_version,
    566             self._fUUID,
--> 567         ) = uproot4.source.cursor.Cursor(0).fields(
    568             self._begin_chunk, _file_header_fields_small, {}
    569         )

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/source/cursor.py in fields(self, chunk, format, context, move)
    193         if move:
    194             self._index = stop
--> 195         return format.unpack(chunk.get(start, stop, self, context))
    196 
    197     def field(self, chunk, format, context, move=True):

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/source/chunk.py in get(self, start, stop, cursor, context)
    360         already.
    361         """
--> 362         self.wait()
    363 
    364         if (start, stop) in self:

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/source/chunk.py in wait(self)
    314         """
    315         if self._raw_data is None:
--> 316             self._raw_data = numpy.frombuffer(self._future.result(), dtype=self._dtype)
    317             if len(self._raw_data) != self._stop - self._start:
    318                 raise OSError(

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/source/futures.py in result(self, timeout)
    124             return self._result
    125         else:
--> 126             delayed_raise(*self._excinfo)
    127 
    128     def _run(self):

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/source/futures.py in delayed_raise(exception_class, exception_value, traceback)
     44         exec("raise exception_class, exception_value, traceback")
     45     else:
---> 46         raise exception_value.with_traceback(traceback)
     47 
     48 

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/source/futures.py in _run(self, resource)
    281     def _run(self, resource):
    282         try:
--> 283             self._result = self._task(resource)
    284         except Exception:
    285             self._excinfo = sys.exc_info()

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/source/http.py in task(resource)
    199 
    200         def task(resource):
--> 201             return resource.get(connection, start, stop)
    202 
    203         return uproot4.source.futures.ResourceFuture(task)

~/anaconda3/envs/starterkit38/lib/python3.8/site-packages/uproot4/source/http.py in get(self, connection, start, stop)
    167         if response.status != 206:
    168             connection.close()
--> 169             raise OSError(
    170                 """remote server does not support HTTP range requests
    171 for URL {0}""".format(

OSError: remote server does not support HTTP range requests
for URL https://cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root

```


Discussion:
I haven't tested it, but this _might_ be related to #121.
In fact, you're right! Thanks! closing
Update: sorry, not quite
Actually, the opening works, but then it fails again (while working for `uproot`)

command: 
```uproot4.open('https://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root')['tree'].arrays()```


Error: 
```
OSError: found 0 of 156 expected headers in HTTP multipart
for URL https://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root
```

It looks like something with multipart HTTP requests was broken between v0.0.17 and v0.0.20
Reopened, as it seems to me that this was not entirely fixed. The snippet above still errors:
Actually, the opening works, but then it fails again (while working for `uproot`)

```
file = uproot4.open('https://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root')
_ = file['tree'].arrays()  # fails here
```

It fails "later" though than the fix I assume, so I've rearranged the example.

Error: 
```
OSError: found 0 of 156 expected headers in HTTP multipart
for URL https://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root
```

Darn: the test only checked

```python
data = f["tree/Y1"].array()
```

which works. I'm looking into the issue with multiple branches.

Resolving patch:
diff --git a/uproot4/source/http.py b/uproot4/source/http.py
--- a/uproot4/source/http.py
+++ b/uproot4/source/http.py
@@ -372,8 +372,13 @@ def handle_multipart(self, source, futures, results, response):
         Helper function for :py:meth:`~uproot4.source.http.HTTPResource.multifuture`
         to handle the multipart GET response.
         """
+        if hasattr(response, "readline"):
+            response_buffer = response
+        else:
+            response_buffer = _ResponseBuffer(response)
+
         for i in uproot4._util.range(len(futures)):
-            range_string, size = self.next_header(response)
+            range_string, size = self.next_header(response_buffer)
             if range_string is None:
                 raise OSError(
                     """found {0} of {1} expected headers in HTTP multipart
@@ -397,7 +402,7 @@ def handle_multipart(self, source, futures, results, response):
                 )
 
             length = stop - start
-            results[start, stop] = response.read(length)
+            results[start, stop] = response_buffer.read(length)
 
             if len(results[start, stop]) != length:
                 raise OSError(
@@ -413,12 +418,12 @@ def handle_multipart(self, source, futures, results, response):
 
             future._run(self)
 
-    def next_header(self, response):
+    def next_header(self, response_buffer):
         """
         Helper function for :py:meth:`~uproot4.source.http.HTTPResource.multifuture`
-        to return the next header from the ``response``.
+        to return the next header from the ``response_buffer``.
         """
-        line = response.fp.readline()
+        line = response_buffer.readline()
         range_string, size = None, None
         while range_string is None:
             m = self._content_range_size.match(line)
@@ -430,7 +435,7 @@ def next_header(self, response):
                 if m is not None:
                     range_string = m.group(1)
                     size = None
-            line = response.fp.readline()
+            line = response_buffer.readline()
             if len(line.strip()) == 0:
                 break
         return range_string, size
@@ -452,6 +457,39 @@ def task(resource):
         return uproot4.source.futures.ResourceFuture(task)
 
 
+class _ResponseBuffer(object):
+    CHUNK = 1024
+
+    def __init__(self, stream):
+        self.already_read = b""
+        self.stream = stream
+
+    def read(self, length):
+        if length < len(self.already_read):
+            out = self.already_read[:length]
+            self.already_read = self.already_read[length:]
+            return out
+
+        elif len(self.already_read) > 0:
+            out = self.already_read
+            self.already_read = b""
+            return out + self.stream.read(length - len(out))
+
+        else:
+            return self.stream.read(length)
+
+    def readline(self):
+        while True:
+            try:
+                index = self.already_read.index(b"\n")
+            except ValueError:
+                self.already_read = self.already_read + self.stream.read(self.CHUNK)
+            else:
+                out = self.already_read[: index + 1]
+                self.already_read = self.already_read[index + 1 :]
+                return out
+
+
 class HTTPSource(uproot4.source.chunk.Source):
     """
     Args:


Test patch:
diff --git a/tests/test_0001-source-class.py b/tests/test_0001-source-class.py
--- a/tests/test_0001-source-class.py
+++ b/tests/test_0001-source-class.py
@@ -169,14 +169,14 @@ def test_http_port():
 @pytest.mark.network
 def test_http_size():
     with uproot4.source.http.HTTPSource(
-        "https://scikit-hep.org/uproot/examples/Zmumu.root",
+        "https://scikit-hep.org/uproot3/examples/Zmumu.root",
         timeout=10,
         num_fallback_workers=1,
     ) as source:
         size1 = source.num_bytes
 
     with uproot4.source.http.MultithreadedHTTPSource(
-        "https://scikit-hep.org/uproot/examples/Zmumu.root", num_workers=1, timeout=10
+        "https://scikit-hep.org/uproot3/examples/Zmumu.root", num_workers=1, timeout=10
     ) as source:
         size2 = source.num_bytes
 
@@ -186,14 +186,14 @@ def test_http_size():
 @pytest.mark.network
 def test_http_size_port():
     with uproot4.source.http.HTTPSource(
-        "https://scikit-hep.org:443/uproot/examples/Zmumu.root",
+        "https://scikit-hep.org:443/uproot3/examples/Zmumu.root",
         timeout=10,
         num_fallback_workers=1,
     ) as source:
         size1 = source.num_bytes
 
     with uproot4.source.http.MultithreadedHTTPSource(
-        "https://scikit-hep.org:443/uproot/examples/Zmumu.root",
+        "https://scikit-hep.org:443/uproot3/examples/Zmumu.root",
         num_workers=1,
         timeout=10,
     ) as source:
@@ -217,7 +217,7 @@ def test_http_fail():
 def test_no_multipart():
     for num_workers in [1, 2]:
         with uproot4.source.http.MultithreadedHTTPSource(
-            "https://scikit-hep.org/uproot/examples/Zmumu.root",
+            "https://scikit-hep.org/uproot3/examples/Zmumu.root",
             num_workers=num_workers,
             timeout=10,
         ) as source:
@@ -246,7 +246,7 @@ def test_no_multipart_fail():
 def test_fallback():
     for num_workers in [1, 2]:
         with uproot4.source.http.HTTPSource(
-            "https://scikit-hep.org/uproot/examples/Zmumu.root",
+            "https://scikit-hep.org/uproot3/examples/Zmumu.root",
             timeout=10,
             num_fallback_workers=num_workers,
         ) as source:
diff --git a/tests/test_0006-notify-when-downloaded.py b/tests/test_0006-notify-when-downloaded.py
--- a/tests/test_0006-notify-when-downloaded.py
+++ b/tests/test_0006-notify-when-downloaded.py
@@ -126,7 +126,7 @@ def test_http_workers():
 def test_http_fallback():
     notifications = queue.Queue()
     with uproot4.source.http.HTTPSource(
-        "https://scikit-hep.org/uproot/examples/Zmumu.root",
+        "https://scikit-hep.org/uproot3/examples/Zmumu.root",
         timeout=10,
         num_fallback_workers=1,
     ) as source:
@@ -143,7 +143,7 @@ def test_http_fallback():
 def test_http_fallback_workers():
     notifications = queue.Queue()
     with uproot4.source.http.HTTPSource(
-        "https://scikit-hep.org/uproot/examples/Zmumu.root",
+        "https://scikit-hep.org/uproot3/examples/Zmumu.root",
         timeout=10,
         num_fallback_workers=5,
     ) as source:
diff --git a/tests/test_0045-lazy-arrays-1.py b/tests/test_0045-lazy-arrays-1.py
--- a/tests/test_0045-lazy-arrays-1.py
+++ b/tests/test_0045-lazy-arrays-1.py
@@ -151,7 +151,8 @@ def test_awkward_pluralization():
 
 
 def test_lazy_called_on_nonexistent_file():
+    awkward1 = pytest.importorskip("awkward1")
     filename = "nonexistent_file.root"
-    with pytest.raises(FileNotFoundError) as excinfo:
+    with pytest.raises(uproot4._util._FileNotFoundError) as excinfo:
         uproot4.lazy(filename)
     assert filename in str(excinfo.value)
diff --git a/tests/test_0066-fix-http-fallback-freeze.py b/tests/test_0066-fix-http-fallback-freeze.py
--- a/tests/test_0066-fix-http-fallback-freeze.py
+++ b/tests/test_0066-fix-http-fallback-freeze.py
@@ -11,7 +11,7 @@
 @pytest.mark.network
 def test():
     with uproot4.open(
-        {"http://scikit-hep.org/uproot/examples/HZZ.root": "events"}
+        {"http://scikit-hep.org/uproot3/examples/HZZ.root": "events"}
     ) as t:
-        t["MET_px"].array()
-        t["MET_py"].array()
+        t["MET_px"].array(library="np")
+        t["MET_py"].array(library="np")
diff --git a/tests/test_0088-read-with-http.py b/tests/test_0088-read-with-http.py
--- a/tests/test_0088-read-with-http.py
+++ b/tests/test_0088-read-with-http.py
@@ -16,6 +16,16 @@ def test_issue176():
         assert len(data) == 100000
 
 
+@pytest.mark.network
+def test_issue176_again():
+    with uproot4.open(
+        "https://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root"
+    ) as f:
+        data = f["tree"].arrays(["Y1", "Y2"], library="np")
+        assert len(data["Y1"]) == 100000
+        assert len(data["Y2"]) == 100000
+
+
 @pytest.mark.network
 def test_issue121():
     with uproot4.open(
diff --git a/tests/test_0172-allow-allocators-in-vector-typenames.py b/tests/test_0172-allow-allocators-in-vector-typenames.py
--- a/tests/test_0172-allow-allocators-in-vector-typenames.py
+++ b/tests/test_0172-allow-allocators-in-vector-typenames.py
@@ -15,4 +15,4 @@ def test():
         assert (
             t["rec_part_px_VecOps"].typename == "std::vector<float>"
         )  # without the allocator
-        t["rec_part_px_VecOps"].array()
+        t["rec_part_px_VecOps"].array(library="np")
diff --git a/tests/test_0173-empty-and-multiprocessing-bugs.py b/tests/test_0173-empty-and-multiprocessing-bugs.py
--- a/tests/test_0173-empty-and-multiprocessing-bugs.py
+++ b/tests/test_0173-empty-and-multiprocessing-bugs.py
@@ -23,7 +23,7 @@ def readone(filename):
         f.decompression_executor = uproot4.ThreadPoolExecutor()
         t = f["events"]
         b = t["px1"]
-        b.array()
+        b.array(library="np")
 
 
 def test_multiprocessing():
diff --git a/tests/test_0182-complain-about-missing-files.py b/tests/test_0182-complain-about-missing-files.py
--- a/tests/test_0182-complain-about-missing-files.py
+++ b/tests/test_0182-complain-about-missing-files.py
@@ -15,9 +15,12 @@ def test():
     bad = one.replace(".root", "-DOES-NOT-EXIST.root")
     okay = one.replace(".root", "-DOES-NOT-EXIST-*.root")
 
-    assert len(list(uproot4.iterate([one, two], step_size="1 TB"))) == 2
+    assert len(list(uproot4.iterate([one, two], step_size="1 TB", library="np"))) == 2
 
     with pytest.raises(uproot4._util._FileNotFoundError):
-        list(uproot4.iterate([one, two, bad]))
+        list(uproot4.iterate([one, two, bad], library="np"))
 
-    assert len(list(uproot4.iterate([one, two, okay], step_size="1 TB"))) == 2
+    assert (
+        len(list(uproot4.iterate([one, two, okay], step_size="1 TB", library="np")))
+        == 2
+    )