Repository: fsspec/filesystem_spec (Python)

Issue:
`CachingFileSystem` with http vomits on `__del__` under fsspec 2022.10.0
As of the release of fsspec 2022.10.0 yesterday, the following code:

```python
import fsspec
from fsspec.implementations.cached import CachingFileSystem

fs = CachingFileSystem(
    fs=fsspec.filesystem("http"),
    cache_storage="cache01",
)

with fs.open("https://httpbin.org/uuid", "r", encoding="utf-8") as f:
    print(f.read())
```

outputs the following traceback on termination:

```
Traceback (most recent call last):
  File "/Users/jwodder/work/dev/tmp/fsspec/fsspec-cache01a.py", line 10, in <module>
    with fs.open("https://httpbin.org/uuid", "r", encoding="utf-8") as f:
  File "/Users/jwodder/.local/virtualenvwrapper/venvs/tmp-aaf3a4de95a14b1/lib/python3.10/site-packages/fsspec/implementations/cached.py", line 344, in <lambda>
    f.close = lambda: self.close_and_update(f, close)
  File "/Users/jwodder/.local/virtualenvwrapper/venvs/tmp-aaf3a4de95a14b1/lib/python3.10/site-packages/fsspec/implementations/cached.py", line 406, in <lambda>
    return lambda *args, **kw: getattr(type(self), item).__get__(self)(
  File "/Users/jwodder/.local/virtualenvwrapper/venvs/tmp-aaf3a4de95a14b1/lib/python3.10/site-packages/fsspec/implementations/cached.py", line 357, in close_and_update
    c = self.cached_files[-1][path]
KeyError: '/uuid'
Exception ignored in: <function AbstractBufferedFile.__del__ at 0x10ddf55a0>
Traceback (most recent call last):
  File "/Users/jwodder/.local/virtualenvwrapper/venvs/tmp-aaf3a4de95a14b1/lib/python3.10/site-packages/fsspec/spec.py", line 1751, in __del__
  File "/Users/jwodder/.local/virtualenvwrapper/venvs/tmp-aaf3a4de95a14b1/lib/python3.10/site-packages/fsspec/implementations/cached.py", line 344, in <lambda>
  File "/Users/jwodder/.local/virtualenvwrapper/venvs/tmp-aaf3a4de95a14b1/lib/python3.10/site-packages/fsspec/implementations/cached.py", line 406, in <lambda>
  File "/Users/jwodder/.local/virtualenvwrapper/venvs/tmp-aaf3a4de95a14b1/lib/python3.10/site-packages/fsspec/implementations/cached.py", line 357, in close_and_update
KeyError: '/uuid'
```

Inserting a `print` statement in `WholeFileCacheFileSystem.close_and_update()` reveals that the keys in `self.cached_files[-1]` are URLs (i.e., `'https://httpbin.org/uuid'`) rather than the `/uuid` paths that the code expects.
KeyError: '/blobs/f5f/175/f5f1752f-5227-47d5-8f75-cd71937878aa'
**Context:**
I'm running a Jupyter Notebook in a conda environment that installs the latest fsspec version. After setting up an fsspec caching filesystem, I use code from [pynwb](https://github.com/NeurodataWithoutBorders/pynwb) which reads in a file remotely via a url. This causes an runtime error with fsspec.

**Code:**
```
fs = CachingFileSystem(
    fs=fsspec.filesystem("http"),
    cache_storage="nwb-cache",  # Local folder for the cache
)

# next, open the file
with fs.open(file_url, "rb") as f:
    with h5py.File(f) as file:
        with pynwb.NWBHDF5IO(file=file, mode='r', load_namespaces=True) as io:
            nwb = io.read()
```

**Traceback:**
```
KeyError                                  Traceback (most recent call last)
Input In [6], in <cell line: 7>()
      8 with h5py.File(f) as file:
      9     with pynwb.NWBHDF5IO(file=file, mode='r', load_namespaces=True) as io:
---> 10         nwb = io.read()

File ~\Miniconda3\envs\test_env\lib\site-packages\fsspec\spec.py:1762, in AbstractBufferedFile.__exit__(self, *args)
   1761 def __exit__(self, *args):
-> 1762     self.close()

File ~\Miniconda3\envs\test_env\lib\site-packages\fsspec\implementations\cached.py:344, in CachingFileSystem._open.<locals>.<lambda>()
    342 f.cache = MMapCache(f.blocksize, f._fetch_range, f.size, fn, blocks)
    343 close = f.close
--> 344 f.close = lambda: self.close_and_update(f, close)
    345 self.save_cache()
    346 return f

File ~\Miniconda3\envs\test_env\lib\site-packages\fsspec\implementations\cached.py:406, in CachingFileSystem.__getattribute__.<locals>.<lambda>(*args, **kw)
    371 def __getattribute__(self, item):
    372     if item in [
    373         "load_cache",
    374         "_open",
   (...)
    404         # all the methods defined in this class. Note `open` here, since
    405         # it calls `_open`, but is actually in superclass
--> 406         return lambda *args, **kw: getattr(type(self), item).__get__(self)(
    407             *args, **kw
    408         )
    409     if item in ["__reduce_ex__"]:
    410         raise AttributeError

File ~\Miniconda3\envs\test_env\lib\site-packages\fsspec\implementations\cached.py:357, in CachingFileSystem.close_and_update(self, f, close)
    354     return
    355 path = self._strip_protocol(f.path)
--> 357 c = self.cached_files[-1][path]
    358 if c["blocks"] is not True and len(c["blocks"]) * f.blocksize >= f.size:
    359     c["blocks"] = True

KeyError: '/blobs/f5f/175/f5f1752f-5227-47d5-8f75-cd71937878aa'
```


Discussion:
This appears required but not sufficient to fix the problem:
```diff
--- a/fsspec/implementations/http.py
+++ b/fsspec/implementations/http.py
@@ -133,7 +133,7 @@ class HTTPFileSystem(AsyncFileSystem):
     @classmethod
     def _strip_protocol(cls, path):
         """For HTTP, we always want to keep the full URL"""
-        return path
+        return str(path)

```
(because paths are now URL class, not simple string; this should maybe be fixed on the `.path` attribute of the file instance)

I tried to run something like this, but got
```
File ~/conda/envs/py39/lib/python3.9/site-packages/hdmf/utils.py:450, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    448     if parse_err:
    449         msg = '%s: %s' % (func.__qualname__, ', '.join(parse_err))
--> 450         raise ExceptionType(msg)
    452 if is_method:
    453     return func(args[0], **parsed['args'])

TypeError: NWBHDF5IO.__init__: missing argument 'path'
```
within pynwb; however, `fs.cached_files[-1]` *did* get populated. Can you check what yours contains?
This is an error which comes from an older version of pynwb. Should be fixed in the newest release.
For me, fs.cached_files[-1] contained 3 objects whose keys are secure urls.
Looks like this might be the same issue here #1078 
That link doesn't seem to work, perhaps you meant #1078. You mean you do have keys, but they don't match what the filesystem is looking for? It is specifically the "http://" part that is missing, then?
The urls I mentioned do contain `https://`. I think, according to @jwodder, they should not contain this.
Sorry, I have so far failed to reproduce 
```
In [38]: with fs.open("http://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc", "rb") as f:
    ...:     f.read(4)
    ...:
2022-10-25 13:11:41,016 - fsspec - DEBUG - _open -- Creating local sparse file for http://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc
2022-10-25 13:11:41,016 - fsspec.http - DEBUG - _file_info -- Retrieve file size for http://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc
2022-10-25 13:11:41,420 - fsspec - DEBUG - read -- <File-like object HTTPFileSystem, http://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc> read: 0 - 4
2022-10-25 13:11:41,420 - fsspec - DEBUG - _fetch -- MMap cache fetching 0-4
2022-10-25 13:11:41,420 - fsspec - DEBUG - _fetch -- MMap get block #0 (0-2767916
2022-10-25 13:11:41,420 - fsspec.http - DEBUG - async_fetch_range -- Fetch range for <File-like object HTTPFileSystem, http://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc>: 0-2767916
2022-10-25 13:11:41,421 - fsspec.http - DEBUG - async_fetch_range -- http://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc : bytes=0-2767915
2022-10-25 13:11:43,464 - fsspec - DEBUG - close_and_update -- going to save
2022-10-25 13:11:43,466 - fsspec - DEBUG - close_and_update -- saved

In [39]: with fs.open("http://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc", "rb") as f:
    ...:     f.read(4)
    ...:
2022-10-25 13:12:14,134 - fsspec - DEBUG - _open -- Opening local copy of http://www.unidata.ucar.edu/software/netcdf/examples/sresa1b_ncar_ccsm3-example.nc
```

Is there any chance you can make a simpler reproducer without pynwb/h5py?
The file is closed at the end of the `with` block, which calls close_and_update. The entry in `.cached_files[-1]` goes to `"blocks": True`.
The other issue demonstrates a simpler case. I wonder if it may be related to inconsistent usage of the `_strip_protocol` method.
Indeed, that seems to be right. I've tried a couple of things, but no good so far



Patch:
diff --git a/ci/environment-win.yml b/ci/environment-win.yml
index db1e6cfb7..1b72f97fd 100644
--- a/ci/environment-win.yml
+++ b/ci/environment-win.yml
@@ -18,6 +18,7 @@ dependencies:
   - pytest-mock
   - pytest-vcr
   - python-libarchive-c
+  - py
   - numpy
   - nomkl
   - tqdm
diff --git a/fsspec/implementations/cached.py b/fsspec/implementations/cached.py
index 527e3eaf4..a0799a007 100644
--- a/fsspec/implementations/cached.py
+++ b/fsspec/implementations/cached.py
@@ -15,7 +15,7 @@
 from fsspec.spec import AbstractBufferedFile
 from fsspec.utils import infer_compression
 
-logger = logging.getLogger("fsspec")
+logger = logging.getLogger("fsspec.cached")
 
 
 class CachingFileSystem(AbstractFileSystem):
diff --git a/fsspec/implementations/http.py b/fsspec/implementations/http.py
index a2b49a4c9..2572f7563 100644
--- a/fsspec/implementations/http.py
+++ b/fsspec/implementations/http.py
@@ -348,7 +348,7 @@ def _open(
         if block_size and size:
             return HTTPFile(
                 self,
-                self.encode_url(path),
+                path,
                 session=session,
                 block_size=block_size,
                 mode=mode,
@@ -361,7 +361,7 @@ def _open(
         else:
             return HTTPStreamFile(
                 self,
-                self.encode_url(path),
+                path,
                 mode=mode,
                 loop=self.loop,
                 session=session,
@@ -377,7 +377,7 @@ async def open_async(self, path, mode="rb", size=None, **kwargs):
                 pass
         return AsyncStreamFile(
             self,
-            self.encode_url(path),
+            path,
             loop=self.loop,
             session=session,
             size=size,
@@ -597,7 +597,7 @@ async def async_fetch_all(self):
         """
         logger.debug(f"Fetch all for {self}")
         if not isinstance(self.cache, AllBytes):
-            r = await self.session.get(self.url, **self.kwargs)
+            r = await self.session.get(self.fs.encode_url(self.url), **self.kwargs)
             async with r:
                 r.raise_for_status()
                 out = await r.read()
@@ -621,7 +621,9 @@ async def async_fetch_range(self, start, end):
         headers = kwargs.pop("headers", {}).copy()
         headers["Range"] = "bytes=%i-%i" % (start, end - 1)
         logger.debug(str(self.url) + " : " + headers["Range"])
-        r = await self.session.get(self.url, headers=headers, **kwargs)
+        r = await self.session.get(
+            self.fs.encode_url(self.url), headers=headers, **kwargs
+        )
         async with r:
             if r.status == 416:
                 # range request outside file
@@ -690,7 +692,7 @@ def __init__(self, fs, url, mode="rb", loop=None, session=None, **kwargs):
         super().__init__(fs=fs, path=url, mode=mode, cache_type="none", **kwargs)
 
         async def cor():
-            r = await self.session.get(url, **kwargs).__aenter__()
+            r = await self.session.get(self.fs.encode_url(url), **kwargs).__aenter__()
             self.fs._raise_not_found_for_status(r, url)
             return r
 
@@ -733,7 +735,9 @@ def __init__(
 
     async def read(self, num=-1):
         if self.r is None:
-            r = await self.session.get(self.url, **self.kwargs).__aenter__()
+            r = await self.session.get(
+                self.fs.encode_url(self.url), **self.kwargs
+            ).__aenter__()
             self.fs._raise_not_found_for_status(r, self.url)
             self.r = r
         out = await self.r.content.read(num)
diff --git a/tox.ini b/tox.ini
index 09abb6cfc..f831681c6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -30,6 +30,7 @@ conda_deps=
     pytest-cov
     pytest-mock
     pytest-vcr
+    py
     fusepy
     tomli < 2
     msgpack-python


Test patch:
diff --git a/fsspec/implementations/tests/test_http.py b/fsspec/implementations/tests/test_http.py
index 9bb0fd9d6..bb75182c1 100644
--- a/fsspec/implementations/tests/test_http.py
+++ b/fsspec/implementations/tests/test_http.py
@@ -452,3 +452,12 @@ def test_encoded(server):
     fs = fsspec.filesystem("http", encoded=False)
     out = fs.cat(server + "/Hello: Günter", headers={"give_path": "true"})
     assert json.loads(out)["path"] == "/Hello:%20G%C3%BCnter"
+
+
+def test_with_cache(server):
+    fs = fsspec.filesystem("http", headers={"head_ok": "true", "give_length": "true"})
+    fn = server + "/index/realfile"
+    fs1 = fsspec.filesystem("blockcache", fs=fs)
+    with fs1.open(fn, "rb") as f:
+        out = f.read()
+    assert out == fs1.cat(fn)