Repository: fsspec/filesystem_spec

Issue:
HTTPFileSystem.glob misses folders on fsspec>2023.10.0
Hello everyone,

Since #1382 was merged HTTPFileSystem's glob is missing folders for patterns ending in `"/*"`.

To reproduce:

```shell
mkdir share
mkdir share/folder1
touch share/file1.txt
python -m http.server -d share 8080
```

execute in another shell:

```console
# OLD BEHAVIOR
$ python -m venv venvA
$ source venvA/bin/activate
$ python -m pip install "fsspec[http]==2023.10.0"
$ python -c "import fsspec; print(fsspec.__version__); print(fsspec.filesystem('http').glob('http://127.0.0.1:8080/*'))"
2023.10.0
['http://127.0.0.1:8080/file1.txt', 'http://127.0.0.1:8080/folder1/']
```

```console
# CURRENT BEHAVIOR
$ python -m venv venvB
$ source venvB/bin/activate
$ python -m pip install "fsspec[http]>2023.10.0"
$ python -c "import fsspec; print(fsspec.__version__); print(fsspec.filesystem('http').glob('http://127.0.0.1:8080/*'))"
2023.12.2
['http://127.0.0.1:8080/file1.txt']
```

I'll open a PR with a fix right away.

Cheers,
Andreas


Discussion:
ap--: Looks like the python-3.10 and python-3.11 test failures are unrelated. The failing tests are for the smb filesystem.
martindurant: > Looks like the python-3.10 and python-3.11 test failures are unrelated. The failing tests are for the smb filesystem.

A long standing flakiness I have yet to track down.
ap--: I made the changes to use the existing `HTTPTestHandler` in `test/conftest.py` and cleaned them up and force pushed to this PR.

The first commit adds the failing test, and the second commit restores the glob behavior.

Patch:
diff --git a/fsspec/implementations/http.py b/fsspec/implementations/http.py
index 2ffb7ed..4580764 100644
--- a/fsspec/implementations/http.py
+++ b/fsspec/implementations/http.py
@@ -451,7 +451,7 @@ class HTTPFileSystem(AsyncFileSystem):
 
         ends_with_slash = path.endswith("/")  # _strip_protocol strips trailing slash
         path = self._strip_protocol(path)
-        append_slash_to_dirname = ends_with_slash or path.endswith("/**")
+        append_slash_to_dirname = ends_with_slash or path.endswith(("/**", "/*"))
         idx_star = path.find("*") if path.find("*") >= 0 else len(path)
         idx_brace = path.find("[") if path.find("[") >= 0 else len(path)
 
@@ -494,15 +494,15 @@ class HTTPFileSystem(AsyncFileSystem):
         pattern = re.compile(pattern)
 
         out = {
-            p: info
+            (
+                p.rstrip("/")
+                if not append_slash_to_dirname
+                and info["type"] == "directory"
+                and p.endswith("/")
+                else p
+            ): info
             for p, info in sorted(allpaths.items())
-            if pattern.match(
-                (
-                    p + "/"
-                    if append_slash_to_dirname and info["type"] == "directory"
-                    else p
-                )
-            )
+            if pattern.match(p.rstrip("/"))
         }
 
         if detail:


Test patch:
diff --git a/fsspec/implementations/tests/test_http.py b/fsspec/implementations/tests/test_http.py
index af11644..fdae51f 100644
--- a/fsspec/implementations/tests/test_http.py
+++ b/fsspec/implementations/tests/test_http.py
@@ -129,6 +129,15 @@ def test_list_cache_with_skip_instance_cache(server):
     assert out == [server + "/index/realfile"]
 
 
+def test_glob_return_subfolders(server):
+    h = fsspec.filesystem("http")
+    out = h.glob(server + "/simple/*")
+    assert set(out) == {
+        server + "/simple/dir/",
+        server + "/simple/file",
+    }
+
+
 def test_isdir(server):
     h = fsspec.filesystem("http")
     assert h.isdir(server + "/index/")
diff --git a/fsspec/tests/conftest.py b/fsspec/tests/conftest.py
index d305e21..fb1efb0 100644
--- a/fsspec/tests/conftest.py
+++ b/fsspec/tests/conftest.py
@@ -19,6 +19,13 @@ listing = open(
 win = os.name == "nt"
 
 
+def _make_listing(*paths):
+    return "\n".join(
+        f'<a href="http://127.0.0.1:{port}{f}">Link_{i}</a>'
+        for i, f in enumerate(paths)
+    ).encode()
+
+
 @pytest.fixture
 def reset_files():
     yield
@@ -34,6 +41,10 @@ class HTTPTestHandler(BaseHTTPRequestHandler):
         "/index/otherfile": data,
         "/index": index,
         "/data/20020401": listing,
+        "/simple/": _make_listing("/simple/file", "/simple/dir/"),
+        "/simple/file": data,
+        "/simple/dir/": _make_listing("/simple/dir/file"),
+        "/simple/dir/file": data,
     }
     dynamic_files = {}
 
@@ -53,7 +64,9 @@ class HTTPTestHandler(BaseHTTPRequestHandler):
             self.wfile.write(data)
 
     def do_GET(self):
-        file_path = self.path.rstrip("/")
+        file_path = self.path
+        if file_path.endswith("/") and file_path.rstrip("/") in self.files:
+            file_path = file_path.rstrip("/")
         file_data = self.files.get(file_path)
         if "give_path" in self.headers:
             return self._respond(200, data=json.dumps({"path": self.path}).encode())