Repository: fsspec/filesystem_spec

Issue:
`fsspec.open()` doesn't work with square brackets in a filename
For example, `fsspec.open("filename[test].csv")`.

Is this by design or a bug? (python builtin `open` function works in this case, at least for python 3.11)



Discussion:
martindurant: Well done, looks like you found it :)
Skylion007: @martindurant '*' and '?' aren't valid in Windows file systems... How should I modify the test suite?
martindurant: We have a WIN variable somewhere (`sys.platform == "nt"`, I think), so you can define a different set of characters in a simple if.
Skylion007: > We have a WIN variable somewhere (`sys.platform == "nt"`, I think), so you can define a different set of characters in a simple if.

True, but someone could be running tests locally on a Linux system mounted on NTFS or some other filesystem that doesn't support these chars and the tests would fail.
martindurant: I see what you mean, but I am not worried about that - it's really unlikely to have ntfs as the temp location.

Patch:
diff --git a/fsspec/core.py b/fsspec/core.py
index 8bc8e8e..c6263a6 100644
--- a/fsspec/core.py
+++ b/fsspec/core.py
@@ -643,7 +643,10 @@ def get_fs_token_paths(
     else:
         paths = fs._strip_protocol(paths)
     if isinstance(paths, (list, tuple, set)):
-        paths = expand_paths_if_needed(paths, mode, num, fs, name_function)
+        if expand:
+            paths = expand_paths_if_needed(paths, mode, num, fs, name_function)
+        elif not isinstance(paths, list):
+            paths = list(paths)
     else:
         if "w" in mode and expand:
             paths = _expand_paths(paths, name_function, num)


Test patch:
diff --git a/fsspec/tests/test_core.py b/fsspec/tests/test_core.py
index a14f688..943ab92 100644
--- a/fsspec/tests/test_core.py
+++ b/fsspec/tests/test_core.py
@@ -239,6 +239,101 @@ def test_pickle_after_open_open():
     of2.close()
 
 
+# Define a list of special glob characters.
+# Note that we need to escape some characters and also consider file system limitations.
+# '*' and '?' are excluded because they are not valid for many file systems.
+# Similarly, we're careful with '{', '}', and '@' as their special meaning is
+# context-specific and might not be considered special for filenames.
+# Add tests for more file systems and for more glob magic later
+glob_magic_characters = ["[", "]", "!"]
+if os.name != "nt":
+    glob_magic_characters.extend(("*", "?"))  # not valid on Windows
+
+
+@pytest.mark.parametrize("char", glob_magic_characters)
+def test_open_file_read_with_special_characters(tmp_path, char):
+    # Create a filename incorporating the special character
+    file_name = f"test{char}.txt"
+    file_path = tmp_path / file_name
+    expected_content = "Hello, world!"
+
+    with open(file_path, "w") as f:
+        f.write(expected_content)
+
+    with fsspec.open(file_path, "r") as f:
+        actual_content = f.read()
+
+    assert actual_content == expected_content
+
+
+@pytest.mark.parametrize("char", glob_magic_characters)
+def test_open_files_read_with_special_characters(tmp_path, char):
+    # Create a filename incorporating the special character
+    file_name = f"test{char}.txt"
+    file_path = tmp_path / file_name
+    expected_content = "Hello, world!"
+
+    with open(file_path, "w") as f:
+        f.write(expected_content)
+
+    with fsspec.open_files(file_path, "r")[0] as f:
+        actual_content = f.read()
+
+    assert actual_content == expected_content
+
+
+@pytest.mark.parametrize("char", glob_magic_characters)
+def test_open_file_write_with_special_characters(tmp_path, char):
+    # Create a filename incorporating the special character
+    file_name = f"test{char}.txt"
+    file_path = tmp_path / file_name
+    expected_content = "Hello, world!"
+
+    with fsspec.open(file_path, "w") as f:
+        f.write(expected_content)
+
+    with open(file_path, "r") as f:
+        actual_content = f.read()
+
+    assert actual_content == expected_content
+
+
+@pytest.mark.parametrize("char", glob_magic_characters)
+def test_open_files_read_with_special_characters(tmp_path, char):
+    # Create a filename incorporating the special character
+    file_name = f"test{char}.txt"
+    file_path = tmp_path / file_name
+    expected_content = "Hello, world!"
+
+    with open(file_path, "w") as f:
+        f.write(expected_content)
+
+    with fsspec.open_files(
+        urlpath=[os.fspath(file_path)], mode="r", auto_mkdir=False, expand=False
+    )[0] as f:
+        actual_content = f.read()
+
+    assert actual_content == expected_content
+
+
+@pytest.mark.parametrize("char", glob_magic_characters)
+def test_open_files_write_with_special_characters(tmp_path, char):
+    # Create a filename incorporating the special character
+    file_name = f"test{char}.txt"
+    file_path = tmp_path / file_name
+    expected_content = "Hello, world!"
+
+    with fsspec.open_files(
+        urlpath=[os.fspath(file_path)], mode="w", auto_mkdir=False, expand=False
+    )[0] as f:
+        f.write(expected_content)
+
+    with open(file_path, "r") as f:
+        actual_content = f.read()
+
+    assert actual_content == expected_content
+
+
 def test_mismatch():
     pytest.importorskip("s3fs")
     with pytest.raises(ValueError):