Repository: fsspec/filesystem_spec

Issue:
`auto_mkdir` broken in `setitems`
When writing array data to a nested path, I get a FileNotFoundError for the file that is supposed to be written. This is because the parent directory does not exist, althoug fsspec should create it when needed. This is especially a problem with [`zarr`](https://github.com/zarr-developers/zarr-python/) (e.g. 2.16.1) which [does something](https://github.com/zarr-developers/zarr-python/blame/4944e66cd847a6ab5ec3a70c2b7bc0973f707bd6/zarr/storage.py#L1441) like the following:

```python
import fsspec

mapper = fsspec.get_mapper("/tmp/deleteme.zarr", auto_mkdir=True)
mapper.setitems({"/tmp/example.zarr/s0/0/0": bytes([0, 100])})

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "…/lib/python3.9/site-packages/fsspec/spec.py", line 824, in pipe
    self.pipe_file(self._strip_protocol(k), v, **kwargs)
  File "…/lib/python3.9/site-packages/fsspec/spec.py", line 803, in pipe_file
    with self.open(path, "wb", **kwargs) as f:
  File "…/lib/python3.9/site-packages/fsspec/spec.py", line 1309, in open
    f = self._open(
  File "…/lib/python3.9/site-packages/fsspec/implementations/local.py", line 180, in _open
    return LocalFileOpener(path, mode, fs=self, **kwargs)
  File "…/lib/python3.9/site-packages/fsspec/implementations/local.py", line 298, in __init__
    self._open()
  File "…/lib/python3.9/site-packages/fsspec/implementations/local.py", line 303, in _open
    self.f = open(self.path, mode=self.mode)
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/example.zarr/s0/0/0'
```

- The alternative `mapper["/tmp/example.zarr/s0/0/0"] = bytes([0, 100])` still creates the directories.
- This broke with `fsspec==2023.9.0` but works with `fsspec==2023.6.0`


Discussion:

Patch:
diff --git a/fsspec/core.py b/fsspec/core.py
index 890ca81..6f3652c 100644
--- a/fsspec/core.py
+++ b/fsspec/core.py
@@ -210,7 +210,6 @@ def open_files(
     num=1,
     protocol=None,
     newline=None,
-    auto_mkdir=True,
     expand=True,
     **kwargs,
 ):
@@ -249,9 +248,6 @@ def open_files(
     newline: bytes or None
         Used for line terminator in text mode. If None, uses system default;
         if blank, uses no translation.
-    auto_mkdir: bool (True)
-        If in write mode, this will ensure the target directory exists before
-        writing, by calling ``fs.mkdirs(exist_ok=True)``.
     expand: bool
     **kwargs: dict
         Extra options that make sense to a particular storage connection, e.g.
@@ -288,9 +284,6 @@ def open_files(
         protocol=protocol,
         expand=expand,
     )
-    if "r" not in mode and auto_mkdir:
-        parents = {fs._parent(path) for path in paths}
-        [fs.makedirs(parent, exist_ok=True) for parent in parents]
     return OpenFiles(
         [
             OpenFile(
@@ -363,7 +356,6 @@ def url_to_fs(url, **kwargs):
     # non-FS arguments that appear in fsspec.open()
     # inspect could keep this in sync with open()'s signature
     known_kwargs = {
-        "auto_mkdir",
         "compression",
         "encoding",
         "errors",


Test patch:
diff --git a/fsspec/tests/test_core.py b/fsspec/tests/test_core.py
index c786643..e2d4fb9 100644
--- a/fsspec/tests/test_core.py
+++ b/fsspec/tests/test_core.py
@@ -150,10 +150,10 @@ def test_pathobject(tmpdir):
 
 def test_automkdir(tmpdir):
     dir = os.path.join(str(tmpdir), "a")
-    of = fsspec.open(os.path.join(dir, "afile"), "w")
-    with of:
-        pass
-    assert "afile" in os.listdir(dir)
+    of = fsspec.open(os.path.join(dir, "afile"), "w", auto_mkdir=False)
+    with pytest.raises(IOError):
+        with of:
+            pass
 
     dir = os.path.join(str(tmpdir), "b")
     of = fsspec.open(os.path.join(dir, "bfile"), "w", auto_mkdir=True)
@@ -306,3 +306,8 @@ def test_chained_url(ftp_writable):
         url += f"::ftp://{username}:{password}@{host}:{port}/archive.zip"
         with fsspec.open(url, "rb") as f:
             assert f.read() == data["afile"]
+
+
+def test_automkdir_local():
+    fs, _ = fsspec.core.url_to_fs("file://", auto_mkdir=True)
+    assert fs.auto_mkdir is True