Repository: pytest-dev/pyfakefs (2023-11-25 ~ 2024-03-08)

problem_statement: FakeOsModule.walk return a (Path, list, list) generator instead of a (str, list, list) when a pathlib object is used as input
**Describe the bug**
method FakeOsModule.walk of <pyfakefs.fake_os.FakeOsModule> is returning a Path object as first element instead of a string as the os.walk method do.

**How To Reproduce**
```python
from pathlib import Path

# Real os part
import os
os_output = list(os.walk(Path(os.getcwd())))  # Deepest element is a tuple of [str, list, list]
# Example ("C:\\", [], [])

# Fake os part
from pyfakefs import fake_filesystem, fake_os
filesystem = fake_filesystem.FakeFilesystem()
os_module = fake_os.FakeOsModule(filesystem)
fs_output = list(os_module.walk(Path(os_module.getcwd())))  # Deepest element is a tuple of [WindowsPath, list, list]
# Example (WindowsPath(C:\), list, list)
fs_output_str = list(os_module.walk(os_module.getcwd()))  # Deepest element is a tuple of [str, list, list] like os.walk
# Example ("C:\\", [], [])

# Warning as walk will return a tuple if there is no subdirectory or a list of tuple otherwise 
assert type(os_output[0][...][0]) is type(fs_output[0])  # Adapt the element to look at the first element of one of the deepest tuples
```

**Your environment**
platform.platform: Windows-10-10.0.22621-SP0
sys.version: Python 3.11.6 (tags/v3.11.6:8b6ee5b, Oct  2 2023, 14:57:12) [MSC v.1935 64 bit (AMD64)]
pyfakefs: 5.3.1
pytest: 7.4.3

date: 2023-11-25
patch: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index b9ae289..de4595e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -45,7 +45,7 @@ repos:
           - flake8-bugbear
         args: ["--extend-ignore=E203", "--max-line-length=88"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.0
+    rev: v1.7.1
     hooks:
       - id: mypy
         exclude: (docs|pyfakefs/tests)
diff --git a/pyfakefs/fake_scandir.py b/pyfakefs/fake_scandir.py
index cfe9f2f..16c7c34 100644
--- a/pyfakefs/fake_scandir.py
+++ b/pyfakefs/fake_scandir.py
@@ -261,7 +261,7 @@ def walk(filesystem, top, topdown=True, onerror=None, followlinks=False):
             if not topdown:
                 yield top_contents
 
-    return do_walk(to_string(top), top_most=True)
+    return do_walk(make_string_path(to_string(top)), top_most=True)
 
 
 class FakeScanDirModule:

test_patch: diff --git a/.github/workflows/testsuite.yml b/.github/workflows/testsuite.yml
index b42a759..24031dd 100644
--- a/.github/workflows/testsuite.yml
+++ b/.github/workflows/testsuite.yml
@@ -10,17 +10,19 @@ jobs:
     - name: Check out repository
       uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: "3.10"
     - name: install pytype
-      run: pip install setuptools pytype pytest scandir pathlib2 pandas xlrd django
+      run: pip install setuptools pytype pytest scandir pathlib2 pandas xlrd django pyarrow
     - name: Run pytype
       run: |
         pytype pyfakefs --keep-going --exclude pyfakefs/tests/* --exclude pyfakefs/pytest_tests/*
 
   tests:
     runs-on: ${{ matrix.os }}
+    env:
+      PYTHONWARNDEFAULTENCODING: true
     strategy:
       fail-fast: false
       matrix:
@@ -37,7 +39,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -49,7 +51,7 @@ jobs:
 
     - name: Cache dependencies
       id: cache-dep
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
         key: ${{ matrix.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/extra_requirements.txt') }}
@@ -107,14 +109,14 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
           pip install -r requirements.txt
           pip install -U pytest==${{ matrix.pytest-version }}
-          pip install opentimelineio
+          pip install opentimelineio undefined pandas parquet pyarrow
           pip install -e .
           if [[ '${{ matrix.pytest-version }}' == '4.0.2' ]]; then
              pip install -U attrs==19.1.0
@@ -139,7 +141,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
diff --git a/pyfakefs/tests/fake_open_test.py b/pyfakefs/tests/fake_open_test.py
index d04bb79..db51e77 100644
--- a/pyfakefs/tests/fake_open_test.py
+++ b/pyfakefs/tests/fake_open_test.py
@@ -17,7 +17,6 @@
 
 import errno
 import io
-import locale
 import os
 import stat
 import sys
@@ -25,7 +24,7 @@ import time
 import unittest
 
 from pyfakefs import fake_filesystem, helpers
-from pyfakefs.helpers import is_root, IS_PYPY
+from pyfakefs.helpers import is_root, IS_PYPY, get_locale_encoding
 from pyfakefs.fake_io import FakeIoModule
 from pyfakefs.fake_filesystem_unittest import PatchMode
 from pyfakefs.tests.test_utils import RealFsTestCase
@@ -119,14 +118,12 @@ class FakeFileOpenTest(FakeFileOpenTestBase):
         try:
             with self.open(file_path, "w") as f:
                 f.write(str_contents)
-        except UnicodeEncodeError:
+            with self.open(file_path, "rb") as f:
+                contents = f.read()
+            self.assertEqual(str_contents, contents.decode(get_locale_encoding()))
+        except UnicodeError:
             # see https://github.com/pytest-dev/pyfakefs/issues/623
             self.skipTest("This test does not work with an ASCII locale")
-        with self.open(file_path, "rb") as f:
-            contents = f.read()
-        self.assertEqual(
-            str_contents, contents.decode(locale.getpreferredencoding(False))
-        )
 
     def test_open_valid_file(self):
         contents = [
diff --git a/pyfakefs/tests/fake_os_test.py b/pyfakefs/tests/fake_os_test.py
index 15ad07c..209e51f 100644
--- a/pyfakefs/tests/fake_os_test.py
+++ b/pyfakefs/tests/fake_os_test.py
@@ -21,7 +21,7 @@ import stat
 import sys
 import unittest
 
-from pyfakefs.helpers import IN_DOCKER, IS_PYPY, get_uid, get_gid
+from pyfakefs.helpers import IN_DOCKER, IS_PYPY, get_uid, get_gid, reset_ids
 
 from pyfakefs import fake_filesystem, fake_os, fake_open, fake_file
 from pyfakefs.fake_filesystem import (
@@ -2139,6 +2139,27 @@ class FakeOsModuleTest(FakeOsModuleTestBase):
         self.assertFalse(self.os.path.exists(file_path))
         self.assert_raises_os_error(errno.ENOENT, self.os.chown, file_path, 100, 100)
 
+    def test_fail_add_entry_to_readonly_dir(self):
+        # regression test for #959
+        self.check_posix_only()
+        self.skip_real_fs()  # cannot change owner to root
+        if is_root():
+            self.skipTest("Non-root test only")
+
+        # create directory owned by root with permissions 0o755 (rwxr-xr-x)
+        ro_dir = self.make_path("readonly-dir")
+        self.create_dir(ro_dir, perm=0o755)
+        self.os.chown(ro_dir, 0, 0)
+
+        # adding a new entry to the readonly subdirectory should fail
+        with self.assertRaises(PermissionError):
+            with self.open(f"{ro_dir}/file.txt", "w"):
+                pass
+        file_path = self.make_path("file.txt")
+        self.create_file(file_path)
+        with self.assertRaises(PermissionError):
+            self.os.link(file_path, self.os.path.join(ro_dir, "file.txt"))
+
     def test_classify_directory_contents(self):
         """Directory classification should work correctly."""
         root_directory = self.make_path("foo")
@@ -2920,6 +2941,53 @@ class FakeOsModuleTestCaseInsensitiveFS(FakeOsModuleTestBase):
         files.sort()
         self.assertEqual(files, sorted(self.os.listdir(self.make_path("SymLink"))))
 
+    def test_listdir_possible_without_exe_permission(self):
+        # regression test for #960
+        self.check_posix_only()
+        self.skip_root()
+        directory = self.make_path("testdir")
+        file_path = self.os.path.join(directory, "file.txt")
+        self.create_file(file_path, contents="hey", perm=0o777)
+        self.os.chmod(directory, 0o655)  # rw-r-xr-x
+        # We cannot create any files in the directory, because that requires
+        # searching it
+        another_file = self.make_path("file.txt")
+        self.create_file(another_file, contents="hey")
+        with self.assertRaises(PermissionError):
+            self.os.link(another_file, self.os.path.join(directory, "link.txt"))
+        # We can enumerate the directory using listdir and scandir:
+        assert self.os.listdir(directory) == ["file.txt"]
+        assert len(list(self.os.scandir(directory))) == 1
+
+        # We cannot read files inside of the directory,
+        # even if we have read access to the file
+        with self.assertRaises(PermissionError):
+            self.os.stat(file_path)
+        with self.assertRaises(PermissionError):
+            with self.open(file_path) as f:
+                f.read()
+
+    def test_listdir_impossible_without_read_permission(self):
+        # regression test for #960
+        self.check_posix_only()
+        self.skip_root()
+        directory = self.make_path("testdir")
+        file_path = self.os.path.join(directory, "file.txt")
+        self.create_file(file_path, contents="hey", perm=0o777)
+        self.os.chmod(directory, 0o355)  # -wxr-xr-x
+        another_file = self.make_path("file.txt")
+        self.create_file(another_file, contents="hey")
+        self.os.link(another_file, self.os.path.join(directory, "link.txt"))
+        # We cannot enumerate the directory using listdir or scandir:
+        with self.assertRaises(PermissionError):
+            self.os.listdir(directory)
+        with self.assertRaises(PermissionError):
+            self.os.scandir(directory)
+        # we can access the file if we know the file name
+        assert self.os.stat(file_path).st_mode & 0o777 == 0o777
+        with self.open(file_path) as f:
+            assert f.read() == "hey"
+
     def test_fdopen_mode(self):
         self.skip_real_fs()
         file_path1 = self.make_path("some_file1")
@@ -4676,228 +4744,252 @@ class RealOsModuleWalkTest(FakeOsModuleWalkTest):
 class FakeOsModuleDirFdTest(FakeOsModuleTestBase):
     def setUp(self):
         super(FakeOsModuleDirFdTest, self).setUp()
-        self.os.supports_dir_fd.clear()
-        self.filesystem.is_windows_fs = False
-        self.filesystem.create_dir("/foo/bar")
-        self.dir_fd = self.os.open("/foo", os.O_RDONLY)
-        self.filesystem.create_file("/foo/baz")
+        self.check_posix_only()
+        if not self.use_real_fs():
+            # in the real OS, we test the option as is, in the fake OS
+            # we test both the supported and unsupported option
+            self.os.supports_dir_fd.clear()
+        self.dir_fd_path = self.make_path("foo")
+        self.create_dir(self.dir_fd_path)
+        self.dir_fd = self.os.open(self.dir_fd_path, os.O_RDONLY)
+        self.fname = "baz"
+        self.fpath = self.os.path.join(self.dir_fd_path, self.fname)
+        self.create_file(self.fpath)
+
+    def add_supported_function(self, fct):
+        if not self.use_real_fs():
+            self.os.supports_dir_fd.add(fct)
 
     def test_access(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.access,
-            "baz",
-            self.os.F_OK,
-            dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.access)
-        self.assertTrue(self.os.access("baz", self.os.F_OK, dir_fd=self.dir_fd))
+        def os_access():
+            return self.os.access(self.fname, self.os.F_OK, dir_fd=self.dir_fd)
+
+        if self.os.access not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_access()
+        self.add_supported_function(self.os.access)
+        if self.os.access in self.os.supports_dir_fd:
+            self.assertTrue(os_access())
 
     def test_chmod(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.chmod,
-            "baz",
-            0o6543,
-            dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.chmod)
-        self.os.chmod("baz", 0o6543, dir_fd=self.dir_fd)
-        st = self.os.stat("/foo/baz")
-        self.assert_mode_equal(0o6543, st.st_mode)
+        def os_chmod():
+            self.os.chmod(self.fname, 0o6543, dir_fd=self.dir_fd)
+
+        if self.os.chmod not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_chmod()
+        self.add_supported_function(self.os.chmod)
+        if self.os.chmod in self.os.supports_dir_fd:
+            os_chmod()
+            st = self.os.stat(self.fpath)
+            self.assert_mode_equal(0o6543, st.st_mode)
 
     @unittest.skipIf(not hasattr(os, "chown"), "chown not on all platforms available")
     def test_chown(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.chown,
-            "baz",
-            100,
-            101,
-            dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.chown)
-        self.os.chown("baz", 100, 101, dir_fd=self.dir_fd)
-        st = self.os.stat("/foo/baz")
-        self.assertEqual(st[stat.ST_UID], 100)
-        self.assertEqual(st[stat.ST_GID], 101)
+        def os_chown():
+            self.os.chown(self.fname, 100, 101, dir_fd=self.dir_fd)
+
+        self.skip_real_fs()
+        if self.os.chown not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_chown()
+        self.add_supported_function(self.os.chown)
+        os_chown()
+        st = self.os.stat(self.fpath)
+        self.assertEqual(100, st[stat.ST_UID])
+        self.assertEqual(101, st[stat.ST_GID])
 
     def test_link_src_fd(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.link,
-            "baz",
-            "/bat",
-            src_dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.link)
-        self.os.link("baz", "/bat", src_dir_fd=self.dir_fd)
-        self.assertTrue(self.os.path.exists("/bat"))
+        def os_link():
+            self.os.link(self.fname, link_dest, src_dir_fd=self.dir_fd)
+
+        link_dest = self.make_path("bat")
+        if self.os.link not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_link()
+        self.add_supported_function(self.os.link)
+        if self.os.link in self.os.supports_dir_fd:
+            os_link()
+            self.assertTrue(self.os.path.exists(link_dest))
 
     def test_link_dst_fd(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.link,
-            "baz",
-            "/bat",
-            dst_dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.link)
-        self.os.link("/foo/baz", "bat", dst_dir_fd=self.dir_fd)
-        self.assertTrue(self.os.path.exists("/foo/bat"))
+        def os_link():
+            self.os.link(self.fpath, "bat", dst_dir_fd=self.dir_fd)
+
+        if self.os.link not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_link()
+        self.add_supported_function(self.os.link)
+        if self.os.link in self.os.supports_dir_fd:
+            os_link()
+            link_path = self.os.path.join(self.dir_fd_path, "bat")
+            self.assertTrue(self.os.path.exists(link_path))
 
     def test_symlink(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.symlink,
-            "baz",
-            "/bat",
-            dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.symlink)
-        self.os.symlink("baz", "/bat", dir_fd=self.dir_fd)
-        self.assertTrue(self.os.path.exists("/bat"))
+        def os_symlink():
+            self.os.symlink(self.fpath, "bat", dir_fd=self.dir_fd)
+
+        if self.os.symlink not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_symlink()
+        self.add_supported_function(self.os.symlink)
+        if self.os.symlink in self.os.supports_dir_fd:
+            os_symlink()
+            link_path = self.os.path.join(self.dir_fd_path, "bat")
+            self.assertTrue(self.os.path.exists(link_path))
 
     def test_readlink(self):
-        self.skip_if_symlink_not_supported()
-        self.filesystem.create_symlink("/meyer/lemon/pie", "/foo/baz")
-        self.filesystem.create_symlink("/geo/metro", "/meyer")
-        self.assertRaises(
-            NotImplementedError,
-            self.os.readlink,
-            "/geo/metro/lemon/pie",
-            dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.readlink)
-        self.assertEqual(
-            "/foo/baz",
-            self.os.readlink("/geo/metro/lemon/pie", dir_fd=self.dir_fd),
-        )
+        def os_readlink():
+            return self.os.readlink("lemon/tree", dir_fd=self.dir_fd)
+
+        link_dir = self.os.path.join(self.dir_fd_path, "lemon")
+        self.create_dir(link_dir)
+        self.create_symlink(self.os.path.join(link_dir, "tree"), self.fpath)
+        if self.os.readlink not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_readlink()
+        self.add_supported_function(self.os.readlink)
+        if self.os.readlink in self.os.supports_dir_fd:
+            self.assertEqual(self.fpath, os_readlink())
 
     def test_stat(self):
-        self.assertRaises(NotImplementedError, self.os.stat, "baz", dir_fd=self.dir_fd)
-        self.os.supports_dir_fd.add(self.os.stat)
-        st = self.os.stat("baz", dir_fd=self.dir_fd)
-        self.assertEqual(st.st_mode, 0o100666)
+        def os_stat():
+            return self.os.stat(self.fname, dir_fd=self.dir_fd)
+
+        if self.os.stat not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_stat()
+        self.add_supported_function(self.os.stat)
+        if self.os.stat in self.os.supports_dir_fd:
+            self.assertEqual(os_stat().st_mode, 0o100666)
 
     def test_lstat(self):
-        self.assertRaises(NotImplementedError, self.os.lstat, "baz", dir_fd=self.dir_fd)
-        self.os.supports_dir_fd.add(self.os.lstat)
-        st = self.os.lstat("baz", dir_fd=self.dir_fd)
+        st = self.os.lstat(self.fname, dir_fd=self.dir_fd)
         self.assertEqual(st.st_mode, 0o100666)
 
     def test_mkdir(self):
-        self.assertRaises(
-            NotImplementedError, self.os.mkdir, "newdir", dir_fd=self.dir_fd
-        )
-        self.os.supports_dir_fd.add(self.os.mkdir)
-        self.os.mkdir("newdir", dir_fd=self.dir_fd)
-        self.assertTrue(self.os.path.exists("/foo/newdir"))
+        def os_mkdir():
+            self.os.mkdir("newdir", dir_fd=self.dir_fd)
+
+        newdir_path = self.os.path.join(self.dir_fd_path, "newdir")
+        if self.os.mkdir not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_mkdir()
+        self.add_supported_function(self.os.mkdir)
+        if self.os.mkdir in self.os.supports_dir_fd:
+            os_mkdir()
+            self.assertTrue(self.os.path.exists(newdir_path))
 
     def test_rmdir(self):
-        self.assertRaises(NotImplementedError, self.os.rmdir, "bar", dir_fd=self.dir_fd)
-        self.os.supports_dir_fd.add(self.os.rmdir)
-        self.os.rmdir("bar", dir_fd=self.dir_fd)
-        self.assertFalse(self.os.path.exists("/foo/bar"))
+        def os_rmdir():
+            self.os.rmdir("dir", dir_fd=self.dir_fd)
+
+        dir_path = self.os.path.join(self.dir_fd_path, "dir")
+        self.create_dir(dir_path)
+        self.assertTrue(self.os.path.exists(dir_path))
+        if self.os.rmdir not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_rmdir()
+        self.add_supported_function(self.os.rmdir)
+        if self.os.rmdir in self.os.supports_dir_fd:
+            os_rmdir()
+            self.assertFalse(self.os.path.exists(dir_path))
 
     @unittest.skipIf(not hasattr(os, "mknod"), "mknod not on all platforms available")
     def test_mknod(self):
-        self.assertRaises(
-            NotImplementedError, self.os.mknod, "newdir", dir_fd=self.dir_fd
-        )
-        self.os.supports_dir_fd.add(self.os.mknod)
-        self.os.mknod("newdir", dir_fd=self.dir_fd)
-        self.assertTrue(self.os.path.exists("/foo/newdir"))
+        def os_mknod():
+            self.os.mknod("newdir", dir_fd=self.dir_fd)
+
+        if self.os.mknod not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_mknod()
+        self.add_supported_function(self.os.mknod)
+        if self.os.mknod in self.os.supports_dir_fd:
+            os_mknod()
+            newdir_path = self.os.path.join(self.dir_fd_path, "newdir")
+            self.assertTrue(self.os.path.exists(newdir_path))
 
     def test_rename_src_fd(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.rename,
-            "baz",
-            "/foo/batz",
-            src_dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.rename)
-        self.os.rename("bar", "/foo/batz", src_dir_fd=self.dir_fd)
-        self.assertTrue(self.os.path.exists("/foo/batz"))
+        def os_rename():
+            self.os.rename(self.fname, new_name, src_dir_fd=self.dir_fd)
+
+        new_name = self.os.path.join(self.dir_fd_path, "batz")
+        if self.os.rename not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_rename()
+        self.add_supported_function(self.os.rename)
+        if self.os.rename in self.os.supports_dir_fd:
+            os_rename()
+            self.assertTrue(self.os.path.exists(new_name))
 
     def test_rename_dst_fd(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.rename,
-            "baz",
-            "/foo/batz",
-            dst_dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.rename)
-        self.os.rename("/foo/bar", "batz", dst_dir_fd=self.dir_fd)
-        self.assertTrue(self.os.path.exists("/foo/batz"))
+        def os_rename():
+            self.os.rename(self.fpath, "batz", dst_dir_fd=self.dir_fd)
+
+        if self.os.rename not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_rename()
+        self.add_supported_function(self.os.rename)
+        if self.os.rename in self.os.supports_dir_fd:
+            os_rename()
+            new_path = self.os.path.join(self.dir_fd_path, "batz")
+            self.assertTrue(self.os.path.exists(new_path))
 
     def test_replace_src_fd(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.rename,
-            "baz",
-            "/foo/batz",
-            src_dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.rename)
-        self.os.replace("bar", "/foo/batz", src_dir_fd=self.dir_fd)
-        self.assertTrue(self.os.path.exists("/foo/batz"))
+        new_name = self.os.path.join(self.dir_fd_path, "batz")
+        self.os.replace(self.fname, new_name, src_dir_fd=self.dir_fd)
+        self.assertTrue(self.os.path.exists(new_name))
 
     def test_replace_dst_fd(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.rename,
-            "baz",
-            "/foo/batz",
-            dst_dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.rename)
-        self.os.replace("/foo/bar", "batz", dst_dir_fd=self.dir_fd)
-        self.assertTrue(self.os.path.exists("/foo/batz"))
+        self.os.replace(self.fpath, "batz", dst_dir_fd=self.dir_fd)
+        new_path = self.os.path.join(self.dir_fd_path, "batz")
+        self.assertTrue(self.os.path.exists(new_path))
 
     def test_remove(self):
-        self.assertRaises(
-            NotImplementedError, self.os.remove, "baz", dir_fd=self.dir_fd
-        )
-        self.os.supports_dir_fd.add(self.os.remove)
-        self.os.remove("baz", dir_fd=self.dir_fd)
-        self.assertFalse(self.os.path.exists("/foo/baz"))
+        self.os.remove(self.fname, dir_fd=self.dir_fd)
+        self.assertFalse(self.os.path.exists(self.fpath))
 
     def test_unlink(self):
-        self.assertRaises(
-            NotImplementedError, self.os.unlink, "baz", dir_fd=self.dir_fd
-        )
-        self.os.supports_dir_fd.add(self.os.unlink)
-        self.os.unlink("baz", dir_fd=self.dir_fd)
-        self.assertFalse(self.os.path.exists("/foo/baz"))
+        def os_unlink():
+            self.os.unlink(self.fname, dir_fd=self.dir_fd)
+
+        if self.os.unlink not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_unlink()
+        self.add_supported_function(self.os.unlink)
+        if self.os.unlink in self.os.supports_dir_fd:
+            os_unlink()
+            self.assertFalse(self.os.path.exists(self.fpath))
 
     def test_utime(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.utime,
-            "baz",
-            (1, 2),
-            dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.utime)
-        self.os.utime("baz", times=(1, 2), dir_fd=self.dir_fd)
-        st = self.os.stat("/foo/baz")
-        self.assertEqual(1, st.st_atime)
-        self.assertEqual(2, st.st_mtime)
+        def os_utime():
+            self.os.utime(self.fname, times=(1, 2), dir_fd=self.dir_fd)
+
+        if self.os.utime not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_utime()
+        self.add_supported_function(self.os.utime)
+        if self.os.utime in self.os.supports_dir_fd:
+            os_utime()
+            st = self.os.stat(self.fpath)
+            self.assertEqual(1, st.st_atime)
+            self.assertEqual(2, st.st_mtime)
 
     def test_open(self):
-        self.assertRaises(
-            NotImplementedError,
-            self.os.open,
-            "baz",
-            os.O_RDONLY,
-            dir_fd=self.dir_fd,
-        )
-        self.os.supports_dir_fd.add(self.os.open)
-        fd = self.os.open("baz", os.O_RDONLY, dir_fd=self.dir_fd)
-        self.assertLess(0, fd)
+        def os_open():
+            return self.os.open(self.fname, os.O_RDONLY, dir_fd=self.dir_fd)
+
+        if self.os.open not in self.os.supports_dir_fd:
+            with self.assertRaises(NotImplementedError):
+                os_open()
+        self.add_supported_function(self.os.open)
+        if self.os.open in self.os.supports_dir_fd:
+            self.assertLess(0, os_open())
+
+
+class RealOsModuleDirFdTest(FakeOsModuleDirFdTest):
+    def use_real_fs(self):
+        return True
 
 
 class StatPropagationTest(TestCase):
@@ -5046,7 +5138,7 @@ class FakeScandirTest(FakeOsModuleTestBase):
             self.create_dir(self.linked_dir_path)
             self.create_file(
                 self.linked_file_path, contents=b"a" * self.LINKED_FILE_SIZE
-            ),
+            )
             self.create_symlink(self.dir_link_path, self.linked_dir_path)
             self.create_symlink(self.file_link_path, self.linked_file_path)
             self.create_symlink(self.dir_rel_link_path, self.rel_linked_dir_path)
@@ -5410,7 +5502,7 @@ class FakeOsUnreadableDirTest(FakeOsModuleTestBase):
         set_uid(uid + 10)
         self.assertEqual(uid + 10, self.os.getuid())
         self.assertEqual(uid + 10, get_uid())
-        set_uid(uid)
+        reset_ids()
         self.assertEqual(uid, self.os.getuid())
 
     def test_getgid(self):
@@ -5420,7 +5512,7 @@ class FakeOsUnreadableDirTest(FakeOsModuleTestBase):
         set_gid(gid + 10)
         self.assertEqual(gid + 10, self.os.getgid())
         self.assertEqual(gid + 10, get_gid())
-        set_gid(gid)
+        reset_ids()
         self.assertEqual(gid, self.os.getgid())
 
     def test_listdir_unreadable_dir(self):
@@ -5442,7 +5534,7 @@ class FakeOsUnreadableDirTest(FakeOsModuleTestBase):
         dir_path = self.make_path("dir1")
         self.create_dir(dir_path, perm=0o600)
         self.assertTrue(self.os.path.exists(dir_path))
-        set_uid(user_id)
+        reset_ids()
         if not is_root():
             with self.assertRaises(PermissionError):
                 self.os.listdir(dir_path)
@@ -5451,12 +5543,11 @@ class FakeOsUnreadableDirTest(FakeOsModuleTestBase):
 
     def test_listdir_group_readable_dir_from_other_user(self):
         self.skip_real_fs()  # won't change user in real fs
-        user_id = get_uid()
-        set_uid(user_id + 1)
+        set_uid(get_uid() + 1)
         dir_path = self.make_path("dir1")
         self.create_dir(dir_path, perm=0o660)
         self.assertTrue(self.os.path.exists(dir_path))
-        set_uid(user_id)
+        reset_ids()
         self.assertEqual([], self.os.listdir(dir_path))
 
     def test_listdir_group_readable_dir_from_other_group(self):
@@ -5475,14 +5566,15 @@ class FakeOsUnreadableDirTest(FakeOsModuleTestBase):
             self.assertEqual([], self.os.listdir(dir_path))
 
     def test_listdir_other_readable_dir_from_other_group(self):
+        self.check_posix_only()
         self.skip_real_fs()  # won't change user in real fs
-        group_id = get_gid()
-        set_gid(group_id + 1)
         dir_path = self.make_path("dir1")
-        self.create_dir(dir_path, perm=0o004)
+        self.create_dir(dir_path, 0o004)
+        set_uid(get_uid() + 1)
+        set_gid(get_gid() + 1)
         self.assertTrue(self.os.path.exists(dir_path))
-        set_gid(group_id)
         self.assertEqual([], self.os.listdir(dir_path))
+        reset_ids()
 
     def test_stat_unreadable_dir(self):
         self.assertEqual(0, self.os.stat(self.dir_path).st_mode & 0o666)
@@ -5509,12 +5601,11 @@ class FakeOsUnreadableDirTest(FakeOsModuleTestBase):
 
     def test_remove_unreadable_dir_from_other_user(self):
         self.skip_real_fs()  # won't change user in real fs
-        user_id = get_uid()
-        set_uid(user_id + 1)
+        set_uid(get_uid() + 1)
         dir_path = self.make_path("dir1")
         self.create_dir(dir_path, perm=0o000)
         self.assertTrue(self.os.path.exists(dir_path))
-        set_uid(user_id)
+        reset_ids()
         if not is_root():
             with self.assertRaises(PermissionError):
                 self.os.rmdir(dir_path)
diff --git a/pyfakefs/tests/fake_pathlib_test.py b/pyfakefs/tests/fake_pathlib_test.py
index 81ebdfd..15bb7a0 100644
--- a/pyfakefs/tests/fake_pathlib_test.py
+++ b/pyfakefs/tests/fake_pathlib_test.py
@@ -1139,6 +1139,18 @@ class FakePathlibUsageInOsFunctionsTest(RealPathlibTestCase):
         with self.assertRaises(NotImplementedError):
             self.path(path).group()
 
+    def test_walk(self):
+        """Regression test for #915 - walk results shall be strings."""
+        base_dir = self.make_path("foo")
+        base_path = self.path(base_dir)
+        self.create_dir(base_path)
+        self.create_file(base_path / "1.txt")
+        self.create_file(base_path / "bar" / "2.txt")
+        result = list(step for step in self.os.walk(base_path))
+        assert len(result) == 2
+        assert result[0] == (base_dir, ["bar"], ["1.txt"])
+        assert result[1] == (self.os.path.join(base_dir, "bar"), [], ["2.txt"])
+
 
 class RealPathlibUsageInOsFunctionsTest(FakePathlibUsageInOsFunctionsTest):
     def use_real_fs(self):


----

problem_statement: EncodingWarnings raised (3) when using pyfakefs
When running the distutils test suite, I see the following warnings:

```
distutils/tests/test_dist.py::TestDistributionBehavior::test_find_config_files_permission_error
  /Users/jaraco/code/pypa/distutils/.tox/py/lib/python3.12/site-packages/pyfakefs/fake_file.py:266: EncodingWarning: UTF-8 Mode affects locale.getpreferredencoding(). Consider locale.getencoding() instead.
    self.encoding or locale.getpreferredencoding(False),

distutils/tests/test_dist.py::TestDistributionBehavior::test_find_config_files_permission_error
  /Users/jaraco/code/pypa/distutils/.tox/py/lib/python3.12/site-packages/pyfakefs/fake_file.py:748: EncodingWarning: UTF-8 Mode affects locale.getpreferredencoding(). Consider locale.getencoding() instead.
    self._encoding = encoding or locale.getpreferredencoding(False)

distutils/tests/test_dist.py::TestDistributionBehavior::test_find_config_files_permission_error
  /Users/jaraco/code/pypa/distutils/.tox/py/lib/python3.12/site-packages/pyfakefs/helpers.py:409: EncodingWarning: 'encoding' argument not specified
    super().__init__(self._bytestream, encoding, errors, newline)
```

date: 2024-02-26
patch: diff --git a/pyfakefs/fake_file.py b/pyfakefs/fake_file.py
index 44d45be..bc37a96 100644
--- a/pyfakefs/fake_file.py
+++ b/pyfakefs/fake_file.py
@@ -16,7 +16,6 @@
 """
 import errno
 import io
-import locale
 import os
 import sys
 from stat import (
@@ -52,6 +51,7 @@ from pyfakefs.helpers import (
     real_encoding,
     AnyPath,
     AnyString,
+    get_locale_encoding,
 )
 
 if TYPE_CHECKING:
@@ -190,7 +190,7 @@ class FakeFile:
         """Return the contents as string with the original encoding."""
         if isinstance(self.byte_contents, bytes):
             return self.byte_contents.decode(
-                self.encoding or locale.getpreferredencoding(False),
+                self.encoding or get_locale_encoding(),
                 errors=self.errors,
             )
         return None
@@ -263,7 +263,7 @@ class FakeFile:
         if is_unicode_string(contents):
             contents = bytes(
                 cast(str, contents),
-                self.encoding or locale.getpreferredencoding(False),
+                self.encoding or get_locale_encoding(),
                 self.errors,
             )
         return cast(bytes, contents)
@@ -745,7 +745,7 @@ class FakeFileWrapper:
         self._use_line_buffer = not binary and buffering == 1
 
         contents = file_object.byte_contents
-        self._encoding = encoding or locale.getpreferredencoding(False)
+        self._encoding = encoding or get_locale_encoding()
         errors = errors or "strict"
         self._io: Union[BinaryBufferIO, TextBufferIO] = (
             BinaryBufferIO(contents)
diff --git a/pyfakefs/helpers.py b/pyfakefs/helpers.py
index 416be19..a5d8d5b 100644
--- a/pyfakefs/helpers.py
+++ b/pyfakefs/helpers.py
@@ -109,6 +109,12 @@ def is_unicode_string(val: Any) -> bool:
     return hasattr(val, "encode")
 
 
+def get_locale_encoding():
+    if sys.version_info >= (3, 11):
+        return locale.getencoding()
+    return locale.getpreferredencoding(False)
+
+
 @overload
 def make_string_path(dir_name: AnyStr) -> AnyStr:
     ...
@@ -127,7 +133,7 @@ def to_string(path: Union[AnyStr, Union[str, bytes]]) -> str:
     """Return the string representation of a byte string using the preferred
     encoding, or the string itself if path is a str."""
     if isinstance(path, bytes):
-        return path.decode(locale.getpreferredencoding(False))
+        return path.decode(get_locale_encoding())
     return path
 
 
@@ -135,7 +141,7 @@ def to_bytes(path: Union[AnyStr, Union[str, bytes]]) -> bytes:
     """Return the bytes representation of a string using the preferred
     encoding, or the byte string itself if path is a byte string."""
     if isinstance(path, str):
-        return bytes(path, locale.getpreferredencoding(False))
+        return bytes(path, get_locale_encoding())
     return path
 
 
@@ -181,7 +187,7 @@ def matching_string(  # type: ignore[misc]
     if string is None:
         return string
     if isinstance(matched, bytes) and isinstance(string, str):
-        return string.encode(locale.getpreferredencoding(False))
+        return string.encode(get_locale_encoding())
     return string  # pytype: disable=bad-return-type
 
 


----

problem_statement: Patched `os.symlink` mishandles directory descriptors
**Describe the bug**
The `os.symlink` function accepts a `dir_fd` argument to specify the destination of the created symlink. The fakefs `os.symlink` is behaves incorrectly and uses the `dir_fd` to rewrite the `src` argument:

```python
src = self._path_with_dir_fd(src, self.symlink, dir_fd)  # (wrong)
self.filesystem.create_symlink(dst, src, create_missing_dirs=False)
```

The `src` argument should be left verbatim as given. I am guessing that the code should have been written to resolve the `dst` argument instead:

```python
dst = self._path_with_dir_fd(dst, self.symlink, dir_fd)
self.filesystem.create_symlink(dst, src, create_missing_dirs=False)
```

The doc comment is written to reflect the incorrect behavior.

**How To Reproduce**
```python
def _test_symlink(base: Path):
    dir_fd = os.open(base, os.O_DIRECTORY)
    os.symlink("file.txt", "link.txt", dir_fd=dir_fd)
    assert os.readlink(base / "link.txt") == "file.txt"

def test_symlink_realfs(tmp_path: Path):
    _test_symlink(tmp_path)

def test_symlink_fakefs(fs):
    _test_symlink(Path("/tmp/"))
```

The `os.symlink` call above will succeed, but creates `link.txt` in the wrong location, leading to  `os.readlink` generating a `FileNotFoundError`.

**Your environment**
Please run the following in the environment where the problem happened and
paste the output.
```console
$ python -c "import platform; print(platform.platform())"
Linux-6.7.6-arch1-1-x86_64-with-glibc2.39
$ python -c "import sys; print('Python', sys.version)"
Python 3.11.7 (main, Jan 29 2024, 16:03:57) [GCC 13.2.1 20230801]
$ python -c "from pyfakefs import __version__; print('pyfakefs', __version__)"
pyfakefs 5.4.dev0
$ python -c "import pytest; print('pytest', pytest.__version__)"
pytest 7.2.0
```

date: 2024-03-08
patch: diff --git a/pyfakefs/fake_os.py b/pyfakefs/fake_os.py
index 152d5a2..6b6a487 100644
--- a/pyfakefs/fake_os.py
+++ b/pyfakefs/fake_os.py
@@ -219,7 +219,7 @@ class FakeOsModule:
         flags: int,
         mode: Optional[int] = None,
         *,
-        dir_fd: Optional[int] = None
+        dir_fd: Optional[int] = None,
     ) -> int:
         """Return the file descriptor for a FakeFile.
 
@@ -422,7 +422,7 @@ class FakeOsModule:
         directory = self.filesystem.resolve(path)
         # A full implementation would check permissions all the way
         # up the tree.
-        if not is_root() and not directory.st_mode | PERM_EXE:
+        if not is_root() and not directory.has_permission(PERM_EXE):
             self.filesystem.raise_os_error(errno.EACCES, directory.name)
         self.filesystem.cwd = path  # type: ignore[assignment]
 
@@ -538,7 +538,7 @@ class FakeOsModule:
         value: bytes,
         flags: int = 0,
         *,
-        follow_symlinks: bool = True
+        follow_symlinks: bool = True,
     ) -> None:
         """Sets the value of the given extended filesystem attribute for
         `path`.
@@ -634,7 +634,7 @@ class FakeOsModule:
         path: AnyStr,
         *,
         dir_fd: Optional[int] = None,
-        follow_symlinks: bool = True
+        follow_symlinks: bool = True,
     ) -> FakeStatResult:
         """Return the os.stat-like tuple for the FakeFile object of entry_path.
 
@@ -671,7 +671,7 @@ class FakeOsModule:
             OSError: if the filesystem object doesn't exist.
         """
         # stat should return the tuple representing return value of os.stat
-        path = self._path_with_dir_fd(path, self.lstat, dir_fd)
+        path = self._path_with_dir_fd(path, self.lstat, dir_fd, check_supported=False)
         return self.filesystem.stat(path, follow_symlinks=False)
 
     def remove(self, path: AnyStr, dir_fd: Optional[int] = None) -> None:
@@ -687,7 +687,7 @@ class FakeOsModule:
             OSError: if path does not exist.
             OSError: if removal failed.
         """
-        path = self._path_with_dir_fd(path, self.remove, dir_fd)
+        path = self._path_with_dir_fd(path, self.remove, dir_fd, check_supported=False)
         self.filesystem.remove(path)
 
     def unlink(self, path: AnyStr, *, dir_fd: Optional[int] = None) -> None:
@@ -712,7 +712,7 @@ class FakeOsModule:
         dst: AnyStr,
         *,
         src_dir_fd: Optional[int] = None,
-        dst_dir_fd: Optional[int] = None
+        dst_dir_fd: Optional[int] = None,
     ) -> None:
         """Rename a FakeFile object at old_file_path to new_file_path,
         preserving all properties.
@@ -774,7 +774,7 @@ class FakeOsModule:
         dst: AnyStr,
         *,
         src_dir_fd: Optional[int] = None,
-        dst_dir_fd: Optional[int] = None
+        dst_dir_fd: Optional[int] = None,
     ) -> None:
         """Renames a FakeFile object at old_file_path to new_file_path,
         preserving all properties.
@@ -798,8 +798,12 @@ class FakeOsModule:
             OSError: if the file would be moved to another filesystem
                 (e.g. mount point)
         """
-        src = self._path_with_dir_fd(src, self.rename, src_dir_fd)
-        dst = self._path_with_dir_fd(dst, self.rename, dst_dir_fd)
+        src = self._path_with_dir_fd(
+            src, self.rename, src_dir_fd, check_supported=False
+        )
+        dst = self._path_with_dir_fd(
+            dst, self.rename, dst_dir_fd, check_supported=False
+        )
         self.filesystem.rename(src, dst, force_replace=True)
 
     def rmdir(self, path: AnyStr, *, dir_fd: Optional[int] = None) -> None:
@@ -891,7 +895,11 @@ class FakeOsModule:
         self.filesystem.makedirs(name, mode, exist_ok)
 
     def _path_with_dir_fd(
-        self, path: AnyStr, fct: Callable, dir_fd: Optional[int]
+        self,
+        path: AnyStr,
+        fct: Callable,
+        dir_fd: Optional[int],
+        check_supported: bool = True,
     ) -> AnyStr:
         """Return the path considering dir_fd. Raise on invalid parameters."""
         try:
@@ -901,7 +909,7 @@ class FakeOsModule:
             path = path
         if dir_fd is not None:
             # check if fd is supported for the built-in real function
-            if fct not in self.supports_dir_fd:
+            if check_supported and (fct not in self.supports_dir_fd):
                 raise NotImplementedError("dir_fd unavailable on this platform")
             if isinstance(path, int):
                 raise ValueError(
@@ -957,7 +965,7 @@ class FakeOsModule:
         *,
         dir_fd: Optional[int] = None,
         effective_ids: bool = False,
-        follow_symlinks: bool = True
+        follow_symlinks: bool = True,
     ) -> bool:
         """Check if a file exists and has the specified permissions.
 
@@ -995,7 +1003,7 @@ class FakeOsModule:
         mode: int,
         *,
         dir_fd: Optional[int] = None,
-        follow_symlinks: bool = True
+        follow_symlinks: bool = True,
     ) -> None:
         """Change the permissions of a file as encoded in integer mode.
 
@@ -1067,7 +1075,7 @@ class FakeOsModule:
         gid: int,
         *,
         dir_fd: Optional[int] = None,
-        follow_symlinks: bool = True
+        follow_symlinks: bool = True,
     ) -> None:
         """Set ownership of a faked file.
 
@@ -1102,7 +1110,7 @@ class FakeOsModule:
         mode: Optional[int] = None,
         device: int = 0,
         *,
-        dir_fd: Optional[int] = None
+        dir_fd: Optional[int] = None,
     ) -> None:
         """Create a filesystem node named 'filename'.
 
@@ -1153,7 +1161,7 @@ class FakeOsModule:
         dst: AnyStr,
         target_is_directory: bool = False,
         *,
-        dir_fd: Optional[int] = None
+        dir_fd: Optional[int] = None,
     ) -> None:
         """Creates the specified symlink, pointed at the specified link target.
 
@@ -1162,12 +1170,12 @@ class FakeOsModule:
             dst: Path to the symlink to create.
             target_is_directory: Currently ignored.
             dir_fd: If not `None`, the file descriptor of a directory,
-                with `src` being relative to this directory.
+                with `dst` being relative to this directory.
 
         Raises:
             OSError:  if the file already exists.
         """
-        src = self._path_with_dir_fd(src, self.symlink, dir_fd)
+        dst = self._path_with_dir_fd(dst, self.symlink, dir_fd)
         self.filesystem.create_symlink(dst, src, create_missing_dirs=False)
 
     def link(
@@ -1176,9 +1184,9 @@ class FakeOsModule:
         dst: AnyStr,
         *,
         src_dir_fd: Optional[int] = None,
-        dst_dir_fd: Optional[int] = None
+        dst_dir_fd: Optional[int] = None,
     ) -> None:
-        """Create a hard link at new_path, pointing at old_path.
+        """Create a hard link at dst, pointing at src.
 
         Args:
             src: An existing path to the target file.