Repository: huggingface/datasets
File: tests/test_data_files.py (Python)

Code before:
    assert Hasher.hash(data_files1) == Hasher.hash(data_files2)

    data_files2 = DataFilesDict(sorted(data_files1.items(), reverse=True))
    assert Hasher.hash(data_files1) == Hasher.hash(data_files2)

    patterns2 = {"train": [_TEST_URL], "test": [_TEST_URL]}
    data_files2 = DataFilesDict.from_patterns(patterns2)
    assert Hasher.hash(data_files1) != Hasher.hash(data_files2)

    with patch("fsspec.implementations.http._file_info", return_value={}):
        data_files2 = DataFilesDict.from_patterns(patterns)
        assert Hasher.hash(data_files1) != Hasher.hash(data_files2)

    with patch("os.stat", return_value=os.stat(__file__)):
        data_files2 = DataFilesDict.from_patterns(patterns)
        assert Hasher.hash(data_files1) != Hasher.hash(data_files2)


def mock_fs(file_paths: List[str]):
    """
    Set up a mock filesystem for fsspec containing the provided files

    Example:

    ```py
    >>> DummyTestFS = mock_fs(["data/train.txt", "data.test.txt"])
    >>> assert fsspec.get_filesystem_class("mock").__name__ == "DummyTestFS"
    >>> assert type(fs).__name__ == "DummyTestFS"
    >>> print(fs.glob("**"))
    ["data", "data/train.txt", "data.test.txt"]
    ```
    """

    dir_paths = {file_path.rsplit("/", 1)[0] for file_path in file_paths if "/" in file_path}
    fs_contents = [{"name": dir_path, "type": "directory"} for dir_path in dir_paths] + [
        {"name": file_path, "type": "file", "size": 10} for file_path in file_paths
    ]

    class DummyTestFS(AbstractFileSystem):
        protocol = ("mock", "dummy")
        _fs_contents = fs_contents

        def ls(self, path, detail=True, refresh=True, **kwargs):
            if kwargs.pop("strip_proto", True):
                path = self._strip_protocol(path)

            files = not refresh and self._ls_from_cache(path)
            if not files:
                files = [file for file in self._fs_contents if path == self._parent(file["name"])]
                files.sort(key=lambda file: file["name"])
                self.dircache[path.rstrip("/")] = files

Review comment (suggestion):
```suggestion
    >>> DummyTestFS = mock_fs(["data/train.txt", "data.test.txt"])
    >>> fs = DummyTestFS()
```

Code after:
    assert Hasher.hash(data_files1) == Hasher.hash(data_files2)

    data_files2 = DataFilesDict(sorted(data_files1.items(), reverse=True))
    assert Hasher.hash(data_files1) == Hasher.hash(data_files2)

    patterns2 = {"train": [_TEST_URL], "test": [_TEST_URL]}
    data_files2 = DataFilesDict.from_patterns(patterns2)
    assert Hasher.hash(data_files1) != Hasher.hash(data_files2)

    with patch("fsspec.implementations.http._file_info", return_value={}):
        data_files2 = DataFilesDict.from_patterns(patterns)
        assert Hasher.hash(data_files1) != Hasher.hash(data_files2)

    with patch("os.stat", return_value=os.stat(__file__)):
        data_files2 = DataFilesDict.from_patterns(patterns)
        assert Hasher.hash(data_files1) != Hasher.hash(data_files2)


def mock_fs(file_paths: List[str]):
    """
    Set up a mock filesystem for fsspec containing the provided files

    Example:

    ```py
    >>> DummyTestFS = mock_fs(["data/train.txt", "data.test.txt"])
    >>> fs = DummyTestFS()
    >>> assert fsspec.get_filesystem_class("mock").__name__ == "DummyTestFS"
    >>> assert type(fs).__name__ == "DummyTestFS"
    >>> print(fs.glob("**"))
    ["data", "data/train.txt", "data.test.txt"]
    ```
    """

    dir_paths = {file_path.rsplit("/", 1)[0] for file_path in file_paths if "/" in file_path}
    fs_contents = [{"name": dir_path, "type": "directory"} for dir_path in dir_paths] + [
        {"name": file_path, "type": "file", "size": 10} for file_path in file_paths
    ]

    class DummyTestFS(AbstractFileSystem):
        protocol = ("mock", "dummy")
        _fs_contents = fs_contents

        def ls(self, path, detail=True, refresh=True, **kwargs):
            if kwargs.pop("strip_proto", True):
                path = self._strip_protocol(path)

            files = not refresh and self._ls_from_cache(path)
            if not files:
                files = [file for file in self._fs_contents if path == self._parent(file["name"])]
                files.sort(key=lambda file: file["name"])