Repository: PyFilesystem/pyfilesystem2

Issue:
Expose SubFS factory
There are some custom methods (i.e. not overrides) on GoogleFS and OneDriveFS to expose specific features of those filesystems.

My problem is that makedir(s) returns a SubFS which then doesn't support those custom methods so you can't use it transparently (e.g. in the tests)
I see that opendir has a factory parameter so that you can customize the return type, but that's missing from the other SubFS-returning functions.
I'd suggest that we could add the same factory parameter to makedir(s), except that it seems to me that it shouldn't be specified by the user but by the filesystem developer - maybe in the _meta dictionary?
I suppose that another way to do it would be to use the custom factory if the user specifies one but if factory is None, use the GoogleFS-specified one.

What do you think?

Comments:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index aaf125a..b36f9a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.4.9] - (Unreleased)
+
+### Changed
+
+- `MemFS` now immediately releases all memory it holds when `close()` is called,
+  rather than when it gets garbage collected. Closes [issue #308](https://github.com/PyFilesystem/pyfilesystem2/issues/308).
+
 ## [2.4.8] - 2019-06-12
 
 ### Changed
diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
index 5c2a9b5..ef1d3a0 100644
--- a/CONTRIBUTORS.md
+++ b/CONTRIBUTORS.md
@@ -2,6 +2,7 @@
 
 Many thanks to the following developers for contributing to this project:
 
+- [Diego Argueta](https://github.com/dargueta)
 - [Geoff Jukes](https://github.com/geoffjukes)
 - [Giampaolo](https://github.com/gpcimino)
 - [Martin Larralde](https://github.com/althonos)
diff --git a/fs/base.py b/fs/base.py
index 25f6db4..9e1f858 100644
--- a/fs/base.py
+++ b/fs/base.py
@@ -101,6 +101,9 @@ class FS(object):
     # most FS will use default walking algorithms
     walker_class = Walker
 
+    # default to SubFS, used by opendir and should be returned by makedir(s)
+    subfs_class = None
+
     def __init__(self):
         # type: (...) -> None
         """Create a filesystem. See help(type(self)) for accurate signature.
@@ -1180,7 +1183,7 @@ class FS(object):
             factory (callable, optional): A callable that when invoked
                 with an FS instance and ``path`` will return a new FS object
                 representing the sub-directory contents. If no ``factory``
-                is supplied then `~fs.subfs.SubFS` will be used.
+                is supplied then `~fs.subfs_class` will be used.
 
         Returns:
             ~fs.subfs.SubFS: A filesystem representing a sub-directory.
@@ -1192,7 +1195,7 @@ class FS(object):
         """
         from .subfs import SubFS
 
-        _factory = factory or SubFS
+        _factory = factory or self.subfs_class or SubFS
 
         if not self.getbasic(path).is_dir:
             raise errors.DirectoryExpected(path=path)
diff --git a/fs/memoryfs.py b/fs/memoryfs.py
index 2446b0d..dcee49d 100644
--- a/fs/memoryfs.py
+++ b/fs/memoryfs.py
@@ -340,6 +340,11 @@ class MemoryFS(FS):
                 current_entry = current_entry.get_entry(path_component)
             return current_entry
 
+    def close(self):
+        # type: () -> None
+        self.root = None
+        return super(MemoryFS, self).close()
+
     def getinfo(self, path, namespaces=None):
         # type: (Text, Optional[Collection[Text]]) -> Info
         namespaces = namespaces or ()


Test patch:
diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py
index 8f8adbd..aaf6e77 100644
--- a/tests/test_memoryfs.py
+++ b/tests/test_memoryfs.py
@@ -1,9 +1,17 @@
 from __future__ import unicode_literals
 
+import posixpath
 import unittest
 
 from fs import memoryfs
 from fs.test import FSTestCases
+from fs.test import UNICODE_TEXT
+
+try:
+    # Only supported on Python 3.4+
+    import tracemalloc
+except ImportError:
+    tracemalloc = None
 
 
 class TestMemoryFS(FSTestCases, unittest.TestCase):
@@ -11,3 +19,50 @@ class TestMemoryFS(FSTestCases, unittest.TestCase):
 
     def make_fs(self):
         return memoryfs.MemoryFS()
+
+    def _create_many_files(self):
+        for parent_dir in {"/", "/one", "/one/two", "/one/other-two/three"}:
+            self.fs.makedirs(parent_dir, recreate=True)
+            for file_id in range(50):
+                self.fs.writetext(
+                    posixpath.join(parent_dir, str(file_id)), UNICODE_TEXT
+                )
+
+    @unittest.skipIf(
+        not tracemalloc, "`tracemalloc` isn't supported on this Python version."
+    )
+    def test_close_mem_free(self):
+        """Ensure all file memory is freed when calling close().
+
+        Prevents regression against issue #308.
+        """
+        trace_filters = [tracemalloc.Filter(True, "*/memoryfs.py")]
+        tracemalloc.start()
+
+        before = tracemalloc.take_snapshot().filter_traces(trace_filters)
+        self._create_many_files()
+        after_create = tracemalloc.take_snapshot().filter_traces(trace_filters)
+
+        self.fs.close()
+        after_close = tracemalloc.take_snapshot().filter_traces(trace_filters)
+        tracemalloc.stop()
+
+        [diff_create] = after_create.compare_to(
+            before, key_type="filename", cumulative=True
+        )
+        self.assertGreater(
+            diff_create.size_diff,
+            0,
+            "Memory usage didn't increase after creating files; diff is %0.2f KiB."
+            % (diff_create.size_diff / 1024.0),
+        )
+
+        [diff_close] = after_close.compare_to(
+            after_create, key_type="filename", cumulative=True
+        )
+        self.assertLess(
+            diff_close.size_diff,
+            0,
+            "Memory usage increased after closing the file system; diff is %0.2f KiB."
+            % (diff_close.size_diff / 1024.0),
+        )
diff --git a/tests/test_subfs.py b/tests/test_subfs.py
index 633048f..c360431 100644
--- a/tests/test_subfs.py
+++ b/tests/test_subfs.py
@@ -3,8 +3,11 @@ from __future__ import unicode_literals
 import os
 import shutil
 import tempfile
+import unittest
 
 from fs import osfs
+from fs.subfs import SubFS
+from fs.memoryfs import MemoryFS
 from fs.path import relpath
 from .test_osfs import TestOSFS
 
@@ -26,3 +29,44 @@ class TestSubFS(TestOSFS):
     def _get_real_path(self, path):
         _path = os.path.join(self.temp_dir, "__subdir__", relpath(path))
         return _path
+
+
+class CustomSubFS(SubFS):
+    """Just a custom class to change the type"""
+
+    def custom_function(self, custom_path):
+        fs, delegate_path = self.delegate_path(custom_path)
+        fs.custom_function(delegate_path)
+
+
+class CustomSubFS2(SubFS):
+    """Just a custom class to change the type"""
+
+
+class CustomFS(MemoryFS):
+    subfs_class = CustomSubFS
+
+    def __init__(self):
+        super(CustomFS, self).__init__()
+        self.custom_path = None
+
+    def custom_function(self, custom_path):
+        self.custom_path = custom_path
+
+
+class TestCustomSubFS(unittest.TestCase):
+    """Test customization of the SubFS returned from opendir etc"""
+
+    def test_opendir(self):
+        fs = CustomFS()
+        fs.makedir("__subdir__")
+        subfs = fs.opendir("__subdir__")
+        # By default, you get the fs's defined custom SubFS
+        assert isinstance(subfs, CustomSubFS)
+
+        subfs.custom_function("filename")
+        assert fs.custom_path == "/__subdir__/filename"
+
+        # Providing the factory explicitly still works
+        subfs = fs.opendir("__subdir__", factory=CustomSubFS2)
+        assert isinstance(subfs, CustomSubFS2)