Repository: PyFilesystem/pyfilesystem2

Issue:
Evaluated options for opener discovery
I'm not sure this is indeed an `fs` issue, but perhaps it's at least good to have it documented here, too.

We're running into a problem with embedding `fs` into an application for MacOS with py2app, where the opener registry can't find anything, because the `entry_points.txt` file is not copied into the application, so `pkg_resources.iter_entry_points("fs.opener")` yields nothing.

See the related issue for py2app:
https://bitbucket.org/ronaldoussoren/py2app/issues/256/fs-module-not-fully-working-from-app

It seems the use of package metadata like `entry_points.txt` is problematic, and I wonder if there isn't a more friendly way to populate a registry such as `fs.opener`.

Cc @typemytype 

Discussion:
justvanrossum: Wow fantastic! I'll see if I can test this locally soon. Cc @typemytype.
willmcgugan: Thanks @althonos Give me a few days to look this over.
justvanrossum: Sorry for my slow reply. I just tested this and it works just fine! +1 from me to merge this. Thanks so much for working on this.
willmcgugan: Thanks @althonos 

Patch:
diff --git a/fs/opener/__init__.py b/fs/opener/__init__.py
index 8f2a264..91870e7 100644
--- a/fs/opener/__init__.py
+++ b/fs/opener/__init__.py
@@ -10,6 +10,9 @@ from .base import Opener
 from .parse import parse_fs_url as parse
 from .registry import registry
 
+# Import opener modules so that `registry.install` if called on each opener
+from . import appfs, ftpfs, memoryfs, osfs, tarfs, tempfs, zipfs
+
 # Alias functions defined as Registry methods
 open_fs = registry.open_fs
 open = registry.open
diff --git a/fs/opener/appfs.py b/fs/opener/appfs.py
index 370285f..4bae42c 100644
--- a/fs/opener/appfs.py
+++ b/fs/opener/appfs.py
@@ -9,9 +9,8 @@ from __future__ import unicode_literals
 import typing
 
 from .base import Opener
+from .registry import registry
 from .errors import OpenerError
-from ..subfs import ClosingSubFS
-from .. import appfs
 
 if False:  # typing.TYPE_CHECKING
     from typing import Text, Union
@@ -20,20 +19,13 @@ if False:  # typing.TYPE_CHECKING
     from ..subfs import SubFS
 
 
+@registry.install
 class AppFSOpener(Opener):
     """``AppFS`` opener.
     """
 
     protocols = ["userdata", "userconf", "sitedata", "siteconf", "usercache", "userlog"]
-
-    _protocol_mapping = {
-        "userdata": appfs.UserDataFS,
-        "userconf": appfs.UserConfigFS,
-        "sitedata": appfs.SiteDataFS,
-        "siteconf": appfs.SiteConfigFS,
-        "usercache": appfs.UserCacheFS,
-        "userlog": appfs.UserLogFS,
-    }
+    _protocol_mapping = None
 
     def open_fs(
         self,
@@ -44,6 +36,20 @@ class AppFSOpener(Opener):
         cwd,  # type: Text
     ):
         # type: (...) -> Union[_AppFS, SubFS[_AppFS]]
+
+        from ..subfs import ClosingSubFS
+        from .. import appfs
+
+        if self._protocol_mapping is None:
+            self._protocol_mapping = {
+                "userdata": appfs.UserDataFS,
+                "userconf": appfs.UserConfigFS,
+                "sitedata": appfs.SiteDataFS,
+                "siteconf": appfs.SiteConfigFS,
+                "usercache": appfs.UserCacheFS,
+                "userlog": appfs.UserLogFS,
+            }
+
         fs_class = self._protocol_mapping[parse_result.protocol]
         resource, delim, path = parse_result.resource.partition("/")
         tokens = resource.split(":", 3)
diff --git a/fs/opener/ftpfs.py b/fs/opener/ftpfs.py
index 91ea596..4c30c96 100644
--- a/fs/opener/ftpfs.py
+++ b/fs/opener/ftpfs.py
@@ -11,6 +11,7 @@ import six
 import typing
 
 from .base import Opener
+from .registry import registry
 from ..errors import FSError, CreateFailed
 
 if False:  # typing.TYPE_CHECKING
@@ -20,6 +21,7 @@ if False:  # typing.TYPE_CHECKING
     from .parse import ParseResult
 
 
+@registry.install
 class FTPOpener(Opener):
     """`FTPFS` opener.
     """
diff --git a/fs/opener/memoryfs.py b/fs/opener/memoryfs.py
index 09be5aa..8b8976c 100644
--- a/fs/opener/memoryfs.py
+++ b/fs/opener/memoryfs.py
@@ -9,6 +9,7 @@ from __future__ import unicode_literals
 import typing
 
 from .base import Opener
+from .registry import registry
 
 if False:  # typing.TYPE_CHECKING
     from typing import Text
@@ -16,6 +17,7 @@ if False:  # typing.TYPE_CHECKING
     from ..memoryfs import MemoryFS
 
 
+@registry.install
 class MemOpener(Opener):
     """`MemoryFS` opener.
     """
diff --git a/fs/opener/osfs.py b/fs/opener/osfs.py
index f2e4331..986de24 100644
--- a/fs/opener/osfs.py
+++ b/fs/opener/osfs.py
@@ -9,6 +9,7 @@ from __future__ import unicode_literals
 import typing
 
 from .base import Opener
+from .registry import registry
 
 if False:  # typing.TYPE_CHECKING
     from typing import Text
@@ -16,6 +17,7 @@ if False:  # typing.TYPE_CHECKING
     from ..osfs import OSFS
 
 
+@registry.install
 class OSFSOpener(Opener):
     """`OSFS` opener.
     """
diff --git a/fs/opener/registry.py b/fs/opener/registry.py
index e2d5ac9..25a4fa1 100644
--- a/fs/opener/registry.py
+++ b/fs/opener/registry.py
@@ -6,6 +6,7 @@ from __future__ import absolute_import
 from __future__ import print_function
 from __future__ import unicode_literals
 
+import collections
 import contextlib
 import typing
 
@@ -17,7 +18,7 @@ from .errors import UnsupportedProtocol, EntryPointError
 from .parse import parse_fs_url
 
 if False:  # typing.TYPE_CHECKING
-    from typing import Iterator, List, Text, Tuple, Union
+    from typing import Iterator, List, Optional, Text, Tuple, Union
     from ..base import FS
 
 
@@ -25,34 +26,58 @@ class Registry(object):
     """A registry for `Opener` instances.
     """
 
-    def __init__(self, default_opener="osfs"):
-        # type: (Text) -> None
+    def __init__(self, default_opener="osfs", load_extern=False):
+        # type: (Text, bool) -> None
         """Create a registry object.
 
         Arguments:
             default_opener (str, optional): The protocol to use, if one
                 is not supplied. The default is to use 'osfs', so that the
                 FS URL is treated as a system path if no protocol is given.
+            load_extern (bool, optional): Set to `True` to load openers from
+                Pyfilesystem2 extensions. Defaults to `False`.
 
         """
         self.default_opener = default_opener
-        self._protocols = None  # type: Union[None, List[Text]]
+        self.load_extern = load_extern
+        self._protocols = {}  # type: Mapping[Text, Opener]
 
     def __repr__(self):
         # type: () -> Text
         return "<fs-registry {!r}>".format(self.protocols)
 
+    def install(self, opener):
+        # type: (Union[Type[Opener], Opener, Callable[[], Opener]]) -> None
+        """Install an opener.
+
+        Arguments:
+            opener (`Opener`): an `Opener` instance, or a callable that
+                returns an opener instance.
+
+        Note:
+            May be used as a class decorator. For example::
+                registry = Registry()
+                @registry.install
+                class ArchiveOpener(Opener):
+                    protocols = ['zip', 'tar']
+        """
+        if not isinstance(opener, Opener):
+            opener = opener()
+        assert opener.protocols, "must list one or more protocols"
+        for protocol in opener.protocols:
+            self._protocols[protocol] = opener
+
     @property
     def protocols(self):
         # type: () -> List[Text]
         """`list`: the list of supported protocols.
         """
-        if self._protocols is None:
-            self._protocols = [
-                entry_point.name
-                for entry_point in pkg_resources.iter_entry_points("fs.opener")
-            ]
-        return self._protocols
+        # we use OrderedDict to build an ordered set of protocols
+        _protocols = collections.OrderedDict((k, None) for k in self._protocols)
+        if self.load_extern:
+            for entry_point in pkg_resources.iter_entry_points("fs.opener"):
+                _protocols[entry_point.name] = None
+        return list(_protocols.keys())
 
     def get_opener(self, protocol):
         # type: (Text) -> Opener
@@ -73,29 +98,41 @@ class Registry(object):
 
         """
         protocol = protocol or self.default_opener
-        entry_point = next(pkg_resources.iter_entry_points("fs.opener", protocol), None)
 
-        if entry_point is None:
-            raise UnsupportedProtocol("protocol '{}' is not supported".format(protocol))
-
-        try:
-            opener = entry_point.load()
-        except Exception as exception:
-            six.raise_from(
-                EntryPointError("could not load entry point; {}".format(exception)),
-                exception,
+        if self.load_extern:
+            entry_point = next(
+                pkg_resources.iter_entry_points("fs.opener", protocol), None
             )
         else:
+            entry_point = None
+
+        # If not entry point was loaded from the extensions, try looking
+        # into the registered protocols
+        if entry_point is None:
+            if protocol in self._protocols:
+                opener_instance = self._protocols[protocol]
+            else:
+                raise UnsupportedProtocol(
+                    "protocol '{}' is not supported".format(protocol)
+                )
+
+        # If an entry point was found in an extension, attempt to load it
+        else:
+            try:
+                opener = entry_point.load()
+            except Exception as exception:
+                raise EntryPointError(
+                    "could not load entry point; {}".format(exception)
+                )
             if not issubclass(opener, Opener):
                 raise EntryPointError("entry point did not return an opener")
 
-        try:
-            opener_instance = opener()
-        except Exception as exception:
-            six.raise_from(
-                EntryPointError("could not instantiate opener; {}".format(exception)),
-                exception,
-            )
+            try:
+                opener_instance = opener()
+            except Exception as exception:
+                raise EntryPointError(
+                    "could not instantiate opener; {}".format(exception)
+                )
 
         return opener_instance
 
@@ -234,4 +271,4 @@ class Registry(object):
                 _fs.close()
 
 
-registry = Registry()
+registry = Registry(load_extern=True)
diff --git a/fs/opener/tarfs.py b/fs/opener/tarfs.py
index 9d60b81..867ca80 100644
--- a/fs/opener/tarfs.py
+++ b/fs/opener/tarfs.py
@@ -9,6 +9,7 @@ from __future__ import unicode_literals
 import typing
 
 from .base import Opener
+from .registry import registry
 from .errors import NotWriteable
 
 if False:  # typing.TYPE_CHECKING
@@ -17,6 +18,7 @@ if False:  # typing.TYPE_CHECKING
     from ..tarfs import TarFS
 
 
+@registry.install
 class TarOpener(Opener):
     """`TarFS` opener.
     """
diff --git a/fs/opener/tempfs.py b/fs/opener/tempfs.py
index e0e4dbf..5fe47a0 100644
--- a/fs/opener/tempfs.py
+++ b/fs/opener/tempfs.py
@@ -9,6 +9,7 @@ from __future__ import unicode_literals
 import typing
 
 from .base import Opener
+from .registry import registry
 
 if False:  # typing.TYPE_CHECKING
     from typing import Text
@@ -16,6 +17,7 @@ if False:  # typing.TYPE_CHECKING
     from ..tempfs import TempFS
 
 
+@registry.install
 class TempOpener(Opener):
     """`TempFS` opener.
     """
diff --git a/fs/opener/zipfs.py b/fs/opener/zipfs.py
index a7e68db..714fe38 100644
--- a/fs/opener/zipfs.py
+++ b/fs/opener/zipfs.py
@@ -9,6 +9,7 @@ from __future__ import unicode_literals
 import typing
 
 from .base import Opener
+from .registry import registry
 from .errors import NotWriteable
 
 if False:  # typing.TYPE_CHECKING
@@ -17,6 +18,7 @@ if False:  # typing.TYPE_CHECKING
     from ..zipfs import ZipFS
 
 
+@registry.install
 class ZipOpener(Opener):
     """`ZipFS` opener.
     """
diff --git a/setup.py b/setup.py
index b31a7d5..13a7161 100644
--- a/setup.py
+++ b/setup.py
@@ -39,22 +39,6 @@ setup(
         ":python_version < '3.6'": ['typing~=3.6'],
         ":python_version < '3.0'": ['backports.os~=0.1']
     },
-    entry_points={'fs.opener': [
-        'ftp  = fs.opener.ftpfs:FTPOpener',
-        'file = fs.opener.osfs:OSFSOpener',
-        'osfs = fs.opener.osfs:OSFSOpener',
-        'mem  = fs.opener.memoryfs:MemOpener',
-        'tar  = fs.opener.tarfs:TarOpener',
-        'temp = fs.opener.tempfs:TempOpener',
-        'zip  = fs.opener.zipfs:ZipOpener',
-
-        'userdata = fs.opener.appfs:AppFSOpener',
-        'userconf = fs.opener.appfs:AppFSOpener',
-        'sitedata = fs.opener.appfs:AppFSOpener',
-        'siteconf = fs.opener.appfs:AppFSOpener',
-        'usercache = fs.opener.appfs:AppFSOpener',
-        'userlog = fs.opener.appfs:AppFSOpener',
-    ]},
     license="MIT",
     name='fs',
     packages=find_packages(exclude=("tests",)),


Test patch:
diff --git a/tests/test_opener.py b/tests/test_opener.py
index 5a3e526..cc0232e 100644
--- a/tests/test_opener.py
+++ b/tests/test_opener.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import os
 import mock
+import sys
 import tempfile
 import unittest
 import pkg_resources
@@ -101,14 +102,17 @@ class TestParse(unittest.TestCase):
 
 class TestRegistry(unittest.TestCase):
     def test_registry_protocols(self):
-        # Check registry.protocols list the names of all available entry points
-
-        protocols = [
-            entry_point.name
-            for entry_point in pkg_resources.iter_entry_points("fs.opener")
+        # Check registry.protocols list the names of all available extension
+        extensions = [
+            pkg_resources.EntryPoint("proto1", "mod1"),
+            pkg_resources.EntryPoint("proto2", "mod2"),
         ]
-
-        self.assertEqual(sorted(protocols), sorted(opener.registry.protocols))
+        m = mock.MagicMock(return_value=extensions)
+        with mock.patch.object(
+            sys.modules["pkg_resources"], "iter_entry_points", new=m
+        ):
+            self.assertIn("proto1", opener.registry.protocols)
+            self.assertIn("proto2", opener.registry.protocols)
 
     def test_unknown_protocol(self):
         with self.assertRaises(errors.UnsupportedProtocol):