Repository: intake/intake
Pull request #636 (2021-12-23 18:10:10)

Config / registry
Trying to greatly simplify the config and driver registry systems, in particular not to have separate paths for registering drivers via the CLI versus via code.

@danielballan , this is largely stepping on your previous attempt to retialise these things. We could take this opportunity to remove package scanning altogether.

Ref #635 


Patch:
diff --git a/intake/__init__.py b/intake/__init__.py
index 6857987..b82298c 100644
--- a/intake/__init__.py
+++ b/intake/__init__.py
@@ -15,7 +15,7 @@ from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions
 
-from .source import register_driver, registry
+from .source import registry
 from .catalog.base import Catalog
 
 imports = {
@@ -29,7 +29,7 @@ imports = {
     "register_driver": "intake.source:register_driver",
     "unregister_driver": "intake.source:unregister_driver",
 }
-openers = {}
+openers = set()
 logger = logging.getLogger('intake')
 
 
@@ -41,7 +41,7 @@ def __getattr__(attr):
     """
     gl = globals()
     if attr in openers:
-        driver = openers[attr].load()
+        driver = registry[attr[5:]]  # "open_..."
         gl[attr] = driver
     else:
         if attr in gl:
@@ -69,21 +69,19 @@ def __dir__(*_, **__):
 
 def make_open_functions():
     """From the current state of ``registry``, create open_* functions"""
-    import entrypoints
+    from .source.discovery import drivers
 
-    for plugin in entrypoints.get_group_all('intake.drivers'):
+    for name in drivers.enabled_plugins():
 
-        register_driver(plugin.name, plugin, True)
-        func_name = 'open_' + plugin.name
+        func_name = 'open_' + name
         if not func_name.isidentifier():
             # primitive name normalization
             func_name = re.sub('[-=~^&|@+]', '_', func_name)
         if func_name.isidentifier():
             # stash name for dir() and later fetch
-            openers[func_name] = plugin
+            openers.add(func_name)
         else:
-            warnings.warn('Invalid Intake plugin name "%s" found.' %
-                          plugin.name)
+            warnings.warn('Invalid Intake plugin name "%s" found.', name)
 
 
 make_open_functions()
diff --git a/intake/catalog/local.py b/intake/catalog/local.py
index e89c193..555eb92 100644
--- a/intake/catalog/local.py
+++ b/intake/catalog/local.py
@@ -530,7 +530,8 @@ def register_plugin_module(mod):
         if k:
             if isinstance(k, (list, tuple)):
                 k = k[0]
-            register_driver(k, v)
+            # we clobber by default here
+            register_driver(k, v, clobber=True)
 
 
 def get_dir(path):
@@ -888,5 +889,5 @@ class EntrypointsCatalog(Catalog):
 # Register these early in the import process to support the default catalog
 # which is built at import time. (Without this, 'yaml_file_cat' is looked for
 # in intake.registry before the registry has been populated.)
-register_driver('yaml_file_cat', YAMLFileCatalog)
-register_driver('yaml_files_cat', YAMLFilesCatalog)
+register_driver('yaml_file_cat', YAMLFileCatalog, clobber=True)
+register_driver('yaml_files_cat', YAMLFilesCatalog, clobber=True)
diff --git a/intake/catalog/remote.py b/intake/catalog/remote.py
index 9406c46..61c2c87 100644
--- a/intake/catalog/remote.py
+++ b/intake/catalog/remote.py
@@ -480,7 +480,7 @@ def open_remote(url, entry, container, user_parameters, description, http_args,
     payload = dict(action='open',
                    name=entry,
                    parameters=user_parameters,
-                   available_plugins=list(plugin_registry.keys()))
+                   available_plugins=list(plugin_registry))
     req = requests.post(urljoin(url, '/v1/source'),
                         data=msgpack.packb(payload, **pack_kwargs),
                         **http_args)
diff --git a/intake/cli/client/subcommands/config.py b/intake/cli/client/subcommands/config.py
index ef5cba4..94812de 100644
--- a/intake/cli/client/subcommands/config.py
+++ b/intake/cli/client/subcommands/config.py
@@ -74,6 +74,6 @@ class Config(Subcommand):
         print(yaml.dump(defaults, default_flow_style=False))
 
     def _reset(self, args):
-        from intake.config import reset_conf, save_conf
-        reset_conf()
-        save_conf()
+        from intake.config import conf
+        conf.reset()
+        conf.save()
diff --git a/intake/cli/client/subcommands/drivers.py b/intake/cli/client/subcommands/drivers.py
index e7147f5..b82b272 100644
--- a/intake/cli/client/subcommands/drivers.py
+++ b/intake/cli/client/subcommands/drivers.py
@@ -8,23 +8,9 @@
 CLI for listing, enabling, disabling intake drivers
 """
 
-
-# -----------------------------------------------------------------------------
-# Imports
-# -----------------------------------------------------------------------------
-
-# Standard library imports
-import inspect
-import sys
-
-# External imports
-
-# Intake imports
 from intake import __version__
 from intake.cli.util import Subcommand
-from intake.source.discovery import autodiscover, autodiscover_all, enable, disable
-from intake.config import confdir
-import intake
+from intake.source.discovery import drivers
 
 import logging
 log = logging.getLogger(__name__)
@@ -55,7 +41,7 @@ class Drivers(Subcommand):
 
         enable = sub_parser.add_parser('enable', help='Enable an intake driver.')
         enable.add_argument('name', type=str, help='Driver name')
-        enable.add_argument('driver', type=str,
+        enable.add_argument('driver', type=str, default=None, nargs='?',
                             help='Module path and class name, as in '
                                  'package.submodule.ClassName')
         enable.set_defaults(invoke=self._enable)
@@ -69,53 +55,38 @@ class Drivers(Subcommand):
         self.parser.print_help()
 
     def _list(self, args):
-        if args.verbose:
-            fmt = '{name:<30}{cls.__module__}.{cls.__name__} @ {file}'
+        if drivers.do_scan:
+            print("Package scan:")
+            for k, v in drivers.scanned.items():
+                print(f'{k:<30}{v.__module__}.{v.__name__}')
+            print()
+
+        print("Entrypoints:")
+        eps = [ep for ep in drivers.from_entrypoints()
+               if ep.name not in drivers.disabled()]
+        if eps:
+            for v in eps:
+                print(f'{v.name:<30}{v.module_name}:{v.object_name}')
         else:
-            fmt = '{name:<30}{cls.__module__}.{cls.__name__}'
-        drivers_by_name = autodiscover()   # dict mapping name to driver
-        all_drivers = autodiscover_all()  # listof (name, driver)
-        direct = {}
-        for k in intake.registry:
-            if k not in all_drivers and k not in drivers_by_name:
-                try:
-                    direct[k] = intake.registry[k]
-                except ImportError:
-                    pass
-
-        print("Direct:", file=sys.stderr)
-        none = True
-        for name in sorted(direct, key=str):
-            cls = direct[name]
-            print(fmt.format(name=str(name), cls=cls, file=inspect.getfile(cls)),
-                  file=sys.stderr)
-            none = False
-        if none:
             print("<none>")
-
-        print("\nEnabled:", file=sys.stderr)
-        none = True
-        for name in sorted(drivers_by_name, key=str):
-            cls = drivers_by_name[name]
-            print(fmt.format(name=str(name), cls=cls, file=inspect.getfile(cls)),
-                  file=sys.stderr)
-            none = False
-        if none:
+        print()
+
+        print("From Config:")
+        eps = [ep for ep in drivers.from_conf()
+               if ep.name not in drivers.disabled()]
+        if eps:
+            for v in eps:
+                if v.name not in drivers.disabled():
+                    print(f'{v.name:<30}{v.module_name}:{v.object_name}')
+        else:
             print("<none>")
+        print()
 
-        print("\nNot enabled:", file=sys.stderr)
-        none = True
-        for name, cls in sorted(all_drivers, key=lambda x: str(x[0])):
-            if drivers_by_name.get(name, None) is not cls:
-                print(fmt.format(name=str(name), cls=cls, file=inspect.getfile(cls)),
-                      file=sys.stderr)
-                none = False
-        if none:
-            print("<none>")
+        print("Disabled: ", drivers.disabled() or "<none>")
 
     def _enable(self, args):
-        enable(args.name, args.driver)
+        drivers.enable(args.name, args.driver)
 
     def _disable(self, args):
         for name in args.names:
-            disable(name)
+            drivers.disable(name)
diff --git a/intake/config.py b/intake/config.py
index 060401e..21149ab 100644
--- a/intake/config.py
+++ b/intake/config.py
@@ -27,9 +27,9 @@ defaults = {
     'cache_download_progress': True,
     'logging': 'INFO',
     'catalog_path': [],
-    'persist_path': posixpath.join(confdir, 'persisted')
-    }
-conf = {}
+    'persist_path': posixpath.join(confdir, 'persisted'),
+    'package_scan': False
+}
 
 
 def cfile():
@@ -37,43 +37,67 @@ def cfile():
         os.getenv('INTAKE_CONF_FILE', posixpath.join(confdir, 'conf.yaml')))
 
 
-def reset_conf():
-    """Set conf values back to defaults"""
-    conf.clear()
-    conf.update(defaults)
-
-
-def save_conf(fn=None):
-    """Save current configuration to file as YAML
-
-    If not given, uses current config directory, ``confdir``, which can be
-    set by INTAKE_CONF_DIR.
-    """
-    if fn is None:
-        fn = cfile()
-    try:
-        os.makedirs(os.path.dirname(fn))
-    except (OSError, IOError):
-        pass
-    with open(fn, 'w') as f:
-        yaml.dump(conf, f)
-
-
-def load_conf(fn=None):
-    """Update global config from YAML file
-
-    If fn is None, looks in global config directory, which is either defined
-    by the INTAKE_CONF_DIR env-var or is ~/.intake/ .
-    """
-    if fn is None:
-        fn = cfile()
-    if os.path.isfile(fn):
-        with open(fn) as f:
-            try:
-                conf.update(yaml_load(f))
-            except Exception as e:
-                logger.warning('Failure to load config file "{fn}": {e}'
-                               ''.format(fn=fn, e=e))
+class Config(dict):
+    def __init__(self, filename=None, **kwargs):
+        self.filename = filename if filename is not None else cfile()
+        self.reload_all()
+        super().__init__(**kwargs)
+
+    def reset(self):
+        """Set conf values back to defaults"""
+        self.clear()
+        self.update(defaults)
+
+    def save(self):
+        """Save current configuration to file as YAML
+
+        Uses ``.filename`` for target location
+        """
+        if self.filename is False:
+            return
+        try:
+            os.makedirs(os.path.dirname(self.filename))
+        except (OSError, IOError):
+            pass
+        with open(self.filename, 'w') as f:
+            yaml.dump(dict(self), f)
+
+    def reload_all(self):
+        self.reset()
+        self.load()
+        self.load_env()
+
+    def load(self, fn=None):
+        """Update global config from YAML file
+
+        If fn is None, looks in global config directory, which is either defined
+        by the INTAKE_CONF_DIR env-var or is ~/.intake/ .
+        """
+        fn = fn or self.filename
+
+        if os.path.isfile(fn):
+            with open(fn) as f:
+                try:
+                    self.update(yaml_load(f))
+                except Exception as e:
+                    logger.warning('Failure to load config file "{fn}": {e}'
+                                   ''.format(fn=fn, e=e))
+
+    def load_env(self):
+        """Analyse environment variables and update conf accordingly"""
+        # environment variables take precedence over conf file
+        for key, envvar in [['cache_dir', 'INTAKE_CACHE_DIR'],
+                            ['catalog_path', 'INTAKE_PATH'],
+                            ['persist_path', 'INTAKE_PERSIST_PATH']]:
+            if envvar in os.environ:
+                self[key] = make_path_posix(os.environ[envvar])
+        self['catalog_path'] = intake_path_dirs(self['catalog_path'])
+        for key, envvar in [['cache_disabled', 'INTAKE_DISABLE_CACHING'],
+                            ['cache_download_progress', 'INTAKE_CACHE_PROGRESS']]:
+            if envvar in os.environ:
+                self[key] = os.environ[envvar].lower() in ['true', 't', 'y', 'yes']
+        if 'INTAKE_LOG_LEVEL' in os.environ:
+            self['logging'] = os.environ['INTAKE_LOG_LEVEL']
 
 
 def intake_path_dirs(path):
@@ -90,30 +114,10 @@ def intake_path_dirs(path):
     return pattern.split(path)
 
 
-def load_env():
-    """Analyse environment variables and update conf accordingly"""
-    # environment variables take precedence over conf file
-    for key, envvar in [['cache_dir', 'INTAKE_CACHE_DIR'],
-                        ['catalog_path', 'INTAKE_PATH'],
-                        ['persist_path', 'INTAKE_PERSIST_PATH']]:
-        if envvar in os.environ:
-            conf[key] = make_path_posix(os.environ[envvar])
-    conf['catalog_path'] = intake_path_dirs(conf['catalog_path'])
-    for key, envvar in [['cache_disabled', 'INTAKE_DISABLE_CACHING'],
-                        ['cache_download_progress', 'INTAKE_CACHE_PROGRESS']]:
-        if envvar in os.environ:
-            conf[key] = os.environ[envvar].lower() in ['true', 't', 'y', 'yes']
-    if 'INTAKE_LOG_LEVEL' in os.environ:
-        conf['logging'] = os.environ['INTAKE_LOG_LEVEL']
-
-
-def reload_all():
-    reset_conf()
-    load_conf(cfile())
-    load_env()
-
-
-reload_all()
+conf = Config()
+conf.reload_all()
+save_conf = conf.save
+load_cond = conf.load
 
 logger.setLevel(conf['logging'].upper())
 ch = logging.StreamHandler()
diff --git a/intake/source/__init__.py b/intake/source/__init__.py
index 62a61c4..9232d12 100644
--- a/intake/source/__init__.py
+++ b/intake/source/__init__.py
@@ -5,79 +5,54 @@
 # The full license is in the LICENSE file, distributed with this software.
 #-----------------------------------------------------------------------------
 
+from collections.abc import MappingView
+
 import entrypoints
 import logging
 
-from ..utils import DriverRegistryView
 from .base import DataSource
+from .discovery import drivers
 
 logger = logging.getLogger('intake')
 
 
-class DriverRegistry(dict):
+class DriverRegistry(MappingView):
     """Dict of driver: DataSource class
 
-    If the value object is a EntryPoint, will load it when accesses, which
+    If the value object is a EntryPoint or str, will load it when accesses, which
     does the import.
     """
+    def __init__(self, drivers_source=drivers):
+        self.drivers = drivers_source
 
     def __getitem__(self, item):
-        if isinstance(super().__getitem__(item), entrypoints.EntryPoint):
-            self[item] = super().__getitem__(item).load()
-        return super().__getitem__(item)
-
+        it = self.drivers.enabled_plugins()[item]
+        if isinstance(it, entrypoints.EntryPoint):
+            return it.load()
+        elif isinstance(it, str):
+            return import_name(it)
+        elif issubclass(it, DataSource):
+            return it
+        raise ValueError
 
-_registry = DriverRegistry()  # internal mutable registry
-registry = DriverRegistryView(_registry)  # public, read-ony wrapper
+    def __iter__(self):
+        return iter(self.drivers.enabled_plugins())
 
+    def keys(self):
+        return list(self)
 
-def register_driver(name, driver, overwrite=False):
-    """
-    Add a driver to intake.registry.
-
-    Parameters
-    ----------
-    name: string
-    driver: DataSource
-    overwrite: bool, optional
-        False by default.
-
-    Raises
-    ------
-    ValueError
-        If name collides with an existing name in the registry and overwrite is
-        False.
-    """
-    if name in _registry and not overwrite:
-        # If we are re-registering the same object, there is no problem.
-        original = _registry[name]
-        if original is driver:
-            return
-        raise ValueError(
-            f"The driver {driver} could not be registered for the "
-            f"name {name} because {_registry[name]} is already "
-            f"registered for that name. Use overwrite=True to force it.")
-    _registry[name] = driver
-
-
-def unregister_driver(name):
-    """
-    Ensure that a given name in the registry is cleared.
+    def __len__(self):
+        return len(self.drivers.enabled_plugins())
 
-    This function is idempotent: if the name does not exist, nothing is done,
-    and the function returns None
+    def __repr__(self):
+        return """<Intake driver registry>"""
 
-    Parameters
-    ----------
-    name: string
+    def __contains__(self, item):
+        return item in self.keys()
 
-    Returns
-    -------
-    driver: DataSource or None
-        Whatever was registered for ``name``, or ``None``
-    """
-    return _registry.pop(name, None)
 
+registry = DriverRegistry()
+register_driver = drivers.register_driver
 
 # A set of fully-qualified package.module.Class mappings
 classes = {}
diff --git a/intake/source/discovery.py b/intake/source/discovery.py
index b239074..af3165f 100644
--- a/intake/source/discovery.py
+++ b/intake/source/discovery.py
@@ -5,238 +5,165 @@
 # The full license is in the LICENSE file, distributed with this software.
 #-----------------------------------------------------------------------------
 
+from collections.abc import MutableMapping
 import pkgutil
 import warnings
 import importlib
 import inspect
-import itertools
 import time
 import logging
 
 import entrypoints
 
-from ..config import save_conf, conf, cfile
+from ..config import conf
 logger = logging.getLogger('intake')
 
 
-def autodiscover(path=None, plugin_prefix='intake_', do_package_scan=False):
-    r"""Discover intake drivers.
-
-    In order of decreasing precedence:
-
-    - Respect the 'drivers' section of the intake configuration file.
-    - Find 'intake.drivers' entrypoints provided by any Python packages in the
-      environment.
-    - Search all packages in the environment for names that begin with
-      ``intake\_``. Import them and scan them for subclasses of
-      ``intake.source.base.DataSourceBase``. This was previously the *only* mechanism
-      for auto-discoverying intake drivers, and it is maintained for backward
-      compatibility.
-
-    Parameters
-    ----------
-    path : str or None
-        Default is ``sys.path``.
-    plugin_prefix : str
-        DEPRECATED. Default is 'intake\_'.
-    do_package_scan : boolean
-        Whether to look for intake source classes in packages named
-        "intake_*". This has been superceded by entrypoints declarations.
-
-    Returns
-    -------
-    drivers : dict
-        Name mapped to driver class.
+class DriverSouces:
     """
-    # Discover drivers via package scan.
-    if do_package_scan:
-        package_scan_results = _package_scan(path, plugin_prefix)
-        if package_scan_results:
-            warnings.warn(
-                "The option `do_package_scan` may be removed in a future release.",
-                PendingDeprecationWarning)
-    else:
-        package_scan_results = {}
-
-    # Discover drivers via entrypoints.
-    group = entrypoints.get_group_named('intake.drivers', path=path)
-    group_all = entrypoints.get_group_all('intake.drivers', path=path)
-    if len(group_all) != len(group):
-        # There are some name collisions. Let's go digging for them.
-        for name, matches in itertools.groupby(group_all, lambda ep: ep.name):
-            matches = list(matches)
-            if len(matches) != 1:
-                winner = group[name]
-                logger.debug(
-                    "There are %d 'intake.driver' entrypoints for the name "
-                    "%r. They are %r. The match %r has won the race.",
-                    len(matches),
-                    name,
-                    matches,
-                    winner)
-
-    for name, entrypoint in group.items():
-        logger.debug("Discovered entrypoint '%s = %s.%s'",
-                     name,
-                     entrypoint.module_name,
-                     entrypoint.object_name)
-        if name in package_scan_results:
-            cls = package_scan_results[name]
-            del package_scan_results[name]
-            logger.debug("Entrypoint shadowed package_scan result '%s = %s.%s'",
-                         name, cls.__module__, cls.__name__)
-
-    # Discover drivers via config.
-    drivers_conf = conf.get('drivers', {})
-    logger.debug("Using configuration file at %s", cfile())
-    for name, dotted_object_name in drivers_conf.items():
-        if not dotted_object_name:
-            logger.debug('Name %s is banned in config file', name)
-            if name in group:
-                entrypoint = group[name]
-                del group[name]
-                logger.debug("Disabled entrypoint '%s = %s.%s'",
-                             entrypoint.name,
-                             entrypoint.module_name,
-                             entrypoint.object_name)
-            if name in package_scan_results:
-                cls = package_scan_results[name]
-                del package_scan_results[name]
-                logger.debug("Disabled package_scan result '%s = %s.%s'",
-                             name, cls.__module__, cls.__name__)
-            continue
-        module_name, object_name = dotted_object_name.rsplit('.', 1)
-        entrypoint = entrypoints.EntryPoint(name, module_name, object_name)
-        logger.debug("Discovered config-specified '%s = %s.%s'",
-                     entrypoint.name,
-                     entrypoint.module_name,
-                     entrypoint.object_name)
-        if name in group:
-            shadowed = group[name]
-            logger.debug("Config shadowed entrypoint '%s = %s.%s'",
-                         shadowed.name,
-                         shadowed.module_name,
-                         shadowed.object_name)
-        if name in package_scan_results:
-            cls = package_scan_results[name]
-            del package_scan_results[name]
-            logger.debug("Config shadowed package scan result '%s = %s.%s'",
-                         name, cls.__module__, cls.__name__)
-        group[name] = entrypoint
-
-    # Discovery is complete.
-
-    if package_scan_results:
-        warnings.warn(
-            f"The drivers {list(package_scan_results)} do not specify entry_"
-            f"points and were only discovered via a package scan. This may "
-            f"break in a future release of intake. The packages should be "
-            f"updated.",
-            FutureWarning)
-
-    # Load entrypoints. Any that were shadowed or banned have already been
-    # removed above.
-    drivers = {}
-    for entrypoint in group.values():
-        try:
-            drivers[entrypoint.name] = _load_entrypoint(entrypoint)
-        except ConfigurationError:
-            logger.debug(
-                "Error while loading entrypoint %s",
-                entrypoint.name)
-            continue
-        logger.debug("Loaded entrypoint '%s = %s.%s'",
-                     entrypoint.name,
-                     entrypoint.module_name,
-                     entrypoint.object_name)
-
-    # Now include any package scan results. Any that were shadowed or
-    # banned have already been removed above.
-    for name, cls in package_scan_results.items():
-        drivers[name] = cls
-        logger.debug("Loaded package scan result '%s = %s.%s'",
-                     name,
-                     cls.__module__,
-                     cls.__name__)
-
-    return drivers
-
-
-def autodiscover_all(path=None, plugin_prefix='intake_', do_package_scan=True):
-    """Discover intake drivers including those registered for the same name.
-
-    Parameters
-    ----------
-    path : str or None
-        Default is ``sys.path``.
-    plugin_prefix : str
-        DEPRECATED. Default is 'intake_'.
-    do_package_scan : boolean
-        Default is True. In the future, the default will be changed to False,
-        and the option may eventually be removed entirely.
-
-    Returns
-    -------
-    drivers : list
-        Each entry is a tuple: ``(name, driver_class)``.
+    Handles the various ways in which drivers can be known to Intake
     """
-    # Discover drivers via package scan.
-    if do_package_scan:
-        warnings.warn(
-            "The option `do_package_scan` may be removed in a future release.",
-            PendingDeprecationWarning)
-        package_scan_results = _package_scan(path, plugin_prefix)
-    else:
-        package_scan_results = {}
-
-    # Discover drivers via entrypoints.
-    group_all = entrypoints.get_group_all('intake.drivers', path=path)
-    for entrypoint in group_all:
-        logger.debug("Discovered entrypoint '%s = %s.%s'",
-                     entrypoint.name,
-                     entrypoint.module_name,
-                     entrypoint.object_name)
-
-    # Discover drivers via config.
-    drivers_conf = conf.get('drivers', {})
-    logger.debug("Using configuration file at %s", cfile())
-    for name, dotted_object_name in drivers_conf.items():
-        if not dotted_object_name:
-            continue
-        module_name, object_name = dotted_object_name.rsplit('.', 1)
-        entrypoint = entrypoints.EntryPoint(name, module_name, object_name)
-        logger.debug("Discovered config-specified '%s = %s.%s'",
-                     entrypoint.name,
-                     entrypoint.module_name,
-                     entrypoint.object_name)
-        group_all.append(entrypoint)
-
-    # Load entrypoints. Any that were shadowed or banned have already been
-    # removed above.
-    drivers = []
-    for entrypoint in group_all:
-        try:
-            drivers.append((entrypoint.name, _load_entrypoint(entrypoint)))
-        except ConfigurationError:
-            logger.debug(
-                "Error while loading entrypoint %s",
-                entrypoint.name)
-            continue
-        logger.debug("Loaded entrypoint '%s = %s.%s'",
-                     entrypoint.name,
-                     entrypoint.module_name,
-                     entrypoint.object_name)
-
-    # Now include any package scan results. Any that were shadowed or
-    # banned have already been removed above.
-    for name, cls in package_scan_results.items():
-        drivers.append((name, cls))
-        logger.debug("Loaded package scan result '%s = %s.%s'",
-                     name,
-                     cls.__module__,
-                     cls.__name__)
-
-    return drivers
+
+    def __init__(self, config=None, do_scan=None):
+        """
+
+        Parameters
+        ----------
+        config: intake.config.Config oinstance, optional
+            If not given, will use ``intake.config.conf`` singleton
+        do_scan: bool or None
+            Whether to scan packages with names ``intake_*`` for valid drivers.
+            This is for backward compatibility only. If not given, value comes
+            from the config key "package_scan", default False.
+        """
+        self.conf = config or conf
+        self.do_scan = do_scan
+        self._scanned = None
+        self._entrypoints = None
+        self.registered = {}
+
+    @property
+    def package_scan(self):
+        return self.conf.get("package_scan", False) if self.do_scan is None else self.do_scan
+
+    @package_scan.setter
+    def package_scan(self, val):
+        self.conf["package_scan"] = val
+
+    def from_entrypoints(self):
+        if self._entrypoints is None:
+            # must cache this, since lookup if fairly slow and we do this a lot
+            self._entrypoints = list(entrypoints.get_group_all("intake.drivers"))
+        return self._entrypoints
+
+    def from_conf(self):
+        return [
+            entrypoints.EntryPoint(
+                name=k,
+                module_name=v.rsplit(":", 1)[0] if ":" in v else v.rsplit(".", 1)[0],
+                object_name=v.rsplit(":", 1)[1] if ":" in v else v.rsplit(".", 1)[1]
+            )
+            for k, v in self.conf.get("drivers", {}).items() if v
+        ]
+
+    def disabled(self):
+        return [k for k, v in self.conf.get("drivers", {}).items() if v is False]
+
+    def __setitem__(self, key, value):
+        super(DriverSouces, self).__setitem__(key, value)
+        self.save()
+
+    def __delitem__(self, key):
+        super(DriverSouces, self).__delitem__(key)
+        self.save()
+
+    @property
+    def scanned(self):
+        # cache since imports should not change during session
+        if self._scanned is None:
+            # these are already classes
+            self._scanned = _package_scan()
+        return self._scanned
+
+    def enabled_plugins(self):
+        # priority order (decreasing): runtime, config, entrypoints, package scan
+        out = {}
+        if self.package_scan:
+            out.update(self.scanned)
+
+        for ep in self.from_entrypoints() + self.from_conf():
+            out[ep.name] = ep
+
+        out.update(self.registered)
+        out = {k: v for k, v in out.items() if k not in self.disabled()}
+        return out
+
+    def register_driver(self, name, value, clobber=False, do_enable=False):
+        """Add runtime driver definition
+
+        Parameters
+        ----------
+        name: str
+            Name of the driver
+        value: str, entrypoint or class
+            Pointer to the implementation
+        clobber: bool
+            If True, perform the operation even if the driver exists
+        do_enable: bool
+            If True, unset the disabled flag for this driver
+        """
+        if not clobber and name in self.enabled_plugins():
+            raise ValueError(f"Driver {name} already enabled")
+        if name in self.disabled():
+            if do_enable:
+                self.enable(name)
+            else:
+                logger.warning(f"Adding driver {name}, but it is disabled")
+
+        self.registered[name] = value
+
+    def unregister_driver(self, name):
+        """Remove runtime registered driver"""
+        self.registered.pop(name)
+
+    def enable(self, name, driver=None):
+        """
+        Explicitly assign a driver to a name, or remove ban
+
+        Updates the associated config, which will be persisted
+
+        Parameters
+        ----------
+        name : string
+            As in ``'zarr'``
+        driver : string
+            Dotted object name, as in ``'intake_xarray.xzarr.ZarrSource'``.
+            If None, simply remove driver disable flag, if it is found
+        """
+        config = self.conf
+        if 'drivers' not in config:
+            config['drivers'] = {}
+        if driver:
+            config['drivers'][name] = driver
+        elif config['drivers'].get(name) is False:
+            del config['drivers'][name]
+        config.save()
+
+    def disable(self, name):
+        """Disable a driver by name.
+
+        Updates the associated config, which will be persisted
+
+        Parameters
+        ----------
+        name : string
+            As in ``'zarr'``
+        """
+        config = self.conf
+        if 'drivers' not in config:
+            config['drivers'] = {}
+        config['drivers'][name] = False
+        config.save()
 
 
 def _load_entrypoint(entrypoint):
@@ -264,6 +191,7 @@ def _package_scan(path=None, plugin_prefix='intake_'):
     of intake.source.base.Plugin.  Any subclasses found will be instantiated
     and returned in a dictionary, with the plugin's name attribute as the key.
     """
+    warnings.warn("Package scanning may be removed", category=PendingDeprecationWarning)
 
     plugins = {}
 
@@ -288,6 +216,9 @@ def _package_scan(path=None, plugin_prefix='intake_'):
     return plugins
 
 
+drivers = DriverSouces()
+
+
 def load_plugins_from_module(module_name):
     """Imports a module and returns dictionary of discovered Intake plugins.
 
@@ -328,38 +259,3 @@ class ConfigurationError(Exception):
     pass
 
 
-def enable(name, driver):
-    """
-    Update config file drivers section to explicitly assign a driver to a name.
-
-    Parameters
-    ----------
-    name : string
-        As in ``'zarr'``
-    driver : string
-        Dotted object name, as in ``'intake_xarray.xzarr.ZarrSource'``
-    """
-    if 'drivers' not in conf:
-        conf['drivers'] = {}
-    conf['drivers'][name] = driver
-    save_conf()
-
-
-def disable(name):
-    """Update config file drivers section to disable a name.
-
-    Parameters
-    ----------
-    name : string
-        As in ``'zarr'``
-    """
-    if 'drivers' not in conf:
-        conf['drivers'] = {}
-    conf['drivers'][name] = False
-    save_conf()
-
-
-def register_all():
-    from intake.source import register_driver
-    for name, driver in autodiscover().items():
-        register_driver(name, driver)


Test patch:
diff --git a/intake/conftest.py b/intake/conftest.py
index dd3fb07..6a2aa90 100644
--- a/intake/conftest.py
+++ b/intake/conftest.py
@@ -53,7 +53,7 @@ def tmp_config_path(tmp_path):
     os.environ[key] = temp_config_path
     assert config.cfile() == temp_config_path
     yield temp_config_path
-    config.reset_conf()
+    config.conf.reset()
     if original:
         os.environ[key] = original
     else:
@@ -187,7 +187,7 @@ def temp_cache(tempdir):
     intake.config.conf.update({'cache_dir': make_path_posix(str(tempdir)),
                                'cache_download_progress': False,
                                'cache_disabled': False})
-    intake.config.save_conf()
+    intake.config.conf.save()
     store.__init__(os.path.join(tempdir, 'persist'))
     try:
         yield
diff --git a/intake/source/tests/test_discovery.py b/intake/source/tests/test_discovery.py
index 6419595..f6411cf 100644
--- a/intake/source/tests/test_discovery.py
+++ b/intake/source/tests/test_discovery.py
@@ -13,6 +13,7 @@ import sys
 
 import pytest
 
+import intake
 from intake.source import discovery
 
 
@@ -56,7 +57,7 @@ def test_discover_cli(extra_pythonpath, tmp_config_path):
     ), stderr=subprocess.STDOUT, env=env)
 
     assert b'foo' in out
-    assert out.index(b'Not enabled') > out.index(b'foo')
+    assert out.index(b'Disabled') > out.index(b'foo')
 
     subprocess.check_output(shlex.split(
         "intake drivers disable foo"
@@ -66,25 +67,15 @@ def test_discover_cli(extra_pythonpath, tmp_config_path):
         "intake drivers list"
     ), stderr=subprocess.STDOUT, env=env)
     assert b'foo' in out
-    assert out.index(b'Not enabled') < out.index(b'foo')
+    assert out.index(b'Disabled') < out.index(b'foo')
 
 
 def test_discover(extra_pythonpath, tmp_config_path):
+    drivers = intake.source.discovery.DriverSouces(do_scan=True)
     with pytest.warns(PendingDeprecationWarning):
-        registry = discovery.autodiscover(do_package_scan=True)
-
-    # Check that package scan (name-based) discovery worked.
-    assert 'foo' in registry
-    registry['foo']()
-    # Check that entrypoints-based discovery worked.
-    assert 'some_test_driver' in registry
-    registry['some_test_driver']()
-
-    # Now again, giving the special PYTHONPATH explicit via kwarg.
-
-    with pytest.warns(PendingDeprecationWarning):
-        registry = discovery.autodiscover(path=[extra_pythonpath], do_package_scan=True)
+        assert "foo" in drivers.scanned
 
+    registry = intake.source.DriverRegistry(drivers)
     # Check that package scan (name-based) discovery worked.
     assert 'foo' in registry
     registry['foo']()
@@ -94,9 +85,8 @@ def test_discover(extra_pythonpath, tmp_config_path):
 
     # Now again, turning off the package scan.
 
-    registry = discovery.autodiscover(
-        path=[extra_pythonpath],
-        do_package_scan=False)
+    drivers = intake.source.discovery.DriverSouces()
+    registry = intake.source.DriverRegistry(drivers)
 
     # Check that package scan (name-based) discovery did *not* happen.
     assert 'foo' not in registry
@@ -109,48 +99,34 @@ def test_enable_and_disable(extra_pythonpath, tmp_config_path):
     # Disable and then enable a package scan result.
 
     try:
-        discovery.disable('foo')
+        drivers = intake.source.discovery.DriverSouces(do_scan=True)
+        drivers.disable('foo')
+        registry = intake.source.DriverRegistry(drivers)
         with pytest.warns(PendingDeprecationWarning):
-            registry = discovery.autodiscover(do_package_scan=True)
+            assert 'foo' in discovery.drivers.scanned
         assert 'foo' not in registry
 
-        discovery.enable('foo', 'intake_foo.FooPlugin')
-        with pytest.warns(PendingDeprecationWarning):
-            registry = discovery.autodiscover(do_package_scan=True)
+        drivers.enable('foo', 'intake_foo.FooPlugin')
         assert 'foo' in registry
     finally:
-        discovery.enable('foo', 'intake_foo.FooPlugin')
+        drivers.enable('foo', 'intake_foo.FooPlugin')
 
     # Disable and then enable an entrypoint result.
 
     try:
-        discovery.disable('some_test_driver')
-        with pytest.warns(PendingDeprecationWarning):
-            registry = discovery.autodiscover(do_package_scan=True)
+        drivers.disable('some_test_driver')
         assert 'some_test_driver' not in registry
 
-        discovery.enable(
+        drivers.enable(
             'some_test_driver',
             'driver_with_entrypoints.SomeTestDriver')
-        with pytest.warns(PendingDeprecationWarning):
-            registry = discovery.autodiscover(do_package_scan=True)
         assert 'some_test_driver' in registry
     finally:
-        discovery.enable(
+        drivers.enable(
             'some_test_driver',
             'driver_with_entrypoints.SomeTestDriver')
 
 
-def test_discover_pluginprefix(extra_pythonpath, tmp_config_path):
-    with pytest.warns(PendingDeprecationWarning):
-        registry = discovery.autodiscover(plugin_prefix='not_intake_',
-                                          do_package_scan=True)
-
-    assert 'otherfoo' in registry
-    registry['otherfoo']()
-    registry.pop('otherfoo', None)
-
-
 def test_discover_collision(extra_pythonpath, tmp_config_path):
     with pytest.warns(UserWarning):
-        discovery.autodiscover(plugin_prefix='collision_', do_package_scan=True)
+        discovery._package_scan(plugin_prefix='collision_')
diff --git a/intake/tests/test_config.py b/intake/tests/test_config.py
index 47ca8b7..ab47666 100644
--- a/intake/tests/test_config.py
+++ b/intake/tests/test_config.py
@@ -11,9 +11,11 @@ import requests
 
 import intake
 from intake import config
+from intake.config import Config, defaults
 from intake.util_tests import temp_conf, server
 from intake.catalog.remote import RemoteCatalog
 
+
 @pytest.mark.parametrize('conf', [
     {},
     {'port': 5000},
@@ -21,14 +23,15 @@ from intake.catalog.remote import RemoteCatalog
 ])
 def test_load_conf(conf):
     # This test will only work if your config is set to default
-    inconf = config.defaults.copy()
+    inconf = defaults.copy()
     expected = inconf.copy()
     with temp_conf(conf) as fn:
-        config.load_conf(fn)
+        config = Config(fn)
+        config.load(fn)
         expected.update(conf)
-        assert config.conf == expected
-        config.reset_conf()
-        assert config.conf == inconf
+        assert dict(config) == expected
+        config.reset()
+        assert dict(config) == inconf
 
 
 # Tests with a real separate server process
diff --git a/intake/tests/test_top_level.py b/intake/tests/test_top_level.py
index fbf78a1..c3045dc 100644
--- a/intake/tests/test_top_level.py
+++ b/intake/tests/test_top_level.py
@@ -45,7 +45,7 @@ def tmp_path_catalog():
 
 
 def test_autoregister_open():
-    assert hasattr(intake, 'open_csv')
+    assert intake.open_csv
 
 
 def test_default_catalogs():
diff --git a/intake/util_tests.py b/intake/util_tests.py
index d149f46..5d35e33 100644
--- a/intake/util_tests.py
+++ b/intake/util_tests.py
@@ -7,7 +7,6 @@
 
 from contextlib import contextmanager
 import os
-import posixpath
 import requests
 import shutil
 import subprocess