Repository: intake/intake
Pull request #526 (2020-08-24 21:14:42)

Lighten imports
With this PR, import times look like
```
  cumtime (ms)    intime (ms)  name
         210.3          104.7  intake
         105              0.6  +source
           5.4            1.2  ++entrypoints
           1.6            1.5  +++zipfile
           2.6            2.6  +++configparser
          19.7            0.7  ++utils
          18.9            0.7  +++yaml
          13.2            0.4  ++++loader
           7.2            7.2  +++++reader
           1.5            1.4  +++++constructor
           2.7            2.7  +++++resolver
           1.8            0.4  ++++dumper
           1.7            0.6  ++++cyaml
           1.1            1.1  +++++_yaml
          79.3            1    ++base
          78.3            0.7  +++cache
          74.7           60.2  ++++fsspec.utils
           1.6            0.8  +++++spec
           1.6            0.3  +++++registry
           1.3            1.3  ++++++distutils.version
          10.5            1.5  +++++mapping
           8.9            0.8  ++++++core
           7.7            0.8  +++++++compression
           3.8            0.6  ++++++++snappy
           2.6            0.6  +++++++++snappy
           2              2    ++++++++++_snappy
           1.3            0.4  ++++++++zstandard
           2.7            2.7  ++++intake.config
```

(I had trouble doing the timing on master, because pyarrow breaks the profiler; but I am satisfied that there is not much out of place in the output above. I did have to remove pyarrow to get the import profile above too, because fsspec optionally imports it. There should probably be an environment variable to prevent that.)

One big problem, causing failures here: `intake.gui` is a panel displaying instance OR a module. This unfortunate situation is totally my fault.



Patch:
diff --git a/intake/__init__.py b/intake/__init__.py
index e6f8147..27a3330 100644
--- a/intake/__init__.py
+++ b/intake/__init__.py
@@ -5,87 +5,91 @@
 # The full license is in the LICENSE file, distributed with this software.
 #-----------------------------------------------------------------------------
 
+import importlib
 import re
 import logging
 import warnings
-import sys
 
 from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions
-from . import source
-from .source.base import Schema, DataSource
-from .catalog.base import RemoteCatalog
-from .catalog import local
-from .catalog.default import load_combo_catalog
-from .catalog.local import MergedCatalog, EntrypointsCatalog
-from .catalog.zarr import ZarrGroupCatalog
-from .container import upload
-from .source import register_driver, registry
-from .source.discovery import autodiscover
-from .gui import InstanceMaker
 
-# Populate list of autodetected drivers (plugins).
-# The built-in ones are included here because intake itself declares them via
-# its own entry_points.
-for name, driver in autodiscover().items():
-    register_driver(name, driver)
+from .source import register_driver, registry
 
+imports = {
+    "DataSource": "intake.source.base:DataSoruce",
+    'Schema': "intake.source.base:Schema",
+    "cat": "intake.catalog:builtin",
+    "load_combo_catalog": "intake.catalog.default:load_combo_catalog",
+    "upload": "intake.container:upload",
+    "gui": "intake.interface:instance",
+    "output_notebook": "intake.interface:output_notebook",
+    "register_driver": "intake.source:register_driver",
+    "unregister_driver": "intake.source:unregister_driver",
+}
+openers = {}
 logger = logging.getLogger('intake')
-if sys.version_info >= (3, 7):
-    def __getattr__(attr):
-        if attr == 'Catalog':
-            from .catalog.base import Catalog
-            warnings.warn('deprecation: intake.Catalog now references the base class intake.catalog.base.Catalog\n '
-                          'If you want to open a generic URL, you should use intake.open_catalog')
-            return Catalog
 
 
-def make_open_functions():
-    """From the current state of ``registry``, create open_* functions"""
-    for plugin_name, plugin in registry.items():
-        try:
-            func_name = 'open_' + plugin_name
-            if not func_name.isidentifier():
-                # primitive name normalization
-                func_name = re.sub('[-=~^&|@+]', '_', func_name)
-            if func_name.isidentifier():
-                globals()[func_name] = plugin
+def __getattr__(attr):
+    """Lazy attribute propagator
+
+    Defers inputs of functions until they are needed, according to the
+    contents of the ``imports`` and ``openers`` dicts
+    """
+    gl = globals()
+    if attr == 'Catalog':
+        from .catalog.base import Catalog
+        warnings.warn('deprecation: intake.Catalog now references the base class intake.catalog.base.Catalog\n '
+                      'If you want to open a generic URL, you should use intake.open_catalog')
+        return Catalog
+    elif attr in openers:
+        driver = openers[attr].load()
+        gl[attr] = driver
+    else:
+        if attr in gl:
+            return gl[attr]
+        elif attr in imports:
+            dest = imports[attr]
+            modname = dest.split(":", 1)[0]
+            logger.debug("Importing: %s" % modname)
+            mod = importlib.import_module(modname)
+            if ":" in dest:
+                gl[attr] = getattr(mod, dest.split(":")[1])
             else:
-                warnings.warn('Invalid Intake plugin name "%s" found.' %
-                              plugin_name)
-        except:
-            logger.warning("Creation of open function failed for %s"
-                           "" % plugin_name)
+                gl[attr] = mod
+    if attr == "__all__":
+        return __dir__()
+    try:
+        return gl[attr]
+    except KeyError:
+        raise AttributeError(attr)
 
 
-def output_notebook(inline=True, logo=False):
-    """
-    Load the notebook extension
+def __dir__(*_, **__):
+    return sorted(list(globals()) + list(openers) + list(imports))
 
-    Parameters
-    ----------
-    inline : boolean (optional)
-        Whether to inline JS code or load it from a CDN
-    logo : boolean (optional)
-        Whether to show the logo(s)
-    """
-    try:
-        import hvplot
-    except ImportError:
-        raise ImportError("The intake plotting API requires hvplot."
-                          "hvplot may be installed with:\n\n"
-                          "`conda install -c pyviz hvplot` or "
-                          "`pip install hvplot`.")
-    import holoviews as hv
-    return hv.extension('bokeh', inline=inline, logo=logo)
+
+def make_open_functions():
+    """From the current state of ``registry``, create open_* functions"""
+    import entrypoints
+
+    for plugin in entrypoints.get_group_all('intake.drivers'):
+
+        register_driver(plugin.name, plugin, True)
+        func_name = 'open_' + plugin.name
+        if not func_name.isidentifier():
+            # primitive name normalization
+            func_name = re.sub('[-=~^&|@+]', '_', func_name)
+        if func_name.isidentifier():
+            # stash name for dir() and later fetch
+            openers[func_name] = plugin
+        else:
+            warnings.warn('Invalid Intake plugin name "%s" found.' %
+                          plugin.name)
 
 
 make_open_functions()
-cat = MergedCatalog(
-    [EntrypointsCatalog(), load_combo_catalog()],
-    name='builtin',
-    description='Generated from data packages found on your intake search path')
 
 
 def open_catalog(uri=None, **kwargs):
@@ -160,6 +164,3 @@ def open_catalog(uri=None, **kwargs):
         raise ValueError('Unknown catalog driver (%s), supply one of: %s'
                          % (driver, list(sorted(registry))))
     return registry[driver](uri, **kwargs)
-
-
-gui = InstanceMaker()
diff --git a/intake/catalog/__init__.py b/intake/catalog/__init__.py
index 7eed5f6..f4cb662 100644
--- a/intake/catalog/__init__.py
+++ b/intake/catalog/__init__.py
@@ -6,3 +6,10 @@
 #-----------------------------------------------------------------------------
 
 from .base import Catalog
+from .local import MergedCatalog, EntrypointsCatalog
+from .default import load_combo_catalog
+
+builtin = MergedCatalog(
+    [EntrypointsCatalog(), load_combo_catalog()],
+    name='builtin',
+    description='Generated from data packages found on your intake search path')
diff --git a/intake/catalog/base.py b/intake/catalog/base.py
index 7861a3a..a096e8a 100644
--- a/intake/catalog/base.py
+++ b/intake/catalog/base.py
@@ -5,24 +5,13 @@
 # The full license is in the LICENSE file, distributed with this software.
 #-----------------------------------------------------------------------------
 
-import collections
-import collections.abc
-import copy
-import datetime
 import keyword
 import logging
-import posixpath
 import re
 import time
-import warnings
 
-import msgpack
-
-from ..auth.base import BaseClientAuth, AuthenticationFailure
-from .remote import RemoteCatalogEntry
-from .utils import flatten, reload_on_change, RemoteCatalogError
+from .utils import flatten, reload_on_change
 from ..source.base import DataSource, NoEntry
-from ..compat import unpack_kwargs, pack_kwargs
 logger = logging.getLogger('intake')
 
 
@@ -50,7 +39,7 @@ class Catalog(DataSource):
     name = 'catalog'
 
     def __init__(self, *args, name=None, description=None, metadata=None,
-                 auth=None, ttl=60, getenv=True, getshell=True,
+                 ttl=60, getenv=True, getshell=True,
                  persist_mode='default', storage_options=None):
         """
         Parameters
@@ -65,8 +54,6 @@ class Catalog(DataSource):
             is stated in the cat file itself. Defaults to None.
         metadata: dict
             Additional information about this data
-        auth : BaseClientAuth or None
-            Default, None, falls back to BaseClientAuth.
         ttl : float, optional
             Lifespan (time to live) of cached modification time. Units are in
             seconds. Defaults to 1.
@@ -99,10 +86,6 @@ class Catalog(DataSource):
             raise ValueError('Persist mode (%s) not understood' % persist_mode)
         self.pmode = persist_mode
 
-        if auth is None:
-            auth = BaseClientAuth()
-        self.auth = auth
-
         if all(isinstance(a, (tuple, list)) for a in args):
             args = list(flatten(args))
         if len(args) == 1:
@@ -192,7 +175,6 @@ class Catalog(DataSource):
             ttl=self.ttl,
             getenv=self.getenv,
             getshell=self.getshell,
-            auth=self.auth,
             metadata=(self.metadata or {}).copy(),
             storage_options=self.storage_options)
         cat.metadata['search'] = {'text': text, 'upstream': self.name}
@@ -446,361 +428,3 @@ class Catalog(DataSource):
         else:
             self._gui.visible = True
         return self._gui
-
-
-class Entries(collections.abc.Mapping):
-    """Fetches entries from server on item lookup and iteration.
-
-    This fetches pages of entries from the server during iteration and
-    caches them. On __getitem__ it fetches the specific entry from the
-    server.
-    """
-    # This has PY3-style lazy methods (keys, values, items). Since it's
-    # internal we should not need the PY2-only iter* variants.
-    def __init__(self, catalog):
-        self._catalog = catalog
-        self._page_cache = collections.OrderedDict()
-        # Put lookups that were due to __getitem__ in a separate cache
-        # so that iteration reflects the server's order, not an
-        # arbitrary cache order.
-        self._direct_lookup_cache = {}
-        self._page_offset = 0
-        # True if all pages are cached locally
-        self.complete = self._catalog.page_size is None
-
-    def reset(self):
-        "Clear caches to force a reload."
-        self._page_cache.clear()
-        self._direct_lookup_cache.clear()
-        self._page_offset = 0
-        self.complete = self._catalog.page_size is None
-
-    def __iter__(self):
-        for key in self._page_cache:
-            yield key
-        if self._catalog.page_size is None:
-            # We are not paginating, either because the user set page_size=None
-            # or the server is a version of intake before pagination parameters
-            # were added.
-            return
-        # Fetch more entries from the server.
-        while True:
-            page = self._catalog.fetch_page(self._page_offset)
-            self._page_cache.update(page)
-            self._page_offset += len(page)
-            for key in page:
-                yield key
-            if len(page) < self._catalog.page_size:
-                # Partial or empty page.
-                # We are done until the next call to items(), when we
-                # will resume at the offset where we left off.
-                self.complete = True
-                break
-
-    def cached_items(self):
-        """
-        Iterate over items that are already cached. Perform no requests.
-        """
-        for item in self._page_cache.items():
-            yield item
-        for item in self._direct_lookup_cache.items():
-            yield item
-
-    def __getitem__(self, key):
-        try:
-            return self._direct_lookup_cache[key]
-        except KeyError:
-            try:
-                return self._page_cache[key]
-            except KeyError:
-                source = self._catalog.fetch_by_name(key)
-                self._direct_lookup_cache[key] = source
-                return source
-
-    def __len__(self):
-        return len(self._catalog)
-
-
-class RemoteCatalog(Catalog):
-    """The state of a remote Intake server"""
-    name = 'intake_remote'
-
-    def __init__(self, url, http_args=None, page_size=None,
-                 name=None, source_id=None, metadata=None, auth=None, ttl=1,
-                 getenv=True, getshell=True,
-                 storage_options=None, parameters=None):
-        """Connect to remote Intake Server as a catalog
-
-        Parameters
-        ----------
-        url: str
-            Address of the server, e.g., "intake://localhost:5000".
-        http_args: dict
-            Arguments to add to HTTP calls, including "ssl" (True/False) for
-            secure connections.
-        page_size : int, optional
-            The number of entries fetched at a time during iteration.
-            Default is None (no pagination; fetch all entries in bulk).
-        name : str, optional
-            Unique identifier for catalog. This is primarily useful when
-            manually constructing a catalog. Defaults to None.
-        source_id : str, optional
-            Emphemeral unique ID generated by the server, if known.
-        metadata: dict
-            Additional information about this data
-        auth : BaseClientAuth or None
-            Default, None, falls back to BaseClientAuth.
-        ttl : float, optional
-            Lifespan (time to live) of cached modification time. Units are in
-            seconds. Defaults to 1.
-        getenv: bool
-            Can parameter default fields take values from the environment
-        getshell: bool
-            Can parameter default fields run shell commands
-        storage_options : dict
-            If using a URL beginning with 'intake://' (remote Intake server),
-            parameters to pass to requests when issuing http commands; otherwise
-            parameters to pass to remote backend file-system. Ignored for
-            normal local files.
-        parameters: dict
-            To pass to the server when it instantiates the data source
-        """
-        from requests.compat import urljoin, urlparse
-        if http_args is None:
-            http_args = {}
-        else:
-            # Make a deep copy to avoid mutating input.
-            http_args = copy.deepcopy(http_args)
-        secure = http_args.pop('ssl', False)
-        scheme = 'https' if secure else 'http'
-        url = url.replace('intake', scheme, 1)
-        if not url.endswith('/'):
-            url = url + '/'
-        self.url = url
-        self.info_url = urljoin(url, 'v1/info')
-        self.source_url = urljoin(url, 'v1/source')
-        self.http_args = http_args
-        self.http_args.update(storage_options or {})
-        self.http_args['headers'] = self.http_args.get('headers', {})
-        self._page_size = page_size
-        self._source_id = source_id
-        self._parameters = parameters
-        self._len = None
-        if self._source_id is None:
-            name = urlparse(url).netloc.replace(
-                '.', '_').replace(':', '_')
-        super(RemoteCatalog, self).__init__(
-            name=name, metadata=name, auth=auth, ttl=ttl, getenv=getenv,
-            getshell=getshell, storage_options=storage_options)
-
-    def _make_entries_container(self):
-        return Entries(self)
-
-    def __dir__(self):
-        # Include (cached) tab-completable entries and normal attributes.
-        return (
-            [key for key in self._ipython_key_completions_() if
-             re.match("[_A-Za-z][_a-zA-Z0-9]*$", key)  # valid Python identifier
-             and not keyword.iskeyword(key)]  # not a Python keyword
-            + list(self.__dict__.keys()))
-
-    def _ipython_key_completions_(self):
-        if not self._entries.complete:
-            # Ensure that at least one page of data has been loaded so that
-            # *some* entries are included.
-            next(iter(self))
-        if not self._entries.complete:
-            warnings.warn(
-                "Tab-complete and dir() on RemoteCatalog may include only a "
-                "subset of the available entries.")
-        # Loop through the cached entries, but do not trigger iteration over
-        # the full set.
-        # Intentionally access _entries directly to avoid paying for a reload.
-        return [key for key, _ in self._entries.cached_items()]
-
-    @property
-    def page_size(self):
-        return self._page_size
-
-    def fetch_page(self, page_offset):
-        import requests
-        logger.debug("Request page entries %d-%d",
-                     page_offset, page_offset + self._page_size)
-        params = {'page_offset': page_offset,
-                  'page_size': self._page_size}
-        http_args = self._get_http_args(params)
-        response = requests.get(self.info_url, **http_args)
-        # Produce a chained exception with both the underlying HTTPError
-        # and our own more direct context.
-        try:
-            response.raise_for_status()
-        except requests.HTTPError as err:
-            raise RemoteCatalogError(
-                "Failed to fetch page of entries {}-{}."
-                "".format(page_offset, page_offset + self._page_size)) from err
-        info = msgpack.unpackb(response.content, **unpack_kwargs)
-        page = {}
-        for source in info['sources']:
-            user_parameters = source.get('user_parameters', [])
-            # TODO Do something with self._parameters.
-            page[source['name']] = RemoteCatalogEntry(
-                url=self.url,
-                getenv=self.getenv,
-                getshell=self.getshell,
-                auth=self.auth,
-                http_args=self.http_args,
-                page_size=self._page_size,
-                # user_parameters=user_parameters,
-                **source)
-        return page
-
-    def fetch_by_name(self, name):
-        import requests
-        logger.debug("Requesting info about entry named '%s'", name)
-        params = {'name': name}
-        http_args = self._get_http_args(params)
-        response = requests.get(self.source_url, **http_args)
-        if response.status_code == 404:
-            raise KeyError(name)
-        try:
-            response.raise_for_status()
-        except requests.HTTPError as err:
-            raise RemoteCatalogError(
-                "Failed to fetch entry {!r}.".format(name)) from err
-        info = msgpack.unpackb(response.content, **unpack_kwargs)
-        return RemoteCatalogEntry(
-            url=self.url,
-            getenv=self.getenv,
-            getshell=self.getshell,
-            auth=self.auth,
-            http_args=self.http_args,
-            page_size=self._page_size,
-            **info['source'])
-
-    def _get_http_args(self, params):
-        """
-        Return a copy of the http_args
-
-        Adds auth headers and 'source-id', merges in params.
-        """
-        # Add the auth headers to any other headers
-        headers = self.http_args.get('headers', {})
-        if self.auth is not None:
-            auth_headers = self.auth.get_headers()
-            headers.update(auth_headers)
-
-        # build new http args with these headers
-        http_args = self.http_args.copy()
-        if self._source_id is not None:
-            headers['source-id'] = self._source_id
-        http_args['headers'] = headers
-
-        # Merge in any params specified by the caller.
-        merged_params = http_args.get('params', {})
-        merged_params.update(params)
-        http_args['params'] = merged_params
-        return http_args
-
-    def _load(self):
-        """Fetch metadata from remote. Entries are fetched lazily."""
-        # This will not immediately fetch any sources (entries). It will lazily
-        # fetch sources from the server in paginated blocks when this Catalog
-        # is iterated over. It will fetch specific sources when they are
-        # accessed in this Catalog via __getitem__.
-        import requests
-
-        if self.page_size is None:
-            # Fetch all source info.
-            params = {}
-        else:
-            # Just fetch the metadata now; fetch source info later in pages.
-            params = {'page_offset': 0, 'page_size': 0}
-        http_args = self._get_http_args(params)
-        response = requests.get(self.info_url, **http_args)
-        try:
-            response.raise_for_status()
-            error = False
-        except requests.HTTPError as err:
-            if '403' in err.args[0]:
-                error = "Your current level of authentication does not have access"
-            else:
-                raise RemoteCatalogError(
-                    "Failed to fetch metadata.") from err
-        if error:
-            raise AuthenticationFailure(error)
-        info = msgpack.unpackb(response.content, **unpack_kwargs)
-        self.metadata = info['metadata']
-        # The intake server now always provides a length, but the server may be
-        # running an older version of intake.
-        self._len = info.get('length')
-        self._entries.reset()
-        # If we are paginating (page_size is not None) and the server we are
-        # working with is new enough to support pagination, info['sources']
-        # should be empty. If either of those things is not true,
-        # info['sources'] will contain all the entries and we should cache them
-        # now.
-        if info['sources']:
-            # Signal that we are not paginating, even if we were asked to.
-            self._page_size = None
-            self._entries._page_cache.update(
-                {source['name']: RemoteCatalogEntry(
-                    url=self.url,
-                    getenv=self.getenv,
-                    getshell=self.getshell,
-                    auth=self.auth,
-                    http_args=self.http_args, **source)
-                 for source in info['sources']})
-
-    def search(self, *args, **kwargs):
-        import requests
-        request = {'action': 'search', 'query': (args, kwargs),
-                   'source_id': self._source_id}
-        response = requests.post(
-            url=self.source_url, **self._get_http_args({}),
-            data=msgpack.packb(request, **pack_kwargs))
-        try:
-            response.raise_for_status()
-        except requests.HTTPError as err:
-            raise RemoteCatalogError("Failed search query.") from err
-        source = msgpack.unpackb(response.content, **unpack_kwargs)
-        source_id = source['source_id']
-        cat = RemoteCatalog(
-            url=self.url,
-            http_args=self.http_args,
-            source_id=source_id,
-            name="")
-        cat.cat = self
-        return cat
-
-    def __len__(self):
-        if self._len is None:
-            # The server is running an old version of intake and did not
-            # provide a length, so we have no choice but to do this the
-            # expensive way.
-            return sum(1 for _ in self)
-        else:
-            return self._len
-
-    @staticmethod
-    def _persist(source, path, **kwargs):
-        return RemoteCatalog._data_to_source(source, path, **kwargs)
-
-    @staticmethod
-    def _data_to_source(cat, path, **kwargs):
-        from intake.catalog.local import YAMLFileCatalog
-        from fsspec import open_files
-        import yaml
-        if not isinstance(cat, Catalog):
-            raise NotImplementedError
-        out = {}
-        # reach down into the private state because we apparently need the
-        # Entry here rather than the public facing DataSource objects.
-        for name, entry in cat._entries.items():
-            out[name] = entry.__getstate__()
-            out[name]['parameters'] = [up._captured_init_kwargs for up
-                                       in entry._user_parameters]
-            out[name]['kwargs'].pop('parameters')
-        fn = posixpath.join(path, 'cat.yaml')
-        with open_files([fn], 'wt')[0] as f:
-            yaml.dump({'sources': out}, f)
-        return YAMLFileCatalog(fn)
diff --git a/intake/catalog/gui.py b/intake/catalog/gui.py
index 04528cd..a453d79 100644
--- a/intake/catalog/gui.py
+++ b/intake/catalog/gui.py
@@ -8,7 +8,7 @@ from functools import partial
 
 try:
     import panel as pn
-    from ..gui.gui import SourceGUI, MAX_WIDTH, ICONS
+    from ..interface.gui import SourceGUI, MAX_WIDTH, ICONS
     css = """
     .scrolling {
         overflow: scroll;
diff --git a/intake/catalog/local.py b/intake/catalog/local.py
index b6c7143..262d5f9 100644
--- a/intake/catalog/local.py
+++ b/intake/catalog/local.py
@@ -11,7 +11,6 @@ import logging
 import os
 import warnings
 
-from jinja2 import Template
 from fsspec import open_files, get_filesystem_class
 from fsspec.core import split_protocol
 
diff --git a/intake/catalog/remote.py b/intake/catalog/remote.py
index 1e03c0e..6da49e9 100644
--- a/intake/catalog/remote.py
+++ b/intake/catalog/remote.py
@@ -5,11 +5,385 @@
 # The full license is in the LICENSE file, distributed with this software.
 #-----------------------------------------------------------------------------
 
+import collections
+import copy
+import keyword
+import logging
+import posixpath
+import re
+import warnings
+
+import msgpack
+
 from ..source import registry as plugin_registry
+from . import Catalog
 from .entry import CatalogEntry
-from .utils import expand_defaults, coerce
+from .utils import expand_defaults, coerce, RemoteCatalogError
 from ..compat import unpack_kwargs, pack_kwargs
 from ..utils import remake_instance
+from intake.auth.base import BaseClientAuth, AuthenticationFailure
+logger = logging.getLogger('intake')
+
+
+class RemoteCatalog(Catalog):
+    """The state of a remote Intake server"""
+    name = 'intake_remote'
+
+    def __init__(self, url, http_args=None, page_size=None,
+                 name=None, source_id=None, metadata=None, auth=None, ttl=1,
+                 getenv=True, getshell=True,
+                 storage_options=None, parameters=None):
+        """Connect to remote Intake Server as a catalog
+
+        Parameters
+        ----------
+        url: str
+            Address of the server, e.g., "intake://localhost:5000".
+        http_args: dict
+            Arguments to add to HTTP calls, including "ssl" (True/False) for
+            secure connections.
+        page_size : int, optional
+            The number of entries fetched at a time during iteration.
+            Default is None (no pagination; fetch all entries in bulk).
+        name : str, optional
+            Unique identifier for catalog. This is primarily useful when
+            manually constructing a catalog. Defaults to None.
+        source_id : str, optional
+            Emphemeral unique ID generated by the server, if known.
+        metadata: dict
+            Additional information about this data
+        auth : BaseClientAuth or None
+            Default, None, falls back to BaseClientAuth.
+        ttl : float, optional
+            Lifespan (time to live) of cached modification time. Units are in
+            seconds. Defaults to 1.
+        getenv: bool
+            Can parameter default fields take values from the environment
+        getshell: bool
+            Can parameter default fields run shell commands
+        storage_options : dict
+            If using a URL beginning with 'intake://' (remote Intake server),
+            parameters to pass to requests when issuing http commands; otherwise
+            parameters to pass to remote backend file-system. Ignored for
+            normal local files.
+        parameters: dict
+            To pass to the server when it instantiates the data source
+        """
+        from requests.compat import urljoin, urlparse
+        if http_args is None:
+            http_args = {}
+        else:
+            # Make a deep copy to avoid mutating input.
+            http_args = copy.deepcopy(http_args)
+        secure = http_args.pop('ssl', False)
+        scheme = 'https' if secure else 'http'
+        url = url.replace('intake', scheme, 1)
+        if not url.endswith('/'):
+            url = url + '/'
+        self.url = url
+        self.info_url = urljoin(url, 'v1/info')
+        self.source_url = urljoin(url, 'v1/source')
+        self.http_args = http_args
+        self.http_args.update(storage_options or {})
+        self.http_args['headers'] = self.http_args.get('headers', {})
+        self._page_size = page_size
+        self._source_id = source_id
+        self._parameters = parameters
+        self._len = None
+        self.auth = auth or BaseClientAuth()
+
+        if self._source_id is None:
+            name = urlparse(url).netloc.replace(
+                '.', '_').replace(':', '_')
+        super(RemoteCatalog, self).__init__(
+            name=name, metadata=name, ttl=ttl, getenv=getenv,
+            getshell=getshell, storage_options=storage_options)
+
+    def _make_entries_container(self):
+        return Entries(self)
+
+    def __dir__(self):
+        # Include (cached) tab-completable entries and normal attributes.
+        return (
+            [key for key in self._ipython_key_completions_() if
+             re.match("[_A-Za-z][_a-zA-Z0-9]*$", key)  # valid Python identifier
+             and not keyword.iskeyword(key)]  # not a Python keyword
+            + list(self.__dict__.keys()))
+
+    def _ipython_key_completions_(self):
+        if not self._entries.complete:
+            # Ensure that at least one page of data has been loaded so that
+            # *some* entries are included.
+            next(iter(self))
+        if not self._entries.complete:
+            warnings.warn(
+                "Tab-complete and dir() on RemoteCatalog may include only a "
+                "subset of the available entries.")
+        # Loop through the cached entries, but do not trigger iteration over
+        # the full set.
+        # Intentionally access _entries directly to avoid paying for a reload.
+        return [key for key, _ in self._entries.cached_items()]
+
+    @property
+    def page_size(self):
+        return self._page_size
+
+    def fetch_page(self, page_offset):
+        import requests
+        logger.debug("Request page entries %d-%d",
+                     page_offset, page_offset + self._page_size)
+        params = {'page_offset': page_offset,
+                  'page_size': self._page_size}
+        http_args = self._get_http_args(params)
+        response = requests.get(self.info_url, **http_args)
+        # Produce a chained exception with both the underlying HTTPError
+        # and our own more direct context.
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as err:
+            raise RemoteCatalogError(
+                "Failed to fetch page of entries {}-{}."
+                "".format(page_offset, page_offset + self._page_size)) from err
+        info = msgpack.unpackb(response.content, **unpack_kwargs)
+        page = {}
+        for source in info['sources']:
+            user_parameters = source.get('user_parameters', [])
+            # TODO Do something with self._parameters.
+            page[source['name']] = RemoteCatalogEntry(
+                url=self.url,
+                getenv=self.getenv,
+                getshell=self.getshell,
+                auth=self.auth,
+                http_args=self.http_args,
+                page_size=self._page_size,
+                # user_parameters=user_parameters,
+                **source)
+        return page
+
+    def fetch_by_name(self, name):
+        import requests
+        logger.debug("Requesting info about entry named '%s'", name)
+        params = {'name': name}
+        http_args = self._get_http_args(params)
+        response = requests.get(self.source_url, **http_args)
+        if response.status_code == 404:
+            raise KeyError(name)
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as err:
+            raise RemoteCatalogError(
+                "Failed to fetch entry {!r}.".format(name)) from err
+        info = msgpack.unpackb(response.content, **unpack_kwargs)
+        return RemoteCatalogEntry(
+            url=self.url,
+            getenv=self.getenv,
+            getshell=self.getshell,
+            auth=self.auth,
+            http_args=self.http_args,
+            page_size=self._page_size,
+            **info['source'])
+
+    def _get_http_args(self, params):
+        """
+        Return a copy of the http_args
+
+        Adds auth headers and 'source-id', merges in params.
+        """
+        # Add the auth headers to any other headers
+        headers = self.http_args.get('headers', {})
+        if self.auth is not None:
+            auth_headers = self.auth.get_headers()
+            headers.update(auth_headers)
+
+        # build new http args with these headers
+        http_args = self.http_args.copy()
+        if self._source_id is not None:
+            headers['source-id'] = self._source_id
+        http_args['headers'] = headers
+
+        # Merge in any params specified by the caller.
+        merged_params = http_args.get('params', {})
+        merged_params.update(params)
+        http_args['params'] = merged_params
+        return http_args
+
+    def _load(self):
+        """Fetch metadata from remote. Entries are fetched lazily."""
+        # This will not immediately fetch any sources (entries). It will lazily
+        # fetch sources from the server in paginated blocks when this Catalog
+        # is iterated over. It will fetch specific sources when they are
+        # accessed in this Catalog via __getitem__.
+        import requests
+
+        if self.page_size is None:
+            # Fetch all source info.
+            params = {}
+        else:
+            # Just fetch the metadata now; fetch source info later in pages.
+            params = {'page_offset': 0, 'page_size': 0}
+        http_args = self._get_http_args(params)
+        response = requests.get(self.info_url, **http_args)
+        try:
+            response.raise_for_status()
+            error = False
+        except requests.HTTPError as err:
+            if '403' in err.args[0]:
+                error = "Your current level of authentication does not have access"
+            else:
+                raise RemoteCatalogError(
+                    "Failed to fetch metadata.") from err
+        if error:
+            raise AuthenticationFailure(error)
+        info = msgpack.unpackb(response.content, **unpack_kwargs)
+        self.metadata = info['metadata']
+        # The intake server now always provides a length, but the server may be
+        # running an older version of intake.
+        self._len = info.get('length')
+        self._entries.reset()
+        # If we are paginating (page_size is not None) and the server we are
+        # working with is new enough to support pagination, info['sources']
+        # should be empty. If either of those things is not true,
+        # info['sources'] will contain all the entries and we should cache them
+        # now.
+        if info['sources']:
+            # Signal that we are not paginating, even if we were asked to.
+            self._page_size = None
+            self._entries._page_cache.update(
+                {source['name']: RemoteCatalogEntry(
+                    url=self.url,
+                    getenv=self.getenv,
+                    getshell=self.getshell,
+                    auth=self.auth,
+                    http_args=self.http_args, **source)
+                 for source in info['sources']})
+
+    def search(self, *args, **kwargs):
+        import requests
+        request = {'action': 'search', 'query': (args, kwargs),
+                   'source_id': self._source_id}
+        response = requests.post(
+            url=self.source_url, **self._get_http_args({}),
+            data=msgpack.packb(request, **pack_kwargs))
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as err:
+            raise RemoteCatalogError("Failed search query.") from err
+        source = msgpack.unpackb(response.content, **unpack_kwargs)
+        source_id = source['source_id']
+        cat = RemoteCatalog(
+            url=self.url,
+            http_args=self.http_args,
+            source_id=source_id,
+            name="")
+        cat.cat = self
+        return cat
+
+    def __len__(self):
+        if self._len is None:
+            # The server is running an old version of intake and did not
+            # provide a length, so we have no choice but to do this the
+            # expensive way.
+            return sum(1 for _ in self)
+        else:
+            return self._len
+
+    @staticmethod
+    def _persist(source, path, **kwargs):
+        return RemoteCatalog._data_to_source(source, path, **kwargs)
+
+    @staticmethod
+    def _data_to_source(cat, path, **kwargs):
+        from intake.catalog.local import YAMLFileCatalog
+        from fsspec import open_files
+        import yaml
+        if not isinstance(cat, Catalog):
+            raise NotImplementedError
+        out = {}
+        # reach down into the private state because we apparently need the
+        # Entry here rather than the public facing DataSource objects.
+        for name, entry in cat._entries.items():
+            out[name] = entry.__getstate__()
+            out[name]['parameters'] = [up._captured_init_kwargs for up
+                                       in entry._user_parameters]
+            out[name]['kwargs'].pop('parameters')
+        fn = posixpath.join(path, 'cat.yaml')
+        with open_files([fn], 'wt')[0] as f:
+            yaml.dump({'sources': out}, f)
+        return YAMLFileCatalog(fn)
+
+
+class Entries(collections.abc.Mapping):
+    """Fetches entries from server on item lookup and iteration.
+
+    This fetches pages of entries from the server during iteration and
+    caches them. On __getitem__ it fetches the specific entry from the
+    server.
+    """
+    # This has PY3-style lazy methods (keys, values, items). Since it's
+    # internal we should not need the PY2-only iter* variants.
+    def __init__(self, catalog):
+        self._catalog = catalog
+        self._page_cache = collections.OrderedDict()
+        # Put lookups that were due to __getitem__ in a separate cache
+        # so that iteration reflects the server's order, not an
+        # arbitrary cache order.
+        self._direct_lookup_cache = {}
+        self._page_offset = 0
+        # True if all pages are cached locally
+        self.complete = self._catalog.page_size is None
+
+    def reset(self):
+        "Clear caches to force a reload."
+        self._page_cache.clear()
+        self._direct_lookup_cache.clear()
+        self._page_offset = 0
+        self.complete = self._catalog.page_size is None
+
+    def __iter__(self):
+        for key in self._page_cache:
+            yield key
+        if self._catalog.page_size is None:
+            # We are not paginating, either because the user set page_size=None
+            # or the server is a version of intake before pagination parameters
+            # were added.
+            return
+        # Fetch more entries from the server.
+        while True:
+            page = self._catalog.fetch_page(self._page_offset)
+            self._page_cache.update(page)
+            self._page_offset += len(page)
+            for key in page:
+                yield key
+            if len(page) < self._catalog.page_size:
+                # Partial or empty page.
+                # We are done until the next call to items(), when we
+                # will resume at the offset where we left off.
+                self.complete = True
+                break
+
+    def cached_items(self):
+        """
+        Iterate over items that are already cached. Perform no requests.
+        """
+        for item in self._page_cache.items():
+            yield item
+        for item in self._direct_lookup_cache.items():
+            yield item
+
+    def __getitem__(self, key):
+        try:
+            return self._direct_lookup_cache[key]
+        except KeyError:
+            try:
+                return self._page_cache[key]
+            except KeyError:
+                source = self._catalog.fetch_by_name(key)
+                self._direct_lookup_cache[key] = source
+                return source
+
+    def __len__(self):
+        return len(self._catalog)
+
 
 class RemoteCatalogEntry(CatalogEntry):
     """An entry referring to a remote data definition"""
diff --git a/intake/config.py b/intake/config.py
index ef4dcbd..060401e 100644
--- a/intake/config.py
+++ b/intake/config.py
@@ -91,7 +91,7 @@ def intake_path_dirs(path):
 
 
 def load_env():
-    """Analyse enviroment variables and update conf accordingly"""
+    """Analyse environment variables and update conf accordingly"""
     # environment variables take precedence over conf file
     for key, envvar in [['cache_dir', 'INTAKE_CACHE_DIR'],
                         ['catalog_path', 'INTAKE_PATH'],
diff --git a/intake/container/__init__.py b/intake/container/__init__.py
index 42b4948..3f3392f 100644
--- a/intake/container/__init__.py
+++ b/intake/container/__init__.py
@@ -8,7 +8,7 @@
 from .dataframe import RemoteDataFrame
 from .ndarray import RemoteArray
 from .semistructured import RemoteSequenceSource
-from ..catalog.base import RemoteCatalog
+from ..catalog.remote import RemoteCatalog
 from ..utils import ContainerRegistryView
 
 # each container type is represented in the remote by one of the classes in
diff --git a/intake/container/persist.py b/intake/container/persist.py
index 5f7398c..7770b0f 100644
--- a/intake/container/persist.py
+++ b/intake/container/persist.py
@@ -9,7 +9,7 @@ import posixpath
 import time
 import yaml
 from ..catalog.local import YAMLFileCatalog, CatalogEntry
-from .. import DataSource
+from ..source import DataSource
 from ..config import conf, logger
 from ..source import import_name
 from ..utils import make_path_posix
diff --git a/intake/gui/__init__.py b/intake/gui/__init__.py
deleted file mode 100644
index 777b4bc..0000000
--- a/intake/gui/__init__.py
+++ /dev/null
@@ -1,71 +0,0 @@
-#-----------------------------------------------------------------------------
-# Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors
-# All rights reserved.
-#
-# The full license is in the LICENSE file, distributed with this software.
-#-----------------------------------------------------------------------------
-from distutils.version import LooseVersion
-
-
-def do_import():
-    error = too_old = False
-    try:
-        import panel as pn
-        too_old = LooseVersion(pn.__version__) < LooseVersion("0.9.5")
-    except ImportError as e:
-        error = e
-
-    if too_old or error:
-        class GUI(object):
-            def __repr__(self):
-                raise RuntimeError("Please install panel to use the GUI `conda "
-                                   "install -c conda-forge panel>=0.8.0`. Import "
-                                   "failed with error: %s" % error)
-        return GUI
-
-    try:
-        from .gui import GUI
-        css = """
-        .scrolling {
-          overflow: scroll;
-        }
-        """
-        pn.config.raw_css.append(css)  # add scrolling class from css (panel GH#383, GH#384)
-        pn.extension()
-
-    except Exception as e:
-
-        class GUI(object):
-            def __repr__(self):
-                raise RuntimeError("Initialisation of GUI failed, even though "
-                                   "panel is installed. Please update it "
-                                   "to a more recent version (`conda install -c"
-                                   " conda-forge panel>=0.9.5`).")
-    return GUI
-
-
-class InstanceMaker(object):
-
-    def __init__(self):
-        self._instance = None
-
-    def _instantiate(self):
-        if self._instance is None:
-            GUI = do_import()
-            self._instance = GUI()
-
-    def __getattr__(self, attr, *args, **kwargs):
-        self._instantiate()
-        return getattr(self._instance, attr, *args, **kwargs)
-
-    def __getitem__(self, item):
-        self._instantiate()
-        return self._instance[item]
-
-    def __repr__(self):
-        self._instantiate()
-        return repr(self._instance)
-
-    def __dir__(self):
-        self._instantiate()
-        return dir(self._instance)
diff --git a/intake/interface/__init__.py b/intake/interface/__init__.py
new file mode 100644
index 0000000..165fcd9
--- /dev/null
+++ b/intake/interface/__init__.py
@@ -0,0 +1,61 @@
+#-----------------------------------------------------------------------------
+# Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors
+# All rights reserved.
+#
+# The full license is in the LICENSE file, distributed with this software.
+#-----------------------------------------------------------------------------
+from distutils.version import LooseVersion
+
+gl = globals()
+
+
+def do_import():
+    error = too_old = False
+    try:
+        import panel as pn
+        too_old = LooseVersion(pn.__version__) < LooseVersion("0.9.5")
+    except ImportError as e:
+        error = e
+
+    if too_old or error:
+        raise RuntimeError("Please install panel to use the GUI `conda "
+                           "install -c conda-forge panel>=0.8.0`. Import "
+                           "failed with error: %s" % error)
+
+    from .gui import GUI
+    css = """
+    .scrolling {
+      overflow: scroll;
+    }
+    """
+    pn.config.raw_css.append(css)  # add scrolling class from css (panel GH#383, GH#384)
+    pn.extension()
+    gl['instance'] = GUI()
+
+
+def __getattr__(attr):
+    if attr == 'instance':
+        do_import()
+    return gl['instance']
+
+
+def output_notebook(inline=True, logo=False):
+    """
+    Load the notebook extension
+
+    Parameters
+    ----------
+    inline : boolean (optional)
+        Whether to inline JS code or load it from a CDN
+    logo : boolean (optional)
+        Whether to show the logo(s)
+    """
+    try:
+        import hvplot
+    except ImportError:
+        raise ImportError("The intake plotting API requires hvplot."
+                          "hvplot may be installed with:\n\n"
+                          "`conda install -c pyviz hvplot` or "
+                          "`pip install hvplot`.")
+    import holoviews as hv
+    return hv.extension('bokeh', inline=inline, logo=logo)
diff --git a/intake/gui/base.py b/intake/interface/base.py
similarity index 100%
rename from intake/gui/base.py
rename to intake/interface/base.py
diff --git a/intake/gui/catalog/__init__.py b/intake/interface/catalog/__init__.py
similarity index 100%
rename from intake/gui/catalog/__init__.py
rename to intake/interface/catalog/__init__.py
diff --git a/intake/gui/catalog/add.py b/intake/interface/catalog/add.py
similarity index 100%
rename from intake/gui/catalog/add.py
rename to intake/interface/catalog/add.py
diff --git a/intake/gui/catalog/gui.py b/intake/interface/catalog/gui.py
similarity index 100%
rename from intake/gui/catalog/gui.py
rename to intake/interface/catalog/gui.py
diff --git a/intake/gui/catalog/search.py b/intake/interface/catalog/search.py
similarity index 100%
rename from intake/gui/catalog/search.py
rename to intake/interface/catalog/search.py
diff --git a/intake/gui/catalog/select.py b/intake/interface/catalog/select.py
similarity index 100%
rename from intake/gui/catalog/select.py
rename to intake/interface/catalog/select.py
diff --git a/intake/gui/catalog/tests/__init__.py b/intake/interface/catalog/tests/__init__.py
similarity index 100%
rename from intake/gui/catalog/tests/__init__.py
rename to intake/interface/catalog/tests/__init__.py
diff --git a/intake/gui/catalog/tests/test_catalog_add.py b/intake/interface/catalog/tests/test_catalog_add.py
similarity index 100%
rename from intake/gui/catalog/tests/test_catalog_add.py
rename to intake/interface/catalog/tests/test_catalog_add.py
diff --git a/intake/gui/catalog/tests/test_catalog_search.py b/intake/interface/catalog/tests/test_catalog_search.py
similarity index 100%
rename from intake/gui/catalog/tests/test_catalog_search.py
rename to intake/interface/catalog/tests/test_catalog_search.py
diff --git a/intake/gui/catalog/tests/test_gui.py b/intake/interface/catalog/tests/test_gui.py
similarity index 100%
rename from intake/gui/catalog/tests/test_gui.py
rename to intake/interface/catalog/tests/test_gui.py
diff --git a/intake/gui/catalog/tests/test_select.py b/intake/interface/catalog/tests/test_select.py
similarity index 100%
rename from intake/gui/catalog/tests/test_select.py
rename to intake/interface/catalog/tests/test_select.py
diff --git a/intake/gui/conftest.py b/intake/interface/conftest.py
similarity index 100%
rename from intake/gui/conftest.py
rename to intake/interface/conftest.py
diff --git a/intake/gui/gui.py b/intake/interface/gui.py
similarity index 100%
rename from intake/gui/gui.py
rename to intake/interface/gui.py
diff --git a/intake/gui/icons/baseline-check-24px.svg b/intake/interface/icons/baseline-check-24px.svg
similarity index 100%
rename from intake/gui/icons/baseline-check-24px.svg
rename to intake/interface/icons/baseline-check-24px.svg
diff --git a/intake/gui/icons/baseline-error-24px.svg b/intake/interface/icons/baseline-error-24px.svg
similarity index 100%
rename from intake/gui/icons/baseline-error-24px.svg
rename to intake/interface/icons/baseline-error-24px.svg
diff --git a/intake/gui/icons/logo.png b/intake/interface/icons/logo.png
similarity index 100%
rename from intake/gui/icons/logo.png
rename to intake/interface/icons/logo.png
diff --git a/intake/gui/server.py b/intake/interface/server.py
similarity index 94%
rename from intake/gui/server.py
rename to intake/interface/server.py
index 2048f29..ffb56c2 100644
--- a/intake/gui/server.py
+++ b/intake/interface/server.py
@@ -13,4 +13,4 @@ panel serve intake/gui/server.py
 """
 
 import intake
-intake.gui.servable()
+intake.interface.servable()
diff --git a/intake/gui/source/__init__.py b/intake/interface/source/__init__.py
similarity index 100%
rename from intake/gui/source/__init__.py
rename to intake/interface/source/__init__.py
diff --git a/intake/gui/source/defined_plots.py b/intake/interface/source/defined_plots.py
similarity index 100%
rename from intake/gui/source/defined_plots.py
rename to intake/interface/source/defined_plots.py
diff --git a/intake/gui/source/description.py b/intake/interface/source/description.py
similarity index 100%
rename from intake/gui/source/description.py
rename to intake/interface/source/description.py
diff --git a/intake/gui/source/gui.py b/intake/interface/source/gui.py
similarity index 100%
rename from intake/gui/source/gui.py
rename to intake/interface/source/gui.py
diff --git a/intake/gui/source/select.py b/intake/interface/source/select.py
similarity index 100%
rename from intake/gui/source/select.py
rename to intake/interface/source/select.py
diff --git a/intake/gui/source/tests/__init__.py b/intake/interface/source/tests/__init__.py
similarity index 100%
rename from intake/gui/source/tests/__init__.py
rename to intake/interface/source/tests/__init__.py
diff --git a/intake/gui/source/tests/test_gui.py b/intake/interface/source/tests/test_gui.py
similarity index 100%
rename from intake/gui/source/tests/test_gui.py
rename to intake/interface/source/tests/test_gui.py
diff --git a/intake/gui/source/tests/test_source_select.py b/intake/interface/source/tests/test_source_select.py
similarity index 100%
rename from intake/gui/source/tests/test_source_select.py
rename to intake/interface/source/tests/test_source_select.py
diff --git a/intake/gui/source/tests/test_source_view.py b/intake/interface/source/tests/test_source_view.py
similarity index 100%
rename from intake/gui/source/tests/test_source_view.py
rename to intake/interface/source/tests/test_source_view.py
diff --git a/intake/gui/tests/__init__.py b/intake/interface/tests/__init__.py
similarity index 100%
rename from intake/gui/tests/__init__.py
rename to intake/interface/tests/__init__.py
diff --git a/intake/gui/tests/catalogs/catalog1.yaml b/intake/interface/tests/catalogs/catalog1.yaml
similarity index 100%
rename from intake/gui/tests/catalogs/catalog1.yaml
rename to intake/interface/tests/catalogs/catalog1.yaml
diff --git a/intake/gui/tests/catalogs/catalog2.yaml b/intake/interface/tests/catalogs/catalog2.yaml
similarity index 100%
rename from intake/gui/tests/catalogs/catalog2.yaml
rename to intake/interface/tests/catalogs/catalog2.yaml
diff --git a/intake/gui/tests/catalogs/parent.yaml b/intake/interface/tests/catalogs/parent.yaml
similarity index 100%
rename from intake/gui/tests/catalogs/parent.yaml
rename to intake/interface/tests/catalogs/parent.yaml
diff --git a/intake/gui/tests/data/crime.csv b/intake/interface/tests/data/crime.csv
similarity index 100%
rename from intake/gui/tests/data/crime.csv
rename to intake/interface/tests/data/crime.csv
diff --git a/intake/gui/tests/test_base.py b/intake/interface/tests/test_base.py
similarity index 100%
rename from intake/gui/tests/test_base.py
rename to intake/interface/tests/test_base.py
diff --git a/intake/gui/tests/test_gui.py b/intake/interface/tests/test_gui.py
similarity index 100%
rename from intake/gui/tests/test_gui.py
rename to intake/interface/tests/test_gui.py
diff --git a/intake/gui/tests/test_init_gui.py b/intake/interface/tests/test_init_gui.py
similarity index 100%
rename from intake/gui/tests/test_init_gui.py
rename to intake/interface/tests/test_init_gui.py
diff --git a/intake/source/__init__.py b/intake/source/__init__.py
index 7e69080..da23227 100644
--- a/intake/source/__init__.py
+++ b/intake/source/__init__.py
@@ -5,13 +5,29 @@
 # The full license is in the LICENSE file, distributed with this software.
 #-----------------------------------------------------------------------------
 
+import entrypoints
 import logging
-logger = logging.getLogger('intake')
 
 from ..utils import DriverRegistryView
+from .base import DataSource
+
+logger = logging.getLogger('intake')
+
+
+class DriverRegistry(dict):
+    """Dict of driver: DataSource class
+
+    If the value object is a EntryPoint, will load it when accesses, which
+    does the import.
+    """
+
+    def __getitem__(self, item):
+        if isinstance(super().__getitem__(item), entrypoints.EntryPoint):
+            self[item] = super().__getitem__(item).load()
+        return super().__getitem__(item)
+
 
-# The registry is the mapping of plugin name (aka driver) to DataSource class
-_registry = {}  # internal mutable registry
+_registry = DriverRegistry()  # internal mutable registry
 registry = DriverRegistryView(_registry)  # public, read-ony wrapper
 
 
@@ -63,7 +79,7 @@ def unregister_driver(name):
     return _registry.pop(name, None)
 
 
-# A set of fully-qualifies package.module.Class mappings
+# A set of fully-qualified package.module.Class mappings
 classes = {}
 
 
diff --git a/intake/source/discovery.py b/intake/source/discovery.py
index 95cf159..09f0d51 100644
--- a/intake/source/discovery.py
+++ b/intake/source/discovery.py
@@ -15,13 +15,11 @@ import logging
 
 import entrypoints
 
-from .base import DataSource
-from ..catalog.base import Catalog
 from ..config import save_conf, conf, cfile
 logger = logging.getLogger('intake')
 
 
-def autodiscover(path=None, plugin_prefix='intake_', do_package_scan=True):
+def autodiscover(path=None, plugin_prefix='intake_', do_package_scan=False):
     r"""Discover intake drivers.
 
     In order of decreasing precedence:
@@ -33,8 +31,7 @@ def autodiscover(path=None, plugin_prefix='intake_', do_package_scan=True):
       ``intake\_``. Import them and scan them for subclasses of
       ``intake.source.base.Plugin``. This was previously the *only* mechanism
       for auto-discoverying intake drivers, and it is maintained for backward
-      compatibility. In a future release, intake will issue a warning if any
-      packages are located by the method that do not also have entrypoints.
+      compatibility.
 
     Parameters
     ----------
@@ -43,8 +40,8 @@ def autodiscover(path=None, plugin_prefix='intake_', do_package_scan=True):
     plugin_prefix : str
         DEPRECATED. Default is 'intake\_'.
     do_package_scan : boolean
-        Default is True. In the future, the default will be changed to False,
-        and the option may eventually be removed entirely.
+        Whether to look for intake source classes in packages named
+        "intake_*". This has been superceded by entrypoints declarations.
 
     Returns
     -------
@@ -297,6 +294,8 @@ def load_plugins_from_module(module_name):
     Plugin classes are instantiated and added to the dictionary, keyed by the
     name attribute of the plugin object.
     """
+    from intake.source import DataSource
+    from intake.catalog import Catalog
     plugins = {}
 
     try:
@@ -358,3 +357,9 @@ def disable(name):
         conf['drivers'] = {}
     conf['drivers'][name] = False
     save_conf()
+
+
+def register_all():
+    from intake.source import register_driver
+    for name, driver in autodiscover().items():
+        register_driver(name, driver)
diff --git a/intake/utils.py b/intake/utils.py
index e71ec25..ef0dd25 100644
--- a/intake/utils.py
+++ b/intake/utils.py
@@ -11,6 +11,7 @@ import datetime
 from contextlib import contextmanager
 import warnings
 import yaml
+import sys
 
 
 def make_path_posix(path):
@@ -242,7 +243,6 @@ class RegistryView(collections.abc.Mapping):
             f"not be directly mutable. Use {self._unregister_func_name}.",
             DeprecationWarning)
         del self._registry[key]
-        self._registry[key] = value
         # raise TypeError(
         #     f"The registry cannot be edited directly. "
         #     f"Instead, use the {self._unregister_func_name{")
@@ -262,3 +262,21 @@ class ContainerRegistryView(RegistryView):
     _registry_name = "intake.container_map"
     _register_func_name  = "intake.register_container"
     _unregister_func_name  = "intake.unregister_container"
+
+
+class ModuleImporter:
+    def __init__(self, destination):
+        self.destination = destination
+        self.module = None
+
+    def __getattribute__(self, item):
+        d = object.__getattribute__(self, "__dict__")
+        if item in d:
+            return d[item]
+        if self.module is None:
+            print("Importing module: ", self.destination)
+            self.module = __import__(self.destination)
+        else:
+            print("Referencing module: ", self.destination)
+        sys.modules[self.destination] = self.module
+        return getattr(self.module, item)
diff --git a/setup.py b/setup.py
index 0e71f2c..91e0baa 100644
--- a/setup.py
+++ b/setup.py
@@ -49,7 +49,7 @@ setup(
             'csv = intake.source.csv:CSVSource',
             'textfiles = intake.source.textfiles:TextFilesSource',
             'catalog = intake.catalog.base:Catalog',
-            'intake_remote = intake.catalog.base:RemoteCatalog',
+            'intake_remote = intake.catalog.remote:RemoteCatalog',
             'numpy = intake.source.npy:NPySource',
             'ndzarr = intake.source.zarr:ZarrArraySource',
             'zarr_cat = intake.catalog.zarr:ZarrGroupCatalog',


Test patch:
diff --git a/intake/catalog/tests/test_gui.py b/intake/catalog/tests/test_gui.py
index 86aed8e..e7d231e 100644
--- a/intake/catalog/tests/test_gui.py
+++ b/intake/catalog/tests/test_gui.py
@@ -15,6 +15,7 @@ def panel_importable():
     except:
         return False
 
+
 EXPECTED_ERROR_TEXT = "Please install panel to use the GUI"
 
 
diff --git a/intake/catalog/tests/test_remote_integration.py b/intake/catalog/tests/test_remote_integration.py
index 1144f1b..cfc00e7 100644
--- a/intake/catalog/tests/test_remote_integration.py
+++ b/intake/catalog/tests/test_remote_integration.py
@@ -13,7 +13,8 @@ import pandas as pd
 
 from intake.source.tests.util import verify_datasource_interface
 from .util import assert_items_equal
-from intake import open_catalog, RemoteCatalog
+from intake import open_catalog
+from intake.catalog.remote import RemoteCatalog
 
 TEST_CATALOG_PATH = os.path.join(os.path.dirname(__file__), 'catalog1.yml')
 
diff --git a/intake/source/tests/test_discovery.py b/intake/source/tests/test_discovery.py
index 2852bc1..403db89 100644
--- a/intake/source/tests/test_discovery.py
+++ b/intake/source/tests/test_discovery.py
@@ -46,9 +46,14 @@ def test_discover_cli(extra_pythonpath, tmp_config_path):
     env["INTAKE_CONF_FILE"] = tmp_config_path
     env['PYTHONPATH'] = extra_pythonpath
 
+    # directory is not automatically scanned any more
+    subprocess.call(shlex.split(
+        "intake drivers enable foo intake_foo.FooPlugin"
+    ), stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=env)
+
     out = subprocess.check_output(shlex.split(
         "intake drivers list"
-    ), stderr=subprocess.STDOUT, env=env)
+    ), stderr=subprocess.STDOUT)
 
     assert b'foo' in out
     assert out.index(b'Not enabled') > out.index(b'foo')
@@ -61,12 +66,12 @@ def test_discover_cli(extra_pythonpath, tmp_config_path):
         "intake drivers list"
     ), stderr=subprocess.STDOUT, env=env)
     assert b'foo' in out
-    assert b'foo' in out[out.index(b'Not enabled'):]
+    assert out.index(b'Not enabled') < out.index(b'foo')
 
 
 def test_discover(extra_pythonpath, tmp_config_path):
     with pytest.warns(PendingDeprecationWarning):
-        registry = discovery.autodiscover()
+        registry = discovery.autodiscover(do_package_scan=True)
 
     # Check that package scan (name-based) discovery worked.
     assert 'foo' in registry
@@ -78,7 +83,7 @@ def test_discover(extra_pythonpath, tmp_config_path):
     # Now again, giving the special PYTHONPATH explicit via kwarg.
 
     with pytest.warns(PendingDeprecationWarning):
-        registry = discovery.autodiscover(path=[extra_pythonpath])
+        registry = discovery.autodiscover(path=[extra_pythonpath], do_package_scan=True)
 
     # Check that package scan (name-based) discovery worked.
     assert 'foo' in registry
@@ -99,18 +104,19 @@ def test_discover(extra_pythonpath, tmp_config_path):
     assert 'some_test_driver' in registry
     registry['some_test_driver']()
 
+
 def test_enable_and_disable(extra_pythonpath, tmp_config_path):
     # Disable and then enable a package scan result.
 
     try:
         discovery.disable('foo')
         with pytest.warns(PendingDeprecationWarning):
-            registry = discovery.autodiscover()
+            registry = discovery.autodiscover(do_package_scan=True)
         assert 'foo' not in registry
 
         discovery.enable('foo', 'intake_foo.FooPlugin')
         with pytest.warns(PendingDeprecationWarning):
-            registry = discovery.autodiscover()
+            registry = discovery.autodiscover(do_package_scan=True)
         assert 'foo' in registry
     finally:
         discovery.enable('foo', 'intake_foo.FooPlugin')
@@ -120,14 +126,14 @@ def test_enable_and_disable(extra_pythonpath, tmp_config_path):
     try:
         discovery.disable('some_test_driver')
         with pytest.warns(PendingDeprecationWarning):
-            registry = discovery.autodiscover()
+            registry = discovery.autodiscover(do_package_scan=True)
         assert 'some_test_driver' not in registry
 
         discovery.enable(
             'some_test_driver',
             'driver_with_entrypoints.SomeTestDriver')
         with pytest.warns(PendingDeprecationWarning):
-            registry = discovery.autodiscover()
+            registry = discovery.autodiscover(do_package_scan=True)
         assert 'some_test_driver' in registry
     finally:
         discovery.enable(
@@ -135,10 +141,10 @@ def test_enable_and_disable(extra_pythonpath, tmp_config_path):
             'driver_with_entrypoints.SomeTestDriver')
 
 
-
 def test_discover_pluginprefix(extra_pythonpath, tmp_config_path):
     with pytest.warns(PendingDeprecationWarning):
-        registry = discovery.autodiscover(plugin_prefix='not_intake_')
+        registry = discovery.autodiscover(plugin_prefix='not_intake_',
+                                          do_package_scan=True)
 
     assert 'otherfoo' in registry
     registry['otherfoo']()
@@ -147,4 +153,4 @@ def test_discover_pluginprefix(extra_pythonpath, tmp_config_path):
 
 def test_discover_collision(extra_pythonpath, tmp_config_path):
     with pytest.warns(UserWarning):
-        discovery.autodiscover(plugin_prefix='collision_')
+        discovery.autodiscover(plugin_prefix='collision_', do_package_scan=True)
diff --git a/intake/tests/test_top_level.py b/intake/tests/test_top_level.py
index dea2180..6ff4216 100644
--- a/intake/tests/test_top_level.py
+++ b/intake/tests/test_top_level.py
@@ -10,6 +10,7 @@ import posixpath
 import shutil
 import time
 import tempfile
+import sys
 
 import intake
 import appdirs
@@ -114,3 +115,16 @@ def test_output_notebook():
 def test_old_usage():
     with pytest.warns(UserWarning):
         intake.Catalog()
+
+
+def test_no_imports():
+    mods = [mod for mod in sys.modules if mod.startswith('intake')]
+    [sys.modules.pop(mod) for mod in mods]
+
+    import intake
+
+    assert 'intake' in sys.modules
+
+    for mod in ['intake.tests', 'intake.interface', 'intake.source.csv',
+                'intake.cli', 'intake.auth']:
+        assert mod not in sys.modules