Repository: intake/intake

Issue:
Make catalogs lazy by default
Currently, the Catalog base does the following on instantiation:
```
        self._entries = self._make_entries_container()  # gives {} by default
        self.force_reload()
```
so that `_load()` is called at that time (and whenever the TTL runs out, which is on by default!). Now, we *can* construct special lazy versions of `_entries`, but it would be nice if the default were to only do any loading when entries are actually accessed in any way.

Related to #505 (where there is no loading of entries in zarrcat's init, only _load, which is the recommended thing to do).

Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
index 0909fd5..2a472d4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,11 +8,6 @@ branches:
   only:
   - master
 
-env:
-  global:
-    # ANACONDA_TOKEN
-    secure: XJ+hvBFtSNUy6xYQmpEFVoR1MTYdTtLIN9R46qfW5XhAOEzuA0mDBd4FL9mgaw4+hoB4+3B8ItHquGxRN/ugw8O/VRA5CuGBbmfbJfG2vLKAbe3RruTueijbxIpiPAhgEp11HtLWGYiaXR47Hug/hO5nFnHAqF9DpQfvsO5ntlFcMnNRVFp7uHbkdz651yQ3en4frSLP/07sK7zGMculb5VKeFH7RdD7Imyqi/BqXX3Mq4ymcgJ/lSEbgO46hP1wX6q4/SSFEQyb4tUUgM6yK6nZEjs6R6DLXpW9PFncTHsjBu39+aZR5tFQqRFOMoaVY2WVxDFSC4Hx8kUXuDQMfuR4JgVOEDSDLr8iws7Fr7svWEpz0TfasbN2/j4LSL+4bjtpZE/Baz3TrUf1LsmbjIgGIDiruLcVpQLrD3msHjjHiyldUs3gDCjndPuaxg0f3AKz/04xNXTNxg+RyVIGjP/6dHDRNIyHfToBhkt42wjt4bDiyxK78ty5Qn+EnDPJn1R22dULFXTwMnEO9QbzJCKZz2xwClp8toxHYRTUsjD68IDxipqd4gn6R16OgMcA9wtzomoz5UmFbo3ambUluzMVv3v9KS4spqykPWF8wTuGfJy2pEJAm6E9bwa4Jw1RZx9GWSYp/Zm8QR0Vx10dTTy/X7/wX44f/P9pv/dQUiw=
-
 jobs:
   include:
     # ANACONDA
diff --git a/intake/catalog/base.py b/intake/catalog/base.py
index e05514d..7861a3a 100644
--- a/intake/catalog/base.py
+++ b/intake/catalog/base.py
@@ -50,7 +50,7 @@ class Catalog(DataSource):
     name = 'catalog'
 
     def __init__(self, *args, name=None, description=None, metadata=None,
-                 auth=None, ttl=1, getenv=True, getshell=True,
+                 auth=None, ttl=60, getenv=True, getshell=True,
                  persist_mode='default', storage_options=None):
         """
         Parameters
@@ -167,8 +167,8 @@ class Catalog(DataSource):
 
     def force_reload(self):
         """Imperative reload data now"""
-        self._load()
         self.updated = time.time()
+        self._load()
 
     def reload(self):
         """Reload catalog if sufficient time has passed"""
@@ -403,6 +403,8 @@ class Catalog(DataSource):
             e = self._entries[key]
             e._catalog = self
             e._pmode = self.pmode
+            if e._container == 'catalog':
+                return e(name=key)
             return e()
         if isinstance(key, str) and '.' in key:
             key = key.split('.')
diff --git a/intake/catalog/entry.py b/intake/catalog/entry.py
index 1946c92..7b7a07f 100644
--- a/intake/catalog/entry.py
+++ b/intake/catalog/entry.py
@@ -120,6 +120,9 @@ class CatalogEntry(DictSerialiseMixin):
             'application/json': {'root': contents["name"]}
         }, raw=True)
 
+    def _yaml(self):
+        return {"sources": {self.name: self.describe()}}
+
     def __iter__(self):
         # If the entry is a catalog, this allows list(cat.entry)
         if self._container == 'catalog':
diff --git a/intake/catalog/local.py b/intake/catalog/local.py
index 6d1e3af..b6c7143 100644
--- a/intake/catalog/local.py
+++ b/intake/catalog/local.py
@@ -80,7 +80,7 @@ class UserParameter(DictSerialiseMixin):
                             for item in self.allowed]
 
     def __repr__(self):
-        return (f'<{self.__class__.__name__} {self.name!r}>')
+        return f'<{self.__class__.__name__} {self.name!r}>'
 
     __str__ = __repr__
 
@@ -226,7 +226,8 @@ class LocalCatalogEntry(CatalogEntry):
         return {
             'name': self._name,
             'container': self._container,
-            'plugin': pl,
+            'plugin': pl,   # deprecated
+            'driver': pl,
             'description': self._description,
             'direct_access': self._direct_access,
             'user_parameters': [u.describe() for u in self._user_parameters],
@@ -564,6 +565,7 @@ class YAMLFileCatalog(Catalog):
         self.text = None
         self.autoreload = autoreload  # set this to False if don't want reloads
         self.filesystem = kwargs.pop('fs', None)
+        self.access = "name" not in kwargs
         super(YAMLFileCatalog, self).__init__(**kwargs)
 
     def _load(self, reload=False):
@@ -572,6 +574,11 @@ class YAMLFileCatalog(Catalog):
         Will do nothing if auto-reload is off and reload is not explicitly
         requested
         """
+        if self.access is False:
+            # skip first load, if cat has given name (i.e., is subcat)
+            self.updated = 0
+            self.access = True
+            return
         if self.autoreload or reload:
             # First, we load from YAML, failing if syntax errors are found
             options = self.storage_options or {}
@@ -722,11 +729,17 @@ class YAMLFilesCatalog(Catalog):
         self._kwargs = kwargs.copy()
         self._cat_files = []
         self._cats = {}
+        self.access = "name" not in kwargs
         super(YAMLFilesCatalog, self).__init__(**kwargs)
 
     def _load(self):
         # initial: find cat files
         # if flattening, need to get all entries from each.
+        if self.access is False:
+            # skip first load, if cat has given name (i.e., is subcat)
+            self.updated = 0
+            self.access = True
+            return
         self._entries.clear()
         options = self.storage_options or {}
         if isinstance(self.path, (list, tuple)):
diff --git a/intake/catalog/zarr.py b/intake/catalog/zarr.py
index 732b377..19609bc 100644
--- a/intake/catalog/zarr.py
+++ b/intake/catalog/zarr.py
@@ -12,7 +12,7 @@ class ZarrGroupCatalog(Catalog):
     name = 'zarr_cat'
 
     def __init__(self, urlpath, storage_options=None, component=None, metadata=None,
-                 consolidated=False):
+                 consolidated=False, name=None):
         """
 
         Parameters
@@ -35,6 +35,7 @@ class ZarrGroupCatalog(Catalog):
         self._component = component
         self._consolidated = consolidated
         self._grp = None
+        self.name = name
         super().__init__(metadata=metadata)
 
     def _load(self):
diff --git a/intake/cli/server/__main__.py b/intake/cli/server/__main__.py
index 01af63a..1f8b566 100644
--- a/intake/cli/server/__main__.py
+++ b/intake/cli/server/__main__.py
@@ -39,7 +39,8 @@ def main(argv=None):
                         help='Name of catalog YAML file')
     parser.add_argument('--flatten', dest='flatten', action='store_true')
     parser.add_argument('--no-flatten', dest='flatten', action='store_false')
-    parser.add_argument('-a', '--address', type=str, 
+    parser.add_argument('--ttl', dest='ttl',type=int, default=60)
+    parser.add_argument('-a', '--address', type=str,
                         default=conf.get('address', 'localhost'),
                         help='address to use as a host, defaults to the address '
                         'in the configuration file, if provided otherwise localhost')
@@ -54,11 +55,13 @@ def main(argv=None):
         logger.info('  - %s' % arg)
 
     catargs = args.catalog_args
+    ttl = args.ttl
+
     if len(catargs) == 1:
-        catalog = open_catalog(catargs[0])
+        catalog = open_catalog(catargs[0], ttl=ttl)
         logger.info("catalog_args: %s" % catargs[0])
     else:
-        catalog = open_catalog(catargs, flatten=args.flatten)
+        catalog = open_catalog(catargs, flatten=args.flatten, ttl=ttl)
         logger.info("catalog_args: %s" % catargs)
     if args.list_entries:
         # This is not a good idea if the Catalog is huge.
diff --git a/requirements.txt b/requirements.txt
index 19f4b05..f4c6ccb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,3 +8,4 @@ pyyaml
 requests
 msgpack
 fsspec >=0.7.4
+aiohttp
diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh
index 525371e..39a13ad 100755
--- a/scripts/ci/install.sh
+++ b/scripts/ci/install.sh
@@ -9,5 +9,5 @@ conda config --set auto_update_conda off
 
 echo "Installing test dependencies."
 conda install --yes appdirs dask jinja2 numpy pyyaml requests msgpack-numpy pytest-cov coveralls \
-    pytest fsspec intake-parquet zarr notebook panel hvplot bokeh dask -c conda-forge -c defaults
+    pytest fsspec intake-parquet zarr aiohttp notebook panel hvplot bokeh dask -c conda-forge -c defaults
 pip install -e . --no-deps


Test patch:
diff --git a/intake/catalog/tests/test_auth_integration.py b/intake/catalog/tests/test_auth_integration.py
index 4ba8ec5..9d6ae92 100644
--- a/intake/catalog/tests/test_auth_integration.py
+++ b/intake/catalog/tests/test_auth_integration.py
@@ -82,4 +82,4 @@ def test_secret_auth(intake_server_with_auth):
 def test_secret_auth_fail(intake_server_with_auth):
     auth = SecretClientAuth(secret='test_wrong_secret')
     with pytest.raises(AuthenticationFailure):
-        open_catalog(intake_server_with_auth, auth=auth)
+        list(open_catalog(intake_server_with_auth, auth=auth))
diff --git a/intake/catalog/tests/test_local.py b/intake/catalog/tests/test_local.py
index d271afc..05681d6 100644
--- a/intake/catalog/tests/test_local.py
+++ b/intake/catalog/tests/test_local.py
@@ -40,7 +40,8 @@ def test_local_catalog(catalog1):
         'description': 'entry1 full',
         'args': {'urlpath': '{{ CATALOG_DIR }}/entry1_*.csv'},
         'metadata': {'bar': [1, 2, 3], 'foo': 'bar'},
-        'plugin': ['csv']
+        'plugin': ['csv'],
+        'driver': ['csv']
     }
     assert catalog1['entry1_part'].describe() == {
         'name': 'entry1_part',
@@ -58,7 +59,8 @@ def test_local_catalog(catalog1):
         'direct_access': 'allow',
         'args': {'urlpath': '{{ CATALOG_DIR }}/entry1_{{ part }}.csv'},
         'metadata': {'foo': 'baz', 'bar': [2, 4, 6]},
-        'plugin': ['csv']
+        'plugin': ['csv'],
+        'driver': ['csv']
     }
     assert catalog1['entry1'].container == 'dataframe'
     md = catalog1['entry1'].metadata
@@ -266,7 +268,7 @@ def test_user_parameter_validation_allowed():
 ])
 def test_parser_validation_error(filename):
     with pytest.raises(exceptions.ValidationError):
-        open_catalog(abspath(filename + ".yml"))
+        list(open_catalog(abspath(filename + ".yml")))
 
 
 @pytest.mark.parametrize("filename", [
@@ -379,11 +381,11 @@ sources:
 
 def test_catalog_file_removal(temp_catalog_file):
     cat_dir = os.path.dirname(temp_catalog_file)
-    cat = open_catalog(cat_dir + '/*')
+    cat = open_catalog(cat_dir + '/*', ttl=0.1)
     assert set(cat) == {'a', 'b'}
 
     os.remove(temp_catalog_file)
-    time.sleep(1.5)  # wait for catalog refresh
+    time.sleep(0.5)  # wait for catalog refresh
     assert set(cat) == set()
 
 
@@ -435,6 +437,7 @@ def test_cat_with_declared_name():
     cat = open_catalog(fn, name='name_in_func', description=description)
     assert cat.name == 'name_in_func'
     assert cat.description == description
+    cat._load()  # we don't get metadata until load/list/getitem
     assert cat.metadata.get('some') == 'thing'
 
     cat = open_catalog(fn)
diff --git a/intake/catalog/tests/test_reload_integration.py b/intake/catalog/tests/test_reload_integration.py
index 6fc75de..4af1d5f 100644
--- a/intake/catalog/tests/test_reload_integration.py
+++ b/intake/catalog/tests/test_reload_integration.py
@@ -54,7 +54,7 @@ sources:
 
 
 def test_reload_updated_config(intake_server_with_config):
-    catalog = open_catalog(intake_server_with_config)
+    catalog = open_catalog(intake_server_with_config, ttl=0.1)
 
     entries = list(catalog)
     assert entries == ['use_example1']
@@ -76,13 +76,13 @@ sources:
     args: {}
         ''')
 
-    time.sleep(2)
+    time.sleep(1.2)
 
     assert_items_equal(list(catalog), ['use_example1', 'use_example1_1'])
 
 
 def test_reload_updated_directory(intake_server_with_config):
-    catalog = open_catalog(intake_server_with_config)
+    catalog = open_catalog(intake_server_with_config, ttl=0.1)
 
     orig_entries = list(catalog)
     assert 'example2' not in orig_entries
@@ -98,7 +98,7 @@ sources:
         urlpath: none
         ''')
 
-    time.sleep(2)
+    time.sleep(1.2)
 
     assert_items_equal(list(catalog), ['example2'] + orig_entries)
 
@@ -110,7 +110,7 @@ def test_reload_missing_remote_directory(intake_server):
         pass
 
     time.sleep(1)
-    catalog = open_catalog(intake_server)
+    catalog = open_catalog(intake_server, ttl=0.1)
     assert_items_equal(list(catalog), [])
 
     os.mkdir(TMP_DIR)
@@ -126,7 +126,7 @@ sources:
     driver: example1
     args: {}
         ''')
-    time.sleep(2)
+    time.sleep(1.2)
 
     assert_items_equal(list(catalog), ['use_example1'])
     try:
@@ -136,7 +136,7 @@ sources:
 
 
 def test_reload_missing_local_directory(tempdir):
-    catalog = open_catalog(tempdir + '/*')
+    catalog = open_catalog(tempdir + '/*', ttl=0.1)
     assert_items_equal(list(catalog), [])
 
     with open(os.path.join(tempdir, YAML_FILENAME), 'w') as f:
@@ -152,5 +152,5 @@ sources:
     args: {}
         ''')
 
-    time.sleep(1)
+    time.sleep(1.2)
     assert 'use_example1' in catalog
diff --git a/intake/cli/client/tests/test_local_integration.py b/intake/cli/client/tests/test_local_integration.py
index 3279cba..24d41ed 100644
--- a/intake/cli/client/tests/test_local_integration.py
+++ b/intake/cli/client/tests/test_local_integration.py
@@ -51,6 +51,7 @@ def test_describe():
 [entry1] container=dataframe
 [entry1] description=entry1 full
 [entry1] direct_access=forbid
+[entry1] driver=['csv']
 [entry1] metadata={'foo': 'bar', 'bar': [1, 2, 3]}
 [entry1] name=entry1
 [entry1] plugin=['csv']
diff --git a/intake/conftest.py b/intake/conftest.py
index 7a969b9..d179bd3 100644
--- a/intake/conftest.py
+++ b/intake/conftest.py
@@ -104,7 +104,7 @@ def intake_server(request):
         env['INTAKE_CONF_FILE'] = server_conf
     port = pick_port()
     cmd = [ex, '-m', 'intake.cli.server', '--sys-exit-on-sigterm',
-           '--port', str(port)]
+           '--port', str(port), '--ttl', '1']
     if isinstance(catalog_path, list):
         cmd.extend(catalog_path)
     else: