Repository: intake/intake
Pull request #394 (2019-07-26 18:52:19)

Add a load of tests
Pushes coverage over 90%

Anyone know why the coveralls upload isn't working?


Patch:
diff --git a/.travis.yml b/.travis.yml
index b873cca..ae2a612 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,7 +21,7 @@ matrix:
         - export PATH="$HOME/miniconda3/bin:$PATH"
         - export CONDA_PY=36
       install: scripts/ci/install.sh
-      script: pytest
+      script: pytest --cov=intake
       after_success:
         - coveralls
     # PYTHON
diff --git a/intake/__init__.py b/intake/__init__.py
index 52582b9..71dbcf4 100644
--- a/intake/__init__.py
+++ b/intake/__init__.py
@@ -6,6 +6,7 @@
 #-----------------------------------------------------------------------------
 
 import re
+import logging
 import warnings
 
 from ._version import get_versions
@@ -31,27 +32,25 @@ registry['catalog'] = Catalog
 registry['intake_remote'] = RemoteCatalog
 registry['numpy'] = npy.NPySource
 registry['ndzarr'] = zarr.ZarrArraySource
+logger = logging.getLogger('intake')
 
 
 def make_open_functions():
     """From the current state of ``registry``, create open_* functions"""
-    # Create shortcut open methods
-    if hasattr(str, 'isidentifier'):
-        def isidentifier(x):
-            return x.isidentifier()
-    else:
-        IDENTIFIER_REGEX = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*$')
-        isidentifier = IDENTIFIER_REGEX.match
     for plugin_name, plugin in registry.items():
-        func_name = 'open_' + plugin_name
-        if not isidentifier(func_name):
-            # primitive name normalization
-            func_name = re.sub('[-=~^&|@+]', '_', func_name)
-        if isidentifier(func_name):
-            globals()[func_name] = plugin
-        else:
-            warnings.warn('Invalid Intake plugin name "%s" found.' %
-                          plugin_name)
+        try:
+            func_name = 'open_' + plugin_name
+            if not func_name.isidentifier():
+                # primitive name normalization
+                func_name = re.sub('[-=~^&|@+]', '_', func_name)
+            if func_name.isidentifier():
+                globals()[func_name] = plugin
+            else:
+                warnings.warn('Invalid Intake plugin name "%s" found.' %
+                              plugin_name)
+        except:
+            logger.warning("Creation of open function failed for %s"
+                           "" % plugin_name)
 
 
 def output_notebook(inline=True, logo=False):
diff --git a/intake/catalog/local.py b/intake/catalog/local.py
index b2139cb..6e5a9cb 100644
--- a/intake/catalog/local.py
+++ b/intake/catalog/local.py
@@ -643,7 +643,7 @@ class YAMLFileCatalog(Catalog):
         if path:
             return self
         else:
-            return YAMLFileCatalog(path, storage_options=storage_options,
+            return YAMLFileCatalog(self.path, storage_options=storage_options,
                                    autoreload=self.autoreload)
 
     def parse(self, text):
diff --git a/intake/cli/__init__.py b/intake/cli/__init__.py
index fd00168..502f050 100644
--- a/intake/cli/__init__.py
+++ b/intake/cli/__init__.py
@@ -4,3 +4,9 @@
 #
 # The full license is in the LICENSE file, distributed with this software.
 #-----------------------------------------------------------------------------
+
+import os
+
+if os.getenv("COVERAGE_PROCESS_START", False):
+    import coverage
+    coverage.process_startup()
diff --git a/intake/cli/client/subcommands/drivers.py b/intake/cli/client/subcommands/drivers.py
index a78e94a..86c6455 100644
--- a/intake/cli/client/subcommands/drivers.py
+++ b/intake/cli/client/subcommands/drivers.py
@@ -29,6 +29,7 @@ from intake import __version__
 from intake.cli.util import Subcommand
 from intake.source.discovery import autodiscover, autodiscover_all, enable, disable
 from intake.config import confdir
+import intake
 
 #-----------------------------------------------------------------------------
 # API
@@ -45,15 +46,19 @@ class Drivers(Subcommand):
         sub_parser = self.parser.add_subparsers()
 
         list = sub_parser.add_parser('list', help='Show all intake drivers.')
-        list.add_argument('-v', '--verbose', action='store_true', help='Show module path.')
+        list.add_argument(
+            '-v', '--verbose', action='store_true', help='Show module path.')
         list.set_defaults(invoke=self._list)
 
         enable = sub_parser.add_parser('enable', help='Enable an intake driver.')
         enable.add_argument('name', type=str, help='Driver name')
-        enable.add_argument('driver', type=str, help='Module path and class name, as in package.submodule.ClassName')
+        enable.add_argument('driver', type=str,
+                            help='Module path and class name, as in '
+                                 'package.submodule.ClassName')
         enable.set_defaults(invoke=self._enable)
 
-        disable = sub_parser.add_parser('disable', help='Disable one or more intake drivers.')
+        disable = sub_parser.add_parser(
+            'disable', help='Disable one or more intake drivers.')
         disable.add_argument('names', type=str, help='Driver names', nargs='+')
         disable.set_defaults(invoke=self._disable)
 
@@ -67,15 +72,24 @@ class Drivers(Subcommand):
             fmt = '{name:<30}{cls.__module__}.{cls.__name__}'
         drivers_by_name = autodiscover()   # dict mapping name to driver
         all_drivers = autodiscover_all()  # listof (name, driver)
-        print("Enabled:", file=sys.stderr)
+        direct = {k: v for k, v in intake.registry.items()
+                  if k not in all_drivers and k not in drivers_by_name}
+
+        print("Direct:", file=sys.stderr)
+        for name, cls in direct.items():
+            print(fmt.format(name=str(name), cls=cls, file=inspect.getfile(cls)),
+                  file=sys.stderr)
+
+        print("\nEnabled:", file=sys.stderr)
         for name, cls in drivers_by_name.items():
-            print(fmt.format(name=name, cls=cls, file=inspect.getfile(cls)),
+            print(fmt.format(name=str(name), cls=cls, file=inspect.getfile(cls)),
                   file=sys.stderr)
+
         print("\nNot enabled:", file=sys.stderr)
         for name, cls in all_drivers:
-            if drivers_by_name[name] is not cls:
-                print(fmt.format(name=name, cls=cls, file=inspect.getfile(cls)),
-                    file=sys.stderr)
+            if drivers_by_name.get(name, None) is not cls:
+                print(fmt.format(name=str(name), cls=cls, file=inspect.getfile(cls)),
+                      file=sys.stderr)
 
     def _enable(self, args):
         enable(args.name, args.driver)
diff --git a/intake/gui/catalog/select.py b/intake/gui/catalog/select.py
index af6bbf0..f4d26ad 100644
--- a/intake/gui/catalog/select.py
+++ b/intake/gui/catalog/select.py
@@ -82,7 +82,6 @@ class CatSelector(BaseSelector):
 
         self.children = [label, self.widget]
 
-
     def callback(self, event):
         self.expand_nested(event.new)
         if self.done_callback:
diff --git a/intake/source/discovery.py b/intake/source/discovery.py
index 16a62dc..45b2faf 100644
--- a/intake/source/discovery.py
+++ b/intake/source/discovery.py
@@ -129,9 +129,10 @@ def autodiscover(path=None, plugin_prefix='intake_', do_package_scan=True):
 
     if package_scan_results:
         warnings.warn(
-            f"The drivers {package_scan_results} do not specify entry_points "
-            f"and were only discovered via a package scan. This may break in a "
-            f"future release of intake. The packages should be updated.",
+            f"The drivers {list(package_scan_results)} do not specify entry_"
+            f"points and were only discovered via a package scan. This may "
+            f"break in a future release of intake. The packages should be "
+            f"updated.",
             FutureWarning)
 
     # Load entrypoints. Any that were shadowed or banned have already been
@@ -211,15 +212,6 @@ def autodiscover_all(path=None, plugin_prefix='intake_', do_package_scan=True):
                      entrypoint.object_name)
         group_all.append(entrypoint)
 
-    # Discovery is complete.
-    
-    if package_scan_results:
-        warnings.warn(
-            f"The drivers {package_scan_results} do not specify entry_points "
-            f"and were only discovered via a package scan. This may break in a "
-            f"future release of intake. The packages should be updated.",
-            FutureWarning)
-
     # Load entrypoints. Any that were shadowed or banned have already been
     # removed above.
     drivers = []
diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh
index 480165b..c706889 100755
--- a/scripts/ci/install.sh
+++ b/scripts/ci/install.sh
@@ -8,7 +8,7 @@ echo "Configuring conda."
 conda config --set auto_update_conda off
 
 echo "Installing test dependencies."
-conda install --yes appdirs dask==2 nodejs jinja2 numpy pyyaml requests msgpack-numpy coveralls \
+conda install --yes appdirs dask==2 nodejs jinja2 numpy pyyaml requests msgpack-numpy pytest-cov coveralls \
     pytest fsspec intake-parquet zarr notebook panel==0.5.1 hvplot==0.4.0 -c conda-forge -c defaults
 pip install git+https://github.com/dask/dask --upgrade --no-deps
 npm install -g dat


Test patch:
diff --git a/intake/catalog/tests/test_local.py b/intake/catalog/tests/test_local.py
index 6b3e905..12ed6d7 100644
--- a/intake/catalog/tests/test_local.py
+++ b/intake/catalog/tests/test_local.py
@@ -690,3 +690,22 @@ sources:
     assert cat.explicit.read().equals(expected)
     s = cat.extra()
     assert s._storage_options['other']
+
+
+def test_cat_add(tmpdir):
+    tmpdir = str(tmpdir)
+    fn = os.path.join(tmpdir, 'cat.yaml')
+    with open(fn, 'w') as f:
+        f.write('sources: {}')
+    cat = open_catalog(fn)
+    assert list(cat) == []
+
+    # was added in memory
+    cat.add(cat)
+    cat._load()  # this would happen automatically, but not immediately
+    assert list(cat) == ['cat']
+
+    # was added to the file
+    cat = open_catalog(fn)
+    assert list(cat) == ['cat']
+
diff --git a/intake/catalog/tests/test_utils.py b/intake/catalog/tests/test_utils.py
index c71c183..4d35c61 100644
--- a/intake/catalog/tests/test_utils.py
+++ b/intake/catalog/tests/test_utils.py
@@ -8,6 +8,7 @@ import pytest
 import intake.catalog.utils as utils
 import pandas as pd
 
+
 def test_expand_templates():
     pars = {'a': "{{par}} hi"}
     context = {'b': 1, 'par': 'ho'}
@@ -35,3 +36,7 @@ def test_expand_nested_template():
 ])
 def test_coerce_datetime(test_input, expected):
     assert utils.coerce_datetime(test_input) == expected
+
+
+def test_flatten():
+    assert list(utils.flatten([["hi"], ["oi"]])) == ['hi', 'oi']
diff --git a/intake/gui/source/tests/test_source_view.py b/intake/gui/source/tests/test_source_view.py
index 13e1f34..90e84e4 100644
--- a/intake/gui/source/tests/test_source_view.py
+++ b/intake/gui/source/tests/test_source_view.py
@@ -66,9 +66,9 @@ def test_description_source_with_plots(sources2):
         'description: US Crime data [UCRDataTool](https://www.ucrdatatool.gov'
         '/Search/Crime/State/StatebyState.cfm)\n'
         'direct_access: forbid\n'
-        'user_parameters: []\n'
         'metadata: \n'
-        'args:\nurlpath: {{ CATALOG_DIR }}../data/crime.csv').split('\n')
+        'args:\nurlpath: {{ CATALOG_DIR }}../data/crime{{selector}}.csv'
+        '').split('\n')
     for line in lines:
         assert line in description.contents
     assert 'plots' in description.contents
diff --git a/intake/gui/tests/catalogs/catalog2.yaml b/intake/gui/tests/catalogs/catalog2.yaml
index d73f812..674888a 100644
--- a/intake/gui/tests/catalogs/catalog2.yaml
+++ b/intake/gui/tests/catalogs/catalog2.yaml
@@ -3,7 +3,13 @@ sources:
     description: US Crime data [UCRDataTool](https://www.ucrdatatool.gov/Search/Crime/State/StatebyState.cfm)
     driver: csv
     args:
-      urlpath: '{{ CATALOG_DIR }}../data/crime.csv'
+      urlpath: '{{ CATALOG_DIR }}../data/crime{{selector}}.csv'
+    parameters:
+      selector:
+        type: str
+        description: "none"
+        allowed: ["2", ""]
+        default: ""
     metadata:
       plots:
         line_example:
@@ -21,4 +27,4 @@ sources:
   fake2:
     driver: fake
     args:
-      urlpath: ''
\ No newline at end of file
+      urlpath: ''
diff --git a/intake/gui/tests/test_gui.py b/intake/gui/tests/test_gui.py
index 71e5dcf..7938090 100644
--- a/intake/gui/tests/test_gui.py
+++ b/intake/gui/tests/test_gui.py
@@ -13,6 +13,7 @@ def gui(cat1, cat2):
     from ..gui import GUI
     return GUI(cats=[cat1, cat2])
 
+
 def test_gui(gui, cat1, cat2, sources1):
     assert gui.cat.select.items == [cat1, cat2]
     assert gui.cats == [cat1]
@@ -31,6 +32,21 @@ def test_gui(gui, cat1, cat2, sources1):
     assert gui.source.plot_widget.disabled is False
 
 
+def test_par_selector(gui, cat2):
+    gui.cat.select.selected = [cat2]
+    assert gui.source.pars_widget.disabled is False
+
+    gui.source.pars_widget.value = True
+    wid = gui.source.pars_editor.panel[0]
+    assert isinstance(wid, pn.widgets.Select)
+    assert wid.value == ''
+    s = gui.source_instance
+    assert s.urlpath.endswith("crime.csv")
+    wid.value = "2"
+    s = gui.source_instance
+    assert s.urlpath.endswith("crime2.csv")
+
+
 def test_gui_remove_selected_cat(gui, cat1):
     gui.cat.select.remove_selected()
     assert gui.cat.search_widget.disabled is True
diff --git a/intake/source/tests/test_discovery.py b/intake/source/tests/test_discovery.py
index b2f91d7..2852bc1 100644
--- a/intake/source/tests/test_discovery.py
+++ b/intake/source/tests/test_discovery.py
@@ -7,6 +7,8 @@
 
 import os
 import os.path
+import shlex
+import subprocess
 import sys
 
 import pytest
@@ -39,6 +41,29 @@ def test_package_scan(extra_pythonpath, tmp_config_path):
     assert 'foo' in results
 
 
+def test_discover_cli(extra_pythonpath, tmp_config_path):
+    env = os.environ.copy()
+    env["INTAKE_CONF_FILE"] = tmp_config_path
+    env['PYTHONPATH'] = extra_pythonpath
+
+    out = subprocess.check_output(shlex.split(
+        "intake drivers list"
+    ), stderr=subprocess.STDOUT, env=env)
+
+    assert b'foo' in out
+    assert out.index(b'Not enabled') > out.index(b'foo')
+
+    subprocess.check_output(shlex.split(
+        "intake drivers disable foo"
+    ), stderr=subprocess.STDOUT, env=env)
+
+    out = subprocess.check_output(shlex.split(
+        "intake drivers list"
+    ), stderr=subprocess.STDOUT, env=env)
+    assert b'foo' in out
+    assert b'foo' in out[out.index(b'Not enabled'):]
+
+
 def test_discover(extra_pythonpath, tmp_config_path):
     with pytest.warns(PendingDeprecationWarning):
         registry = discovery.autodiscover()
diff --git a/intake/tests/test_top_level.py b/intake/tests/test_top_level.py
index dcd741f..90849ef 100644
--- a/intake/tests/test_top_level.py
+++ b/intake/tests/test_top_level.py
@@ -53,6 +53,7 @@ def tmp_path_catalog():
     # files already there)
     os.remove(target_catalog)
 
+
 def test_autoregister_open():
     assert hasattr(intake, 'open_csv')
 
@@ -65,10 +66,31 @@ def test_default_catalogs():
 
 def test_user_catalog(user_catalog):
     cat = intake.load_combo_catalog()
-    time.sleep(2) # wait 2 seconds for catalog to refresh
     assert set(cat) >= set(['ex1', 'ex2'])
 
 
+def test_open_styles(tmp_path_catalog):
+    cat = intake.Catalog(tmp_path_catalog)
+    cat2 = intake.open_catalog(tmp_path_catalog)
+    assert list(cat) == list(cat2)
+    cat2 = intake.open_catalog([tmp_path_catalog])
+    assert list(cat) == list(cat2)
+    cat2 = intake.open_catalog(os.path.join(
+        os.path.dirname(tmp_path_catalog), "*"))
+    assert list(cat) == list(cat2)
+    assert type(cat2).name == 'yaml_files_cat'
+    cat2 = intake.open_catalog(os.path.dirname(tmp_path_catalog))
+    assert list(cat) == list(cat2)
+    assert type(cat2).name == 'yaml_files_cat'
+    cat2 = intake.open_yaml_file_cat(tmp_path_catalog)
+    assert list(cat) == list(cat2)
+    cat2 = intake.open_yaml_files_cat([tmp_path_catalog])
+    assert list(cat) == list(cat2)
+    cat2 = intake.open_yaml_files_cat(os.path.join(
+        os.path.dirname(tmp_path_catalog), "*"))
+    assert list(cat) == list(cat2)
+
+
 def test_path_catalog(tmp_path_catalog):
     intake.config.conf['catalog_path'] = [posixpath.join(tempfile.gettempdir(), 'intake')]
     cat = intake.load_combo_catalog()