Repository: intake/intake
Pull request #381 (2019-07-11 16:58:53)

rebasing intake on fsspec instead of dask.bytes


Patch:
diff --git a/.travis.yml b/.travis.yml
index 8506219..b873cca 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,7 +21,7 @@ matrix:
         - export PATH="$HOME/miniconda3/bin:$PATH"
         - export CONDA_PY=36
       install: scripts/ci/install.sh
-      script: scripts/ci/build.sh
+      script: pytest
       after_success:
         - coveralls
     # PYTHON
@@ -34,6 +34,7 @@ matrix:
         - sudo apt update
         - sudo apt install libsnappy-dev
         - pip install -e .[complete]
+        - pip install git+https://github.com/dask/dask --upgrade --no-deps
         - pip install pytest
       script:
         - pytest
diff --git a/conda/meta.yaml b/conda/meta.yaml
index afc7a36..91b7e5b 100644
--- a/conda/meta.yaml
+++ b/conda/meta.yaml
@@ -31,6 +31,7 @@ requirements:
     - pyyaml
     - dask >=1.0
     - appdirs
+    - fsspec
 
 test:
   requires:
diff --git a/intake/__init__.py b/intake/__init__.py
index 6313977..52582b9 100644
--- a/intake/__init__.py
+++ b/intake/__init__.py
@@ -146,6 +146,8 @@ def open_catalog(uri=None, **kwargs):
         else:
             # empty cat
             driver = 'catalog'
+    if '_file' not in driver:
+        kwargs.pop('fs', None)
     if driver not in registry:
         raise ValueError('Unknown catalog driver (%s), supply one of: %s'
                          % (driver, list(sorted(registry))))
diff --git a/intake/catalog/base.py b/intake/catalog/base.py
index ae389b6..52fe20e 100644
--- a/intake/catalog/base.py
+++ b/intake/catalog/base.py
@@ -287,7 +287,7 @@ class Catalog(DataSource):
         storage_options : dict
             Extra arguments for the file-system
         """
-        from dask.bytes import open_files
+        from fsspec import open_files
         with open_files([url], **(storage_options or {}), mode='wt')[0] as f:
             f.write(self.serialize())
 
@@ -753,7 +753,7 @@ class RemoteCatalog(Catalog):
     @staticmethod
     def _data_to_source(cat, path, **kwargs):
         from intake.catalog.local import YAMLFileCatalog
-        from dask.bytes.core import open_files
+        from fsspec import open_files
         import yaml
         if not isinstance(cat, Catalog):
             raise NotImplementedError
diff --git a/intake/catalog/local.py b/intake/catalog/local.py
index 1384127..b2139cb 100644
--- a/intake/catalog/local.py
+++ b/intake/catalog/local.py
@@ -11,7 +11,8 @@ import os
 import posixpath
 
 from jinja2 import Template
-from dask.bytes import open_files
+from fsspec import open_files, get_filesystem_class
+from fsspec.core import split_protocol
 
 from .. import __version__
 from .base import Catalog, DataSource
@@ -175,6 +176,7 @@ class LocalCatalogEntry(CatalogEntry):
         self._user_parameters = parameters
         self._metadata = metadata or {}
         self._catalog_dir = catalog_dir
+        self._filesystem = None
         self._catalog = catalog
         if isinstance(driver, str):
             dr = get_plugin_class(driver)
@@ -241,8 +243,11 @@ class LocalCatalogEntry(CatalogEntry):
         if user_parameters.pop('cache', None) or self._cache:
             md['cache'] = user_parameters.pop('cache', None) or self._cache
         params = {'metadata': md,
-                  'CATALOG_DIR': self._catalog_dir}
+                  'CATALOG_DIR': self._catalog_dir,
+                  }
         params.update(self._open_args)
+        if 'storage_options' not in params and self._filesystem is not None:
+            params['storage_options'] = self._filesystem.storage_options
 
         open_args = merge_pars(params, user_parameters, self._user_parameters,
                                getshell=self.getshell, getenv=self.getenv,
@@ -266,7 +271,7 @@ class LocalCatalogEntry(CatalogEntry):
             try:
                 plugin = self._plugin[plugin]
             except KeyError:
-                raise ValueError('Attempt to select not available plugin %s, '
+                raise ValueError('Attempt to select unavailable plugin %s, '
                                  'perhaps import of plugin failed' % plugin)
         return plugin, open_args
 
@@ -521,7 +526,12 @@ def register_plugin_dir(path):
 
 def get_dir(path):
     if '://' in path:
-        return posixpath.dirname(path)
+        protocol, _ = split_protocol(path)
+        out = get_filesystem_class(protocol)._parent(path)
+        if "://" not in out:
+            # some FSs strip this, some do not
+            out = protocol + "://" + out
+        return out
     path = make_path_posix(
         os.path.join(os.getcwd(), os.path.dirname(path)))
     if path[-1] != '/':
@@ -549,6 +559,7 @@ class YAMLFileCatalog(Catalog):
         self.path = path
         self.text = None
         self.autoreload = autoreload  # set this to False if don't want reloads
+        self.filesystem = kwargs.pop('fs', None)
         super(YAMLFileCatalog, self).__init__(**kwargs)
 
     def _load(self, reload=False):
@@ -565,10 +576,13 @@ class YAMLFileCatalog(Catalog):
                 self.path = make_path_posix(
                     getattr(self.path, 'path',
                             getattr(self.path, 'name', 'file')))
-            else:
+            elif self.filesystem is None:
                 file_open = open_files(self.path, mode='rb', **options)
                 assert len(file_open) == 1
                 file_open = file_open[0]
+                self.filesystem = file_open.fs
+            else:
+                file_open = self.filesystem.open(self.path, mode='rb')
             self._dir = get_dir(self.path)
 
             with file_open as f:
@@ -665,6 +679,7 @@ class YAMLFileCatalog(Catalog):
         for entry in cfg['data_sources']:
             entry._catalog = self
             self._entries[entry.name] = entry
+            entry._filesystem = self.filesystem
 
         self.metadata = cfg.get('metadata', {})
         self.name = self.name or cfg.get('name') or self.name_from_path
diff --git a/intake/container/dataframe.py b/intake/container/dataframe.py
index 8f7b200..eb83a64 100644
--- a/intake/container/dataframe.py
+++ b/intake/container/dataframe.py
@@ -141,7 +141,7 @@ class GenericDataFrame(DataSource):
     def _load_metadata(self):
         import dask.dataframe as dd
         import dask.delayed
-        from dask.bytes import open_files
+        from fsspec import open_files
         self.files = open_files(self.url, **self.storage_options)
 
         def read_a_file(open_file, reader, kwargs):
diff --git a/intake/container/persist.py b/intake/container/persist.py
index 9937efd..6064422 100644
--- a/intake/container/persist.py
+++ b/intake/container/persist.py
@@ -20,7 +20,7 @@ from ..utils import make_path_posix
 def _maybe_add_rm(fs):
     # monkey-path local filesystem
     # this goes away if we can use fsspec's local file-system
-    from dask.bytes.local import LocalFileSystem
+    from fsspec.implementations.local import LocalFileSystem
     if isinstance(fs, LocalFileSystem):
         def rm(path, recursive=False):
             if recursive:
@@ -49,12 +49,12 @@ class PersistStore(YAMLFileCatalog):
 
     def __init__(self, path=None):
         # from fsspec.registry import filesystem
-        from dask.bytes.core import get_fs
+        from fsspec import filesystem
         self.pdir = make_path_posix(path or conf.get('persist_path'))
         path = posixpath.join(self.pdir, 'cat.yaml')
         protocol = (self.pdir.split('://', 1)[0]
                     if "://" in self.pdir else 'file')
-        self.fs = get_fs(protocol)[0]
+        self.fs = filesystem(protocol)
         _maybe_add_rm(self.fs)
         super(PersistStore, self).__init__(path)
 
diff --git a/intake/container/semistructured.py b/intake/container/semistructured.py
index e695ed0..9411174 100644
--- a/intake/container/semistructured.py
+++ b/intake/container/semistructured.py
@@ -73,7 +73,7 @@ class RemoteSequenceSource(RemoteSource):
     def _data_to_source(b, path, encoder=None, **kwargs):
         import dask.bag as db
         import posixpath
-        from dask.bytes import open_files
+        from fsspec import open_files
         import dask
         from intake.source.textfiles import TextFilesSource
         if not hasattr(b, 'to_textfiles'):
diff --git a/intake/gui/catalog/add.py b/intake/gui/catalog/add.py
index 161735c..2ad2cfe 100644
--- a/intake/gui/catalog/add.py
+++ b/intake/gui/catalog/add.py
@@ -5,12 +5,15 @@
 # The full license is in the LICENSE file, distributed with this software.
 #-----------------------------------------------------------------------------
 
+import ast
 import os
 from functools import partial
 
 import intake
 import panel as pn
 
+import fsspec
+from fsspec.registry import known_implementations
 from ..base import Base, MAX_WIDTH, enable_widget, ICONS
 
 
@@ -47,11 +50,17 @@ class FileSelector(Base):
         self.filters = filters
         self.panel = pn.Column(name='Local', width_policy='max', margin=0)
         self.done_callback = done_callback
+        self.fs = fsspec.filesystem("file")
         super().__init__(**kwargs)
 
     def setup(self):
-        self.path_text = pn.widgets.TextInput(value=os.getcwd() + os.path.sep,
+        self.path_text = pn.widgets.TextInput(value=os.getcwd() + '/',
                                               width_policy='max')
+        self.protocol = pn.widgets.Select(options=list(known_implementations),
+                                          value='file', name='protocol')
+        self.storage_options = pn.widgets.TextInput(name='kwargs',
+                                                    value="{}")
+        self.go = pn.widgets.Button(name='⇨')
         self.validator = pn.pane.SVG(None, width=25)
         self.main = pn.widgets.MultiSelect(size=15, width_policy='max')
         self.home = pn.widgets.Button(name='🏠', width=40, height=30)
@@ -60,32 +69,47 @@ class FileSelector(Base):
         self.make_options()
 
         self.watchers = [
-            self.path_text.param.watch(self.validate, ['value']),
-            self.path_text.param.watch(self.make_options, ['value']),
+            self.go.param.watch(self.go_clicked, 'clicks'),
+            self.protocol.param.watch(self.protocol_changed, 'value'),
+            #self.path_text.param.watch(self.validate, ['value']),
+            #self.path_text.param.watch(self.make_options, ['value']),
             self.home.param.watch(self.go_home, 'clicks'),
             self.up.param.watch(self.move_up, 'clicks'),
             self.main.param.watch(self.move_down, ['value'])
         ]
 
         self.children = [
-            pn.Row(self.home, self.up, self.path_text, self.validator, margin=0),
+            pn.Row(self.protocol, self.storage_options),
+            pn.Row(self.home, self.up, self.path_text, self.go, margin=0),
             self.main
         ]
 
+    def protocol_changed(self, *_):
+        self.path_text.value = ""
+        self.main.options = []
+        self.main.value = []
+
+    def go_clicked(self, *_):
+        self.fs = fsspec.filesystem(
+            self.protocol.value, **ast.literal_eval(self.storage_options.value))
+        self.make_options()
+
     @property
     def path(self):
-        path = self.path_text.value
-        return path if path.endswith(os.path.sep) else path + os.path.sep
+        return self.path_text.value
 
     @property
     def url(self):
         """Path to local catalog file"""
-        return os.path.join(self.path, self.main.value[0])
+        return (self.protocol.value + "://" +
+                os.path.join(self.path, self.main.value[0]))
 
     def move_up(self, arg=None):
-        self.path_text.value = os.path.dirname(self.path.rstrip(os.path.sep)) + os.path.sep
+        self.path_text.value = self.fs._parent(self.path_text.value)
+        self.make_options()
 
     def go_home(self, arg=None):
+        self.protocol.value = 'file'
         self.path_text.value = os.getcwd() + os.path.sep
 
     def validate(self, arg=None):
@@ -99,26 +123,32 @@ class FileSelector(Base):
         if self.done_callback:
             self.done_callback(False)
         out = []
-        if os.path.isdir(self.path):
-            for f in sorted(os.listdir(self.path)):
-                if f.startswith('.'):
+        try:
+            for f in self.fs.ls(self.path, True):
+                bn = os.path.basename(f['name'].rstrip('/'))
+                if bn.startswith('.'):
                     continue
-                elif os.path.isdir(os.path.join(self.path, f)):
-                    out.append(f + os.path.sep)
-                elif any(f.endswith(ext) for ext in self.filters):
-                    out.append(f)
+                elif f['type'] == 'directory':
+                    out.append(bn + '/')
+                elif any(bn.endswith(ext) for ext in self.filters):
+                    out.append(bn)
+        except Exception as e:
+            print(e)
 
         self.main.value = []
-        self.main.options = dict(zip(out, out))
+        self.main.options = out
 
     def move_down(self, *events):
         for event in events:
             if event.name == 'value' and len(event.new) > 0:
                 fn = event.new[0]
-                if fn.endswith(os.path.sep):
-                    self.path_text.value = self.path + fn
+                if fn.endswith('/'):
+                    if self.path_text.value:
+                        self.path_text.value = self.path_text.value + '/' + fn
+                    else:
+                        self.path_text.value = fn
                     self.make_options()
-                elif os.path.isfile(self.url) and self.done_callback:
+                elif self.done_callback:
                     self.done_callback(True)
 
     def __getstate__(self):
@@ -136,6 +166,7 @@ class FileSelector(Base):
         self.main.value = state['selected']
         return self
 
+
 class URLSelector(Base):
     """
     Panel interface for inputting a URL to a remote catalog
@@ -238,13 +269,22 @@ class CatAdder(Base):
     @property
     def cat_url(self):
         """URL to remote files or path to local files. Depends on active tab."""
-        return self.selectors[self.tabs.active].url
+        url = self.selectors[self.tabs.active].url
+        if self.selectors[self.tabs.active] is self.fs:
+            fs = self.fs.fs
+        else:
+            fs = None
+        return url, fs
 
     @property
     def cat(self):
         """Catalog object initialized from from cat_url"""
         # might want to do some validation in here
-        return intake.open_catalog(self.cat_url)
+        url, fs = self.cat_url
+        if fs:
+            return intake.open_catalog(url, fs=fs)
+        else:
+            return intake.open_catalog(url)
 
     def add_cat(self, arg=None):
         """Add cat and close panel"""
diff --git a/intake/source/base.py b/intake/source/base.py
index 782408e..f3abbbb 100644
--- a/intake/source/base.py
+++ b/intake/source/base.py
@@ -69,7 +69,7 @@ class DataSource(DictSerialiseMixin):
         for c in self.cache:
             c._cache_dir = make_path_posix(cache_dir)
 
-    def __init__(self, metadata=None):
+    def __init__(self, storage_options=None, metadata=None):
         # default data
         self.metadata = metadata or {}
         if isinstance(self.metadata, dict):
diff --git a/intake/source/cache.py b/intake/source/cache.py
index 3c19cbc..282bf56 100644
--- a/intake/source/cache.py
+++ b/intake/source/cache.py
@@ -17,7 +17,7 @@ import re
 import shutil
 import warnings
 
-from dask.bytes.utils import infer_storage_options
+from fsspec.utils import infer_storage_options
 from intake.config import conf
 from intake.utils import make_path_posix
 
@@ -314,7 +314,7 @@ class FileCache(BaseCache):
     """
 
     def _make_files(self, urlpath, **kwargs):
-        from dask.bytes import open_files
+        from fsspec import open_files
 
         self._ensure_cache_dir()
         subdir = self._hash(urlpath)
@@ -334,7 +334,7 @@ class DirCache(BaseCache):
     """
 
     def _make_files(self, urlpath, **kwargs):
-        from dask.bytes import open_files
+        from fsspec import open_files
 
         self._ensure_cache_dir()
         subdir = self._hash(urlpath)
@@ -380,7 +380,7 @@ class CompressedCache(BaseCache):
     def _make_files(self, urlpath, **kwargs):
         import tempfile
         d = tempfile.mkdtemp()
-        from dask.bytes import open_files
+        from fsspec import open_files
 
         self._ensure_cache_dir()
         self._urlpath = urlpath
@@ -454,7 +454,7 @@ class DATCache(BaseCache):
 
     def _load(self, _, __, urlpath, meta=True):
         import subprocess
-        from dask.bytes import open_files
+        from fsspec import open_files
 
         path = os.path.join(self._cache_dir, self._hash(urlpath))
         dat, part = os.path.split(urlpath)
diff --git a/intake/source/npy.py b/intake/source/npy.py
index 075aed9..7a7d4e7 100644
--- a/intake/source/npy.py
+++ b/intake/source/npy.py
@@ -53,7 +53,7 @@ class NPySource(DataSource):
         super(NPySource, self).__init__(metadata=metadata)
 
     def _get_schema(self):
-        from dask.bytes import open_files
+        from fsspec import open_files
         import dask.array as da
         if self._arr is None:
             path = self._get_cache(self.path)[0]
diff --git a/intake/source/textfiles.py b/intake/source/textfiles.py
index 85b5b46..de55319 100644
--- a/intake/source/textfiles.py
+++ b/intake/source/textfiles.py
@@ -71,7 +71,7 @@ class TextFilesSource(base.DataSource):
         super(TextFilesSource, self).__init__(metadata=metadata)
 
     def _get_schema(self):
-        from dask.bytes import open_files
+        from fsspec import open_files
         if self._files is None:
 
             urlpath = self._get_cache(self._urlpath)[0]
diff --git a/intake/source/utils.py b/intake/source/utils.py
index 801c1b4..254f401 100644
--- a/intake/source/utils.py
+++ b/intake/source/utils.py
@@ -273,10 +273,11 @@ def path_to_pattern(path, metadata=None):
     pattern : str
         Pattern style path stripped of everything to the left of cache regex.
     """
+    from fsspec.core import strip_protocol
     if not isinstance(path, str):
         return
 
-    pattern = path
+    pattern = strip_protocol(path)
     if metadata:
         cache = metadata.get('cache')
         if cache:
diff --git a/requirements.txt b/requirements.txt
index a146aac..c263f4d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,3 +7,4 @@ numpy
 pyyaml
 requests
 msgpack
+fsspec
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
deleted file mode 100755
index e5a7ae7..0000000
--- a/scripts/ci/build.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e # exit on error
-
-echo "Building conda package."
-conda build -c defaults -c conda-forge --no-test ./conda
-
-# If tagged, upload package to main channel, otherwise, run tests
-if [ -n "$TRAVIS_TAG" ]; then
-    echo "Uploading conda package."
-    anaconda -t ${ANACONDA_TOKEN} upload -u intake --force `conda build --output ./conda`
-else
-    echo "Installing conda package locally."
-    conda install -y --use-local intake
-
-    echo "Running unit tests."
-    py.test
-fi
diff --git a/scripts/ci/install.sh b/scripts/ci/install.sh
index 0b26333..480165b 100755
--- a/scripts/ci/install.sh
+++ b/scripts/ci/install.sh
@@ -1,29 +1,15 @@
 #!/bin/bash
 
 set -e # exit on error
-
-CONDA_REQS="conda-build=3.17.0 anaconda-client=1.7.2 conda-verify=3.1.1"
-
-PLATFORM="$(case $TRAVIS_OS_NAME in (linux) echo Linux;; (osx) echo MacOSX;;esac)"
-
-echo "Installing Miniconda."
-MINICONDA_URL="https://repo.continuum.io/miniconda"
-MINICONDA_FILENAME=Miniconda3-latest-${PLATFORM}-x86_64.sh
-curl -L -O "${MINICONDA_URL}/${MINICONDA_FILENAME}"
-bash ${MINICONDA_FILENAME} -b
-
-# if emergency, temporary package pins are necessary, they can go here
-PINNED_PKGS=$(cat <<EOF
-EOF
-)
-mkdir -p $HOME/miniconda3/conda-meta
-echo -e "$PINNED_PKGS" > $HOME/miniconda3/conda-meta/pinned
+wget "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda.sh
+bash miniconda.sh -b
 
 echo "Configuring conda."
 conda config --set auto_update_conda off
-conda install --yes ${CONDA_REQS}
 
 echo "Installing test dependencies."
-conda install --yes `python scripts/deps.py` -c defaults -c conda-forge
-conda install -y -c conda-forge nodejs
+conda install --yes appdirs dask==2 nodejs jinja2 numpy pyyaml requests msgpack-numpy coveralls \
+    pytest fsspec intake-parquet zarr notebook panel==0.5.1 hvplot==0.4.0 -c conda-forge -c defaults
+pip install git+https://github.com/dask/dask --upgrade --no-deps
 npm install -g dat
+pip install -e . --no-deps
diff --git a/scripts/deps.py b/scripts/deps.py
deleted file mode 100644
index 54f2cff..0000000
--- a/scripts/deps.py
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env python
-#-----------------------------------------------------------------------------
-# Copyright (c) 2012 - 2018, Anaconda, Inc. and Intake contributors
-# All rights reserved.
-#
-# The full license is in the LICENSE file, distributed with this software.
-#-----------------------------------------------------------------------------
-
-import os
-import sys
-
-from conda_build.api import render
-
-# This is needed because render spams the console
-old_stdout = sys.stdout
-sys.stdout = open(os.devnull, 'w')
-sys.stderr = open(os.devnull, 'w')
-
-meta = render('conda')[0][0].meta
-
-sys.stdout = old_stdout
-
-test_requires = [l.strip() for l in open('test_requirements.txt')]
-
-print(' '.join(meta['test']['requires'] + test_requires))


Test patch:
diff --git a/intake/catalog/tests/test_local.py b/intake/catalog/tests/test_local.py
index 2767b56..6b3e905 100644
--- a/intake/catalog/tests/test_local.py
+++ b/intake/catalog/tests/test_local.py
@@ -120,7 +120,7 @@ def test_use_source_plugin_from_config(catalog1):
 
 
 def test_get_dir():
-    assert get_dir('s3://path/catalog.yml') == 's3://path'
+    assert get_dir('file://path/catalog.yml') == 'file://path'
     assert get_dir('https://example.com/catalog.yml') == 'https://example.com'
     path = 'example/catalog.yml'
     out = get_dir(path)
@@ -655,3 +655,38 @@ def test_no_instance():
 
     # this would error on instantiation with driver not found
     assert e0 != e1
+
+
+def test_fsspec_integration():
+    import fsspec
+    import pandas as pd
+    mem = fsspec.filesystem('memory')
+    with mem.open('cat.yaml', 'wt') as f:
+        f.write("""
+sources:
+  implicit:
+    driver: csv
+    description: o
+    args:
+      urlpath: "{{CATALOG_DIR}}/file.csv"
+  explicit:
+    driver: csv
+    description: o
+    args:
+      urlpath: "memory:///file.csv"
+  extra:
+    driver: csv
+    description: o
+    args:
+      urlpath: "{{CATALOG_DIR}}/file.csv"
+      storage_options: {other: option}"""
+                )
+    with mem.open('/file.csv', 'wt') as f:
+        f.write("a,b\n0,1")
+    expected = pd.DataFrame({'a': [0], 'b': [1]})
+    cat = open_catalog("memory://cat.yaml")
+    assert list(cat) == ['implicit', 'explicit', 'extra']
+    assert cat.implicit.read().equals(expected)
+    assert cat.explicit.read().equals(expected)
+    s = cat.extra()
+    assert s._storage_options['other']
diff --git a/intake/gui/catalog/tests/test_catalog_add.py b/intake/gui/catalog/tests/test_catalog_add.py
index ae63edf..bb0420d 100644
--- a/intake/gui/catalog/tests/test_catalog_add.py
+++ b/intake/gui/catalog/tests/test_catalog_add.py
@@ -41,23 +41,13 @@ def test_file_selector_raises_error_if_no_file_selected(file_selector):
 
 
 def test_file_selector_edit_path(file_selector):
-    expected = os.getcwd() + '/'
+    expected = os.getcwd()
     file_selector.move_up()
     file_selector.path_text.value = os.getcwd()
     assert file_selector.validator.object is None
     assert file_selector.path == expected
 
 
-def test_file_selector_edit_path_bad_value(file_selector):
-    file_selector.done_callback = callback
-    with pytest.raises(ValueError, match='False'):
-        file_selector.path_text.value = 'blah/foo/blah'
-        assert 'error' in file_selector.validator.object
-        assert file_selector.is_valid is False
-        assert file_selector.main.value == []
-        assert file_selector.main.options == {}
-
-
 def test_file_selector_go_home(file_selector):
     expected = os.getcwd() + '/'
     assert file_selector.path == expected
@@ -69,7 +59,7 @@ def test_file_selector_go_home(file_selector):
 def test_file_selector_move_up(file_selector):
     assert file_selector.path == os.getcwd() + '/'
     file_selector.move_up()
-    expected = os.path.abspath('..') + '/'
+    expected = os.path.abspath('..')
     assert file_selector.path == expected
 
 
@@ -106,7 +96,7 @@ def test_cat_adder(cat_adder):
     assert len(cat_adder.panel.objects) == 2
 
     cat_adder.tabs.active = 1
-    assert cat_adder.cat_url == ''
+    assert cat_adder.cat_url[0] == ''
     assert cat_adder.cat.name is None
     assert cat_adder.widget.disabled is False
 
@@ -118,7 +108,7 @@ def test_cat_adder_add_real_cat(cat_adder, cat1_url, cat1):
     cat_adder.tabs.active = 1
     cat_adder.url.main.value = cat1_url
 
-    assert cat_adder.cat_url == cat1_url
+    assert cat_adder.cat_url[0] == cat1_url
     assert cat_adder.cat == cat1
 
     cat_adder.done_callback = callback
diff --git a/intake/source/tests/test_text.py b/intake/source/tests/test_text.py
index dd4d290..4f97928 100644
--- a/intake/source/tests/test_text.py
+++ b/intake/source/tests/test_text.py
@@ -10,7 +10,7 @@ import pytest
 import intake
 from intake.source.textfiles import TextFilesSource
 from intake.source import import_name
-from dask.bytes import open_files
+from fsspec import open_files
 here = os.path.abspath(os.path.dirname(__file__))
 
 
diff --git a/intake/source/tests/test_utils.py b/intake/source/tests/test_utils.py
index 151a3ca..6abbedd 100644
--- a/intake/source/tests/test_utils.py
+++ b/intake/source/tests/test_utils.py
@@ -105,7 +105,7 @@ def test_roundtrip_reverse_formats(pattern):
 
 
 @pytest.mark.parametrize('path,metadata,expected', [
-    ('s3://data/band{band:1d}.tif',
+    ('http://data/band{band:1d}.tif',
      {'cache': [{'argkey': 'urlpath', 'regex': 'data'}]},
      '/band{band:1d}.tif'),
     ('data/band{band:1d}.tif', {},
diff --git a/intake/tests/test_config.py b/intake/tests/test_config.py
index b4ec2f5..4d00355 100644
--- a/intake/tests/test_config.py
+++ b/intake/tests/test_config.py
@@ -103,5 +103,5 @@ def test_pathdirs():
     assert config.intake_path_dirs(['paths']) == ['paths']
     assert config.intake_path_dirs("") == [""]
     assert config.intake_path_dirs("path1:path2") == ['path1', 'path2']
-    assert config.intake_path_dirs("s3://path1:s3://path2") == [
-        's3://path1', 's3://path2']
+    assert config.intake_path_dirs("memory://path1:memory://path2") == [
+        'memory://path1', 'memory://path2']