Repository: canonical/operator
Pull request #754 (2022-05-13 21:53:15)

Recursively push+pull files/dirs to containers
This adds two new methods to the Container interface - push_path and pull_path.  These methods can be used as follows:

```python
# recursively copy a directory
container.push_path('/nice/local-dir', '/far-away-dest')
# gives /far-away-dest/local-dir

# recursively copy a directory's contents
container.push('/nice/local-dir/*', '/far-away-dest')
# gives /far-away-dest/*

# copy multiple files
container.push_path(['/foo.txt', /bar/baz.txt'], /dst)
# gives /dst/foo.txt, /dst/baz.txt
...
```

`push_path` works with the same semantics.  Globbing is not supported - the `/*` suffix is just a special case to distinguish between copy _x_ vs. copy the _contents of x_ and only works for directories.  Copying occurs incrementally - errors are caught/collected while trying to finish the push/pull.  Raises if there were any errors.

jameinel: it's also a little odd as you might be passing in a tuple. but you don't
want to iterate a string.

On Sun, May 15, 2022, 12:10 Pen Gale ***@***.***> wrote:

> ***@***.**** commented on this pull request.
> ------------------------------
>
> In ops/model.py
> <https://github.com/canonical/operator/pull/754#discussion_r873190936>:
>
> > +        source_paths = source_path
> +        if not isinstance(source_paths, list):
> +            source_paths = [source_paths]
> +        source_paths = [str(p) for p in source_paths]
>
> This is more compact, but less legible. I don't object to either one, but
> I don't think that the golfing is strictly necessary (as fun as it is :-) )
>
> —
> Reply to this email directly, view it on GitHub
> <https://github.com/canonical/operator/pull/754#discussion_r873190936>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AABRQ7NUY2EO7MQPHD4H3ZTVKEOWFANCNFSM5V4SO2TA>
> .
> You are receiving this because you are subscribed to this thread.Message
> ID: ***@***.***>
>

rwcarlsen: @rbarry82 - Yeah - I think I'm going to switch this over to pathlib.

Patch:
diff --git a/ops/model.py b/ops/model.py
index cb949eb..083311c 100644
--- a/ops/model.py
+++ b/ops/model.py
@@ -21,6 +21,7 @@ import math
 import os
 import re
 import shutil
+import stat
 import tempfile
 import time
 import typing
@@ -31,7 +32,9 @@ from subprocess import PIPE, CalledProcessError, run
 from typing import (
     Any,
     BinaryIO,
+    Callable,
     Dict,
+    Generator,
     Iterable,
     List,
     Mapping,
@@ -107,6 +110,8 @@ if typing.TYPE_CHECKING:
     })
 
 
+StrOrPath = typing.Union[str, Path]
+
 logger = logging.getLogger(__name__)
 
 MAX_LOG_LINE_LEN = 131071  # Max length of strings to pass to subshell.
@@ -1209,6 +1214,28 @@ class Storage:
         self._location = Path(location)
 
 
+class MultiPushPullError(Exception):
+    """Aggregates multiple push/pull related exceptions into one."""
+
+    def __init__(self, message: str, errors: List[Tuple[str, Exception]]):
+        """Create an aggregation of several push/pull errors.
+
+        Args:
+            message: error message
+            errors: list of errors with each represented by a tuple (<source_path>,<exception>)
+                where source_path is the path being pushed/pulled from.
+        """
+        self.errors = errors
+        self.message = message
+
+    def __str__(self):
+        return '{} ({} errors): {}, ...'.format(
+            self.message, len(self.errors), self.errors[0][1])
+
+    def __repr__(self):
+        return 'MultiError({!r}, {} errors)'.format(self.message, len(self.errors))
+
+
 class Container:
     """Represents a named container in a unit.
 
@@ -1390,7 +1417,8 @@ class Container:
             raise RuntimeError('expected 1 check, got {}'.format(len(checks)))
         return checks[check_name]
 
-    def pull(self, path: str, *, encoding: str = 'utf-8') -> Union[BinaryIO, TextIO]:
+    def pull(self, path: StrOrPath, *,
+             encoding: Optional[str] = 'utf-8') -> Union[BinaryIO, TextIO]:
         """Read a file's content from the remote system.
 
         Args:
@@ -1403,17 +1431,19 @@ class Container:
             objects decoded according to the specified encoding, or bytes if
             encoding is None.
         """
-        return self._pebble.pull(path, encoding=encoding)
-
-    def push(
-            self, path: str, source: Union[bytes, str, BinaryIO, TextIO], *,
-            encoding: str = 'utf-8',
-            make_dirs: Optional[bool] = False,
-            permissions: Optional[int] = None,
-            user_id: Optional[int] = None,
-            user: Optional[str] = None,
-            group_id: Optional[int] = None,
-            group: Optional[str] = None):
+        return self._pebble.pull(str(path), encoding=encoding)
+
+    def push(self,
+             path: StrOrPath,
+             source: Union[bytes, str, BinaryIO, TextIO],
+             *,
+             encoding: str = 'utf-8',
+             make_dirs: Optional[bool] = False,
+             permissions: Optional[int] = None,
+             user_id: Optional[int] = None,
+             user: Optional[str] = None,
+             group_id: Optional[int] = None,
+             group: Optional[str] = None):
         """Write content to a given file path on the remote system.
 
         Args:
@@ -1433,14 +1463,14 @@ class Container:
             group: Group name for file. Group's GID must match group_id if
                 both are specified.
         """
-        self._pebble.push(path, source, encoding=encoding,
+        self._pebble.push(str(path), source, encoding=encoding,
                           # fixme: remove these ignores on pebble.exec signature fix
                           make_dirs=make_dirs,  # type: ignore
                           permissions=permissions,   # type: ignore
                           user_id=user_id, user=user,  # type: ignore
                           group_id=group_id, group=group)   # type: ignore
 
-    def list_files(self, path: str, *, pattern: Optional[str] = None,
+    def list_files(self, path: StrOrPath, *, pattern: Optional[str] = None,
                    itself: bool = False) -> List['pebble.FileInfo']:
         """Return list of directory entries from given path on remote system.
 
@@ -1455,10 +1485,240 @@ class Container:
             itself: If path refers to a directory, return information about the
                 directory itself, rather than its contents.
         """
-        return self._pebble.list_files(path,
+        return self._pebble.list_files(str(path),
                                        # fixme: remove on pebble.exec signature fix
                                        pattern=pattern, itself=itself)  # type: ignore
 
+    def push_path(self,
+                  source_path: Union[StrOrPath, Iterable[StrOrPath]],
+                  dest_dir: StrOrPath):
+        """Recursively push a local path or files to the remote system.
+
+        Only regular files and directories are copied; symbolic links, device files, etc. are
+        skipped.  Pushing is attempted to completion even if errors occur during the process.  All
+        errors are collected incrementally. After copying has completed, if any errors occurred, a
+        single MultiPushPullError is raised containing details for each error.
+
+        Assuming the following files exist locally:
+
+            * /foo/bar/baz.txt
+            * /foo/foobar.txt
+            * /quux.txt
+
+        You could push the following ways::
+
+            # copy one file
+            container.push_path('/foo/foobar.txt', '/dst')
+            # Destination results: /dst/foobar.txt
+
+            # copy a directory
+            container.push_path('/foo', '/dst')
+            # Destination results: /dst/foo/bar/baz.txt, /dst/foo/foobar.txt
+
+            # copy a directory's contents
+            container.push_path('/foo/*', '/dst')
+            # Destination results: /dst/bar/baz.txt, /dst/foobar.txt
+
+            # copy multiple files
+            container.push_path(['/foo/bar/baz.txt', 'quux.txt'], '/dst')
+            # Destination results: /dst/baz.txt, /dst/quux.txt
+
+            # copy a file and a directory
+            container.push_path(['/foo/bar', '/quux.txt'], '/dst')
+            # Destination results: /dst/bar/baz.txt, /dst/quux.txt
+
+        Args:
+            source_path: A single path or list of paths to push to the remote system.  The
+                paths can be either a file or a directory.  If source_path is a directory, the
+                directory base name is attached to the destination directory - i.e. the source path
+                directory is placed inside the destination directory.  If a source path ends with a
+                trailing "/*" it will have its *contents* placed inside the destination directory.
+            dest_dir: Remote destination directory inside which the source dir/files will be
+                placed.  This must be an absolute path.
+        """
+        if os.name == 'nt':
+            raise RuntimeError('Container.push_path is not supported on Windows-based systems')
+
+        if hasattr(source_path, '__iter__') and not isinstance(source_path, str):
+            source_paths = typing.cast(Iterable[StrOrPath], source_path)
+        else:
+            source_paths = typing.cast(Iterable[StrOrPath], [source_path])
+        source_paths = [Path(p) for p in source_paths]
+        dest_dir = Path(dest_dir)
+
+        def local_list(source_path: Path) -> List[pebble.FileInfo]:
+            paths = source_path.iterdir() if source_path.is_dir() else [source_path]
+            files = [self._build_fileinfo(source_path / f) for f in paths]
+            return files
+
+        errors = []  # type: List[Tuple[str, Exception]]
+        for source_path in source_paths:
+            try:
+                for info in Container._list_recursive(local_list, source_path):
+                    dstpath = self._build_destpath(info.path, source_path, dest_dir)
+                    with open(info.path) as src:
+                        self.push(
+                            dstpath,
+                            src,
+                            make_dirs=True,
+                            permissions=info.permissions,
+                            user_id=info.user_id,
+                            user=info.user,
+                            group_id=info.group_id,
+                            group=info.group)
+            except (OSError, pebble.Error) as err:
+                errors.append((str(source_path), err))
+        if errors:
+            raise MultiPushPullError('failed to push one or more files', errors)
+
+    def pull_path(self,
+                  source_path: Union[StrOrPath, Iterable[StrOrPath]],
+                  dest_dir: StrOrPath):
+        """Recursively pull a remote path or files to the local system.
+
+        Only regular files and directories are copied; symbolic links, device files, etc. are
+        skipped.  Pulling is attempted to completion even if errors occur during the process.  All
+        errors are collected incrementally. After copying has completed, if any errors occurred, a
+        single MultiPushPullError is raised containing details for each error.
+
+        Assuming the following files exist remotely:
+
+            * /foo/bar/baz.txt
+            * /foo/foobar.txt
+            * /quux.txt
+
+        You could pull the following ways::
+
+            # copy one file
+            container.pull_path('/foo/foobar.txt', '/dst')
+            # Destination results: /dst/foobar.txt
+
+            # copy a directory
+            container.pull_path('/foo', '/dst')
+            # Destination results: /dst/foo/bar/baz.txt, /dst/foo/foobar.txt
+
+            # copy a directory's contents
+            container.pull_path('/foo/*', '/dst')
+            # Destination results: /dst/bar/baz.txt, /dst/foobar.txt
+
+            # copy multiple files
+            container.pull_path(['/foo/bar/baz.txt', 'quux.txt'], '/dst')
+            # Destination results: /dst/baz.txt, /dst/quux.txt
+
+            # copy a file and a directory
+            container.pull_path(['/foo/bar', '/quux.txt'], '/dst')
+            # Destination results: /dst/bar/baz.txt, /dst/quux.txt
+
+        Args:
+            source_path: A single path or list of paths to pull from the remote system.  The
+                paths can be either a file or a directory but must be absolute paths.  If
+                source_path is a directory, the directory base name is attached to the destination
+                directory - i.e.  the source path directory is placed inside the destination
+                directory.  If a source path ends with a trailing "/*" it will have its *contents*
+                placed inside the destination directory.
+            dest_dir: Local destination directory inside which the source dir/files will be
+                placed.
+        """
+        if os.name == 'nt':
+            raise RuntimeError('Container.pull_path is not supported on Windows-based systems')
+
+        if hasattr(source_path, '__iter__') and not isinstance(source_path, str):
+            source_paths = typing.cast(Iterable[StrOrPath], source_path)
+        else:
+            source_paths = typing.cast(Iterable[StrOrPath], [source_path])
+        source_paths = [Path(p) for p in source_paths]
+        dest_dir = Path(dest_dir)
+
+        errors = []  # type: List[Tuple[str, Exception]]
+        for source_path in source_paths:
+            try:
+                for info in Container._list_recursive(self.list_files, source_path):
+                    dstpath = self._build_destpath(info.path, source_path, dest_dir)
+                    dstpath.parent.mkdir(parents=True, exist_ok=True)
+                    with self.pull(info.path, encoding=None) as src:
+                        with dstpath.open(mode='wb') as dst:
+                            shutil.copyfileobj(typing.cast(BinaryIO, src), dst)
+            except (OSError, pebble.Error) as err:
+                errors.append((str(source_path), err))
+        if errors:
+            raise MultiPushPullError('failed to pull one or more files', errors)
+
+    @staticmethod
+    def _build_fileinfo(path: StrOrPath) -> 'pebble.FileInfo':
+        """Constructs a FileInfo object by stat'ing a local path."""
+        path = Path(path)
+        if path.is_symlink():
+            ftype = pebble.FileType.SYMLINK
+        elif path.is_dir():
+            ftype = pebble.FileType.DIRECTORY
+        elif path.is_file():
+            ftype = pebble.FileType.FILE
+        else:
+            ftype = pebble.FileType.UNKNOWN
+
+        import grp
+        import pwd
+        info = path.lstat()
+        return pebble.FileInfo(
+            path=str(path),
+            name=path.name,
+            type=ftype,
+            size=info.st_size,
+            permissions=stat.S_IMODE(info.st_mode),  # type: ignore
+            last_modified=datetime.datetime.fromtimestamp(info.st_mtime),
+            user_id=info.st_uid,
+            user=pwd.getpwuid(info.st_uid).pw_name,
+            group_id=info.st_gid,
+            group=grp.getgrgid(info.st_gid).gr_name)
+
+    @staticmethod
+    def _list_recursive(list_func: Callable[[Path],
+                        Iterable['pebble.FileInfo']],
+                        path: Path) -> Generator['pebble.FileInfo', None, None]:
+        """Recursively lists all files under path using the given list_func.
+
+        Args:
+            list_func: Function taking 1 Path argument that returns a list of FileInfo objects
+                representing files residing directly inside the given path.
+            path: Filepath to recursively list.
+        """
+        if path.name == '*':
+            # ignore trailing '/*' that we just use for determining how to build paths
+            # at destination
+            path = path.parent
+
+        for info in list_func(path):
+            if info.type is pebble.FileType.DIRECTORY:
+                yield from Container._list_recursive(list_func, Path(info.path))
+            elif info.type in (pebble.FileType.FILE, pebble.FileType.SYMLINK):
+                yield info
+            else:
+                logger.debug(
+                    'skipped unsupported file in Container.[push/pull]_path: %s', info.path)
+
+    @staticmethod
+    def _build_destpath(file_path: StrOrPath, source_path: StrOrPath, dest_dir: StrOrPath) -> Path:
+        """Converts a source file and destination dir into a full destination filepath.
+
+        file_path:
+            Full source-side path for the file being copied to dest_dir.
+        source_path
+            Source prefix under which the given file_path was found.
+        dest_dir
+            Destination directory to place file_path into.
+        """
+        # select between the two following src+dst combos via trailing '/*'
+        # /src/* --> /dst/*
+        # /src --> /dst/src
+        file_path, source_path, dest_dir = Path(file_path), Path(source_path), Path(dest_dir)
+        prefix = str(source_path.parent)
+        if os.path.commonprefix([prefix, str(file_path)]) != prefix:
+            raise RuntimeError(
+                'file "{}" does not have specified prefix "{}"'.format(
+                    file_path, prefix))
+        path_suffix = os.path.relpath(str(file_path), prefix)
+        return dest_dir / path_suffix
+
     def exists(self, path: str) -> bool:
         """Return true if the path exists on the container filesystem."""
         try:
diff --git a/ops/pebble.py b/ops/pebble.py
index 9db8cbb..f1fb06b 100644
--- a/ops/pebble.py
+++ b/ops/pebble.py
@@ -1627,8 +1627,11 @@ class Client:
         resp = self._request('GET', '/v1/services', query)
         return [ServiceInfo.from_dict(info) for info in resp['result']]
 
-    def pull(self, path: str, *, encoding: str = 'utf-8') -> typing.Union[typing.BinaryIO,
-                                                                          typing.TextIO]:
+    def pull(self,
+             path: str,
+             *,
+             encoding: typing.Optional[str] = 'utf-8') -> typing.Union[typing.BinaryIO,
+                                                                       typing.TextIO]:
         """Read a file's content from the remote system.
 
         Args:


Test patch:
diff --git a/test/test_model.py b/test/test_model.py
index 1b79bd8..015d267 100755
--- a/test/test_model.py
+++ b/test/test_model.py
@@ -13,21 +13,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
 import ipaddress
 import json
 import os
 import pathlib
+import tempfile
 import unittest
 from collections import OrderedDict
 from test.test_helpers import fake_script, fake_script_calls
 from textwrap import dedent
 
+import pytest
+
 import ops.charm
 import ops.model
 import ops.testing
 from ops._private import yaml
 from ops.charm import RelationMeta, RelationRole
-from ops.pebble import APIError, ServiceInfo
+from ops.pebble import APIError, FileInfo, FileType, ServiceInfo
 
 
 class TestModel(unittest.TestCase):
@@ -755,6 +759,244 @@ class TestModel(unittest.TestCase):
         self.assertEqual(expected, self.harness._get_backend_calls(reset=reset))
 
 
+class PushPullCase:
+    """Test case for table-driven tests."""
+
+    def __init__(self, **vals):
+        self.pattern = None
+        self.dst = None
+        self.errors = []
+        self.want = set()
+        for key, val in vals.items():
+            setattr(self, key, val)
+
+
+recursive_list_cases = [
+    PushPullCase(
+        name='basic recursive list',
+        path='/',
+        files=['/foo/bar.txt', '/baz.txt'],
+        want={'/foo/bar.txt', '/baz.txt'},
+    ),
+    PushPullCase(
+        name='basic recursive list reverse',
+        path='/',
+        files=['/baz.txt', '/foo/bar.txt'],
+        want={'/foo/bar.txt', '/baz.txt'},
+    ),
+    PushPullCase(
+        name='directly list a (non-directory) file',
+        path='/baz.txt',
+        files=['/baz.txt'],
+        want={'/baz.txt'},
+    ),
+]
+
+
+@pytest.mark.parametrize('case', recursive_list_cases)
+def test_recursive_list(case):
+    def list_func_gen(file_list):
+        args = {
+            'last_modified': datetime.time(),
+            'permissions': 0o777,
+            'size': 42,
+            'user_id': 0,
+            'user': 'foo',
+            'group_id': 1024,
+            'group': 'bar',
+        }
+        file_infos, dirs = [], set()
+        for f in file_list:
+            file_infos.append(
+                FileInfo(
+                    path=f,
+                    name=os.path.basename(f),
+                    type=FileType.FILE,
+                    **args))
+
+            # collect all the directories for the test case's files
+            dirpath = os.path.dirname(f)
+            if dirpath != '' and dirpath not in dirs:
+                dirs.update(dirpath)
+                file_infos.append(
+                    FileInfo(
+                        path=dirpath,
+                        name=os.path.basename(dirpath),
+                        type=FileType.DIRECTORY,
+                        **args))
+
+        def inner(path):
+            path = str(path)
+            matches = []
+            for info in file_infos:
+                # exclude file infos for separate trees and also
+                # for the directory we are listing itself - we only want its contents.
+                if not info.path.startswith(path) or (
+                        info.type == FileType.DIRECTORY and path == info.path):
+                    continue
+                # exclude file infos for files that are in subdirectories of path.
+                # we only want files that are directly in path.
+                if info.path[len(path):].find('/') > 0:
+                    continue
+                matches.append(info)
+            return matches
+        return inner
+
+    # test raw business logic for recursion and dest path construction
+    files = set()
+    case.path = os.path.normpath(case.path)
+    case.files = [os.path.normpath(f) for f in case.files]
+    case.want = {os.path.normpath(f) for f in case.want}
+    for f in ops.model.Container._list_recursive(
+        list_func_gen(
+            case.files), pathlib.Path(
+            case.path)):
+        path = f.path
+        if case.dst is not None:
+            # test destination path construction
+            _, path = f.path, ops.model.Container._build_destpath(
+                f.path, case.path, case.dst)
+        files.add(path)
+    assert case.want == files, 'case {!r} has wrong files: want {}, got {}'.format(
+        case.name, case.want, files)
+
+
+recursive_push_pull_cases = [
+    PushPullCase(
+        name='basic push/pull',
+        path='/foo',
+        dst='/baz',
+        files=['/foo/bar.txt'],
+        want={'/baz/foo/bar.txt'},
+    ),
+    PushPullCase(
+        name='push/pull - trailing slash',
+        path='/foo/',
+        dst='/baz',
+        files=['/foo/bar.txt'],
+        want={'/baz/foo/bar.txt'},
+    ),
+    PushPullCase(
+        name='basic push/pull - root',
+        path='/',
+        dst='/baz',
+        files=['/foo/bar.txt'],
+        want={'/baz/foo/bar.txt'},
+    ),
+    PushPullCase(
+        name='basic push/pull - multicomponent path',
+        path='/foo/bar',
+        dst='/baz',
+        files=['/foo/bar/baz.txt'],
+        want={'/baz/bar/baz.txt'},
+    ),
+    PushPullCase(
+        name='push/pull contents',
+        path='/foo/*',
+        dst='/baz',
+        files=['/foo/bar.txt'],
+        want={'/baz/bar.txt'},
+    ),
+    PushPullCase(
+        name='directly push/pull a specific file',
+        path='/foo/bar.txt',
+        dst='/baz',
+        files=['/foo/bar.txt'],
+        want={'/baz/bar.txt'},
+    ),
+    PushPullCase(
+        name='error on push/pull non-existing file',
+        path='/foo/bar.txt',
+        dst='/baz',
+        files=[],
+        errors={'/foo/bar.txt'},
+    ),
+    PushPullCase(
+        name='push/pull multiple non-existing files',
+        path=['/foo/bar.txt', '/boo/far.txt'],
+        dst='/baz',
+        files=[],
+        errors={'/foo/bar.txt', '/boo/far.txt'},
+    ),
+    PushPullCase(
+        name='push/pull file and dir combo',
+        path=['/foo/foobar.txt', '/foo/bar'],
+        dst='/baz',
+        files=['/foo/bar/baz.txt', '/foo/foobar.txt', '/quux.txt'],
+        want={'/baz/foobar.txt', '/baz/bar/baz.txt'},
+    ),
+]
+
+
+@unittest.skipIf(os.name == 'nt', "These pebble ops are not supported on windows")
+@pytest.mark.parametrize('case', recursive_push_pull_cases)
+def test_recursive_push_and_pull(case):
+    # full "integration" test of push+pull
+    harness = ops.testing.Harness(ops.charm.CharmBase, meta='''
+        name: test-app
+        containers:
+          foo:
+            resource: foo-image
+        ''')
+    harness.begin()
+    c = harness.model.unit.containers['foo']
+
+    # create push test case filesystem structure
+    push_src = tempfile.TemporaryDirectory()
+    for file in case.files:
+        fpath = os.path.join(push_src.name, file[1:])
+        os.makedirs(os.path.dirname(fpath), exist_ok=True)
+        with open(fpath, 'w') as f:
+            f.write('hello')
+
+    # test push
+    if isinstance(case.path, list):
+        # swap slash for dummy dir on root dir so Path.parent doesn't return tmpdir path component
+        # otherwise remove leading slash so we can do the path join properly.
+        push_path = [os.path.join(push_src.name, p[1:] if len(p) > 1 else 'foo')
+                     for p in case.path]
+    else:
+        # swap slash for dummy dir on root dir so Path.parent doesn't return tmpdir path component
+        # otherwise remove leading slash so we can do the path join properly.
+        push_path = os.path.join(push_src.name, case.path[1:] if len(case.path) > 1 else 'foo')
+
+    errors = []
+    try:
+        print(push_path, case.dst)
+        c.push_path(push_path, case.dst)
+    except ops.model.MultiPushPullError as err:
+        if not case.errors:
+            raise
+        errors = {src[len(push_src.name):] for src, _ in err.errors}
+
+    assert case.errors == errors, \
+        'push_path gave wrong expected errors: want {}, got {}'.format(case.errors, errors)
+    for fpath in case.want:
+
+        for f in ops.model.Container._list_recursive(c.list_files, pathlib.Path('/')):
+            print(f)
+        assert c.exists(fpath), 'push_path failed: file {} missing at destination'.format(fpath)
+
+    # create pull test case filesystem structure
+    pull_dst = tempfile.TemporaryDirectory()
+    for fpath in case.files:
+        c.push(fpath, 'hello', make_dirs=True)
+
+    # test pull
+    errors = []
+    try:
+        c.pull_path(case.path, os.path.join(pull_dst.name, case.dst[1:]))
+    except ops.model.MultiPushPullError as err:
+        if not case.errors:
+            raise
+        errors = {src for src, _ in err.errors}
+
+    assert case.errors == errors, \
+        'pull_path gave wrong expected errors: want {}, got {}'.format(case.errors, errors)
+    for fpath in case.want:
+        assert c.exists(fpath), 'pull_path failed: file {} missing at destination'.format(fpath)
+
+
 class TestApplication(unittest.TestCase):
 
     def setUp(self):