Repository: canonical/snapcraft

Issue:
catkin plugin: recursively merge rosinstall files
Estimate: 4 days

The Catkin plugin currently supports parsing rosinstall files to pull the various repositories that make up the workspace that it will build. A request from Shadow Robotics is to add the ability to then look inside the pulled workspace for more rosinstall files, and fetch their repositories as well. This is not supported by rosinstall alone. This will require us to have a concept of being finished. How do we know that we’ve finished pulling everything necessary if we’re doing so in a recursive manner?
catkin plugin: recursively merge rosinstall files
Estimate: 4 days

The Catkin plugin currently supports parsing rosinstall files to pull the various repositories that make up the workspace that it will build. A request from Shadow Robotics is to add the ability to then look inside the pulled workspace for more rosinstall files, and fetch their repositories as well. This is not supported by rosinstall alone. This will require us to have a concept of being finished. How do we know that we’ve finished pulling everything necessary if we’re doing so in a recursive manner?


Discussion:

Resolving patch:
diff --git a/snapcraft/plugins/_ros/wstool.py b/snapcraft/plugins/_ros/wstool.py
--- a/snapcraft/plugins/_ros/wstool.py
+++ b/snapcraft/plugins/_ros/wstool.py
@@ -55,9 +55,22 @@ def __init__(self, message: str) -> None:
 
 
 class Wstool:
+    """This class serves as a Python wrapper for the CLI utility wstool."""
+
     def __init__(self, ros_package_path: str, wstool_path: str,
                  ubuntu_sources: str,
                  project: snapcraft.ProjectOptions) -> None:
+        """Create new Wstool
+
+        :param str ros_package_path: The path where the packages should be
+                                     fetched.
+        :param str wstool_path: Working directory for wstool (where it will be
+                                installed).
+        :param str ubuntu_sources: Ubuntu repositories from which wstool will
+                                   be installed.
+        :param project: Instance of ProjectOptions for project-wide settings.
+        :type project: snapcraft.ProjectOptions
+        """
         self._ros_package_path = ros_package_path
         self._ubuntu_sources = ubuntu_sources
         self._wstool_path = wstool_path
@@ -65,6 +78,8 @@ def __init__(self, ros_package_path: str, wstool_path: str,
         self._project = project
 
     def setup(self) -> None:
+        """Download/install wstool, and initialize workspace."""
+
         os.makedirs(self._wstool_install_path, exist_ok=True)
 
         # wstool isn't a dependency of the project, so we'll unpack it
@@ -89,7 +104,15 @@ def setup(self) -> None:
                 raise WorkspaceInitializationError(stderr)
 
     def merge(self, rosinstall_file: str) -> str:
+        """Merge rosinstall file with existing rosinstall in workspace.
+
+        :param str rosinstall_file: Path to rosinstall file to merge in.
+        """
+
         try:
+            # Summary of arguments used:
+            # --confirm-all: Don't ask for confirmation
+            # -t: Path to workspace
             return self._run(
                 ['merge', rosinstall_file, '--confirm-all', '-t{}'.format(
                     self._ros_package_path)]).strip()
@@ -98,7 +121,15 @@ def merge(self, rosinstall_file: str) -> str:
             raise RosinstallMergeError(rosinstall_file, stderr)
 
     def update(self) -> str:
+        """Update workspace from rosinstall file.
+
+        This actually hits the network and downloads all repositories in
+        the workspace's rosinstall file.
+        """
         try:
+            # Summary of arguments used:
+            # -j: Use multiple jobs to fetch
+            # -t: Path to workspace (into which packages will be fetched)
             return self._run(
                 ['update', '-j{}'.format(self._project.parallel_build_count),
                  '-t{}'.format(self._ros_package_path)])
diff --git a/snapcraft/plugins/catkin.py b/snapcraft/plugins/catkin.py
--- a/snapcraft/plugins/catkin.py
+++ b/snapcraft/plugins/catkin.py
@@ -38,6 +38,11 @@
       (list of strings)
       List of rosinstall files to merge while pulling. Paths are relative to
       the source.
+    - recursive-rosinstall:
+      (boolean)
+      Whether or not to recursively merge/update rosinstall files from fetched
+      sources. Will continue until all rosinstall files have been merged.
+      Defaults to false.
     - catkin-cmake-args:
       (list of strings)
       Configure flags to pass onto the cmake invocation from catkin.
@@ -177,6 +182,11 @@ def schema(cls):
             'default': [],
         }
 
+        schema['properties']['recursive-rosinstall'] = {
+            'type': 'boolean',
+            'default': False,
+        }
+
         schema['properties']['catkin-cmake-args'] = {
             'type': 'array',
             'minitems': 1,
@@ -198,7 +208,8 @@ def get_pull_properties(cls):
         # Inform Snapcraft of the properties associated with pulling. If these
         # change in the YAML Snapcraft will consider the pull step dirty.
         return ['rosdistro', 'catkin-packages', 'source-space',
-                'include-roscore', 'underlay', 'rosinstall-files']
+                'include-roscore', 'underlay', 'rosinstall-files',
+                'recursive-rosinstall']
 
     @classmethod
     def get_build_properties(cls):
@@ -362,7 +373,7 @@ def pull(self):
         # There may be nothing contained within the source but a rosinstall
         # file. We need to use it to flesh out the workspace before continuing
         # with the pull.
-        if self.options.rosinstall_files:
+        if self.options.rosinstall_files or self.options.recursive_rosinstall:
             wstool = _ros.wstool.Wstool(
                 self._ros_package_path, self._wstool_path,
                 self.PLUGIN_STAGE_SOURCES, self.project)
@@ -372,8 +383,23 @@ def pull(self):
             if self.options.source_subdir:
                 source_path = os.path.join(self.sourcedir,
                                            self.options.source_subdir)
-            _handle_rosinstall_files(
-                wstool,  source_path, self.options.rosinstall_files)
+
+            # Recursively handling rosinstall files is a superset of handling
+            # individual rosinstall files. If both are specified, the recursive
+            # option will cover it.
+            if self.options.recursive_rosinstall:
+                _recursively_handle_rosinstall_files(wstool, source_path)
+            else:
+                # The rosinstall files in the YAML are relative to the part's
+                # source. However, _handle_rosinstall_files requires absolute
+                # paths.
+                rosinstall_files = set()
+                for rosinstall_file in self.options.rosinstall_files:
+                    rosinstall_files.add(
+                        os.path.join(source_path, rosinstall_file))
+
+                _handle_rosinstall_files(
+                    wstool, rosinstall_files)
 
         # Make sure the package path exists before continuing. We only care
         # about doing this if there are actually packages to build, which is
@@ -836,17 +862,42 @@ def _dependency_is_in_underlay(catkin, dependency):
     return False
 
 
-def _handle_rosinstall_files(wstool, source_path, rosinstall_files):
+def _handle_rosinstall_files(wstool, rosinstall_files):
     """Merge given rosinstall files into our workspace."""
 
     for rosinstall_file in rosinstall_files:
         logger.info('Merging {}'.format(rosinstall_file))
-        wstool.merge(os.path.join(source_path, rosinstall_file))
+        wstool.merge(rosinstall_file)
 
     logger.info('Updating workspace...')
     wstool.update()
 
 
+def _recursively_handle_rosinstall_files(wstool, source_path, *,
+                                         cache=None):
+    "Recursively find and merge rosinstall files and update workspace"
+
+    rosinstall_files = set()  # type: Set[str]
+    if not cache:
+        cache = set()  # type: Set[str]
+
+    # Walk the entire source directory looking for rosinstall files. Keep track
+    # of any we haven't seen previously.
+    for root, directories, files in os.walk(source_path):
+        for file_name in files:
+            path = os.path.join(root, file_name)
+            if path.endswith('.rosinstall') and path not in cache:
+                rosinstall_files.add(path)
+
+    # If we came across previously-unseen rosinstall files, add them to the
+    # cache. Then handle them (merge/update). Finally, walk again. Do this
+    # until no new rosinstall files are discovered.
+    if rosinstall_files:
+        cache.update(rosinstall_files)
+        _handle_rosinstall_files(wstool, rosinstall_files)
+        _recursively_handle_rosinstall_files(wstool, source_path, cache=cache)
+
+
 class CatkinPackageNotFoundError(errors.SnapcraftError):
     fmt = 'Unable to find Catkin package {package_name!r}'
 


Test patch:
diff --git a/tests/integration/plugins_catkin/test_catkin.py b/tests/integration/plugins_catkin/test_catkin.py
--- a/tests/integration/plugins_catkin/test_catkin.py
+++ b/tests/integration/plugins_catkin/test_catkin.py
@@ -18,6 +18,11 @@
 import subprocess
 import tempfile
 
+from testtools.matchers import (
+    FileContains,
+    FileExists,
+)
+
 from tests import (
     fixture_setup,
     integration,
@@ -61,3 +66,30 @@ def test_catkin_part_builds_after_python_part(self):
 
         # This snap should be staged with no errors
         self.run_snapcraft('stage')
+
+    @skip.skip_unless_codename('xenial', 'ROS Kinetic only targets Xenial')
+    def test_catkin_recursively_parses_rosinstall_files(self):
+        self.copy_project_to_cwd('catkin-recursive-rosinstall')
+
+        # Unpack the git repos for the test
+        subprocess.check_call(['tar', 'xf', 'repos.tar'])
+
+        # Pull should have no errors
+        self.run_snapcraft('pull')
+
+        repo1_path = os.path.join(
+            self.parts_dir, 'catkin-part', 'src', 'src', 'repo1')
+        repo2_path = os.path.join(
+            self.parts_dir, 'catkin-part', 'src', 'src', 'repo2')
+        repo3_path = os.path.join(
+            self.parts_dir, 'catkin-part', 'src', 'src', 'repo3')
+
+        self.assertThat(
+            os.path.join(repo1_path, 'repo1.rosinstall'), FileExists())
+        self.assertThat(os.path.join(repo1_path, 'file1'), FileContains('1\n'))
+
+        self.assertThat(
+            os.path.join(repo2_path, 'repo2.rosinstall'), FileExists())
+        self.assertThat(os.path.join(repo2_path, 'file2'), FileContains('2\n'))
+
+        self.assertThat(os.path.join(repo3_path, 'file3'), FileContains('3\n'))
diff --git a/tests/integration/snaps/catkin-recursive-rosinstall/repos.tar b/tests/integration/snaps/catkin-recursive-rosinstall/repos.tar
new file mode 100644
Binary files /dev/null and b/tests/integration/snaps/catkin-recursive-rosinstall/repos.tar differ
diff --git a/tests/integration/snaps/catkin-recursive-rosinstall/root.rosinstall b/tests/integration/snaps/catkin-recursive-rosinstall/root.rosinstall
new file mode 100644
--- /dev/null
+++ b/tests/integration/snaps/catkin-recursive-rosinstall/root.rosinstall
@@ -0,0 +1,3 @@
+- git:
+    local-name: repo1
+    uri: ../repos/repo1
diff --git a/tests/integration/snaps/catkin-recursive-rosinstall/snap/snapcraft.yaml b/tests/integration/snaps/catkin-recursive-rosinstall/snap/snapcraft.yaml
new file mode 100644
--- /dev/null
+++ b/tests/integration/snaps/catkin-recursive-rosinstall/snap/snapcraft.yaml
@@ -0,0 +1,15 @@
+name: catkin-recursive-rosinstall
+version: '0.1'
+summary: Ensure rosinstall files can be merged recursively
+description: |
+  This example won't build. It's only meant for pulling.
+
+grade: devel
+confinement: strict
+
+parts:
+  catkin-part:
+    plugin: catkin
+    rosdistro: kinetic
+    rosinstall-files: [root.rosinstall]
+    recursive-rosinstall: true
diff --git a/tests/unit/plugins/test_catkin.py b/tests/unit/plugins/test_catkin.py
--- a/tests/unit/plugins/test_catkin.py
+++ b/tests/unit/plugins/test_catkin.py
@@ -79,6 +79,7 @@ class props:
             catkin_cmake_args = []
             underlay = None
             rosinstall_files = None
+            recursive_rosinstall = False
             build_attributes = []
             catkin_ros_master_uri = 'http://localhost:11311'
 
@@ -149,7 +150,8 @@ def test_schema(self):
         properties = schema['properties']
         expected = ('rosdistro', 'catkin-packages', 'source-space',
                     'include-roscore', 'catkin-cmake-args', 'underlay',
-                    'rosinstall-files', 'catkin-ros-master-uri')
+                    'rosinstall-files', 'recursive-rosinstall',
+                    'catkin-ros-master-uri')
         self.assertThat(properties, HasLength(len(expected)))
         for prop in expected:
             self.assertThat(properties, Contains(prop))
@@ -265,6 +267,18 @@ def test_schema_rosinstall_files(self):
         self.assertThat(rosinstall_files['items'], Contains('type'))
         self.assertThat(rosinstall_files['items']['type'], Equals('string'))
 
+    def test_schema_recursive_rosinstall(self):
+        schema = catkin.CatkinPlugin.schema()
+
+        # Check recursive-rosinstall property
+        recursive_rosinstall = schema['properties']['recursive-rosinstall']
+        expected = ('type', 'default')
+        self.assertThat(recursive_rosinstall, HasLength(len(expected)))
+        for prop in expected:
+            self.assertThat(recursive_rosinstall, Contains(prop))
+        self.assertThat(recursive_rosinstall['type'], Equals('boolean'))
+        self.assertThat(recursive_rosinstall['default'], Equals(False))
+
     def test_schema_catkin_ros_master_uri(self):
         schema = catkin.CatkinPlugin.schema()
 
@@ -281,7 +295,8 @@ def test_schema_catkin_ros_master_uri(self):
     def test_get_pull_properties(self):
         expected_pull_properties = ['rosdistro', 'catkin-packages',
                                     'source-space', 'include-roscore',
-                                    'underlay', 'rosinstall-files']
+                                    'underlay', 'rosinstall-files',
+                                    'recursive-rosinstall']
         actual_pull_properties = catkin.CatkinPlugin.get_pull_properties()
 
         self.assertThat(actual_pull_properties,
@@ -1548,14 +1563,20 @@ def setUp(self):
         self.wstool_mock = mock.MagicMock()
 
     def test_single_rosinstall_file(self):
+        rosinstall_file = os.path.join('source_path', 'rosinstall_file')
         catkin._handle_rosinstall_files(
-            self.wstool_mock, 'source_path', ['rosinstall_file'])
+            self.wstool_mock, [rosinstall_file])
         self.wstool_mock.merge.assert_called_once_with(
             os.path.join('source_path', 'rosinstall_file'))
 
     def test_multiple_rosinstall_files(self):
+        rosinstall_files = [
+            os.path.join('source_path', 'file1'),
+            os.path.join('source_path', 'file2'),
+        ]
+
         catkin._handle_rosinstall_files(
-            self.wstool_mock, 'source_path', ['file1', 'file2'])
+            self.wstool_mock, rosinstall_files)
 
         # The order matters here. It should be the same as how they were passed
         self.wstool_mock.merge.assert_has_calls([
@@ -1564,6 +1585,40 @@ def test_multiple_rosinstall_files(self):
         ])
 
 
+class RecursivelyHandleRosinstallFilesTestCase(unit.TestCase):
+
+    def setUp(self):
+        super().setUp()
+
+        self.wstool_mock = mock.MagicMock()
+
+    def test_recursive_rosinstall(self):
+        counter = 0
+
+        # A fake update that plops a new rosinstall file down every time
+        # it's called (up to two times).
+        def _fake_wstool_update():
+            nonlocal counter
+            if counter < 2:
+                counter += 1
+                open(os.path.join('source_path', '{}.rosinstall'.format(
+                    counter)), 'w').close()
+
+        self.wstool_mock.update.side_effect = _fake_wstool_update
+
+        os.mkdir('source_path')
+        open(os.path.join('source_path', '0.rosinstall'), 'w').close()
+
+        catkin._recursively_handle_rosinstall_files(
+            self.wstool_mock, 'source_path')
+
+        self.wstool_mock.merge.assert_has_calls([
+            mock.call(os.path.join('source_path', '0.rosinstall')),
+            mock.call(os.path.join('source_path', '1.rosinstall')),
+            mock.call(os.path.join('source_path', '2.rosinstall'))
+        ])
+
+
 class CompilersTestCase(unit.TestCase):
 
     def setUp(self):