Repository: canonical/snapcraft

Issue:
catkin plugin: support for building all packages in a given workspace
Estimate: 4 days

The Catkin plugin currently requires a list of catkin-packages that represents the packages the developer wants installed into the snap. For larger projects, this list becomes unmaintainable. I got requests from a few different companies to “please” support just installing the entire workspace.

Since we actually support there being no packages (i.e. an empty list), my current plan is to support not specifying catkin-packages at all to indicate this desire.



Discussion:

Resolving patch:
diff --git a/snapcraft/plugins/_ros/rosdep.py b/snapcraft/plugins/_ros/rosdep.py
--- a/snapcraft/plugins/_ros/rosdep.py
+++ b/snapcraft/plugins/_ros/rosdep.py
@@ -99,13 +99,31 @@ def setup(self):
             raise RuntimeError(
                 'Error updating rosdep database:\n{}'.format(output))
 
-    def get_dependencies(self, package_name):
+    def get_dependencies(self, package_name=None):
+        """Obtain dependencies for a given package, or entire workspace.
+
+        :param str package_name: Package name for which dependences will be
+                                 obtained. If not provided, will obtain
+                                 dependencies for the entire workspace.
+        """
+        command = ['keys']
+        if package_name:
+            command.append(package_name)
+        else:
+            # Adding a few flags that will make rosdep search the entire
+            # workspace:
+            #
+            # -a: select all packages in workspace
+            # -i: ignore any resolved keys that are satisfied by other packages
+            #     in the workspace
+            command.append('-a')
+            command.append('-i')
         try:
-            output = self._run(['keys', package_name]).strip()
+            output = self._run(command).strip()
             if output:
-                return output.split('\n')
+                return set(output.split('\n'))
             else:
-                return []
+                return set()
         except subprocess.CalledProcessError:
             raise FileNotFoundError(
                 'Unable to find Catkin package "{}"'.format(package_name))
diff --git a/snapcraft/plugins/catkin.py b/snapcraft/plugins/catkin.py
--- a/snapcraft/plugins/catkin.py
+++ b/snapcraft/plugins/catkin.py
@@ -139,7 +139,6 @@ def schema(cls):
             'items': {
                 'type': 'string'
             },
-            'default': [],
         }
         schema['properties']['source-space'] = {
             'type': 'string',
@@ -192,8 +191,6 @@ def schema(cls):
             'default': 'http://localhost:11311'
         }
 
-        schema['required'].append('catkin-packages')
-
         return schema
 
     @classmethod
@@ -222,16 +219,24 @@ def _pip(self):
 
     @property
     def PLUGIN_STAGE_SOURCES(self):
-        return """
-deb http://packages.ros.org/ros/ubuntu/ {0} main
-deb http://${{prefix}}.ubuntu.com/${{suffix}}/ {0} main universe
-deb http://${{prefix}}.ubuntu.com/${{suffix}}/ {0}-updates main universe
-deb http://${{prefix}}.ubuntu.com/${{suffix}}/ {0}-security main universe
-deb http://${{security}}.ubuntu.com/${{suffix}} {0}-security main universe
-""".format(_ROS_RELEASE_MAP[self.options.rosdistro])
+        ros_repo = 'http://packages.ros.org/ros/ubuntu/'
+        ubuntu_repo = 'http://${prefix}.ubuntu.com/${suffix}/'
+        security_repo = 'http://${security}.ubuntu.com/${suffix}/'
+        return textwrap.dedent("""
+            deb {ros_repo} {codename} main
+            deb {ubuntu_repo} {codename} main universe
+            deb {ubuntu_repo} {codename}-updates main universe
+            deb {ubuntu_repo} {codename}-security main universe
+            deb {security_repo} {codename}-security main universe
+            """.format(
+                ros_repo=ros_repo,
+                ubuntu_repo=ubuntu_repo,
+                security_repo=security_repo,
+                codename=_ROS_RELEASE_MAP[self.options.rosdistro]))
 
     def __init__(self, name, options, project):
         super().__init__(name, options, project)
+
         self.build_packages.extend(['libc6-dev', 'make', 'python-pip'])
         self.__pip = None
 
@@ -241,7 +246,9 @@ def __init__(self, name, options, project):
             'ros-{}-roslib'.format(self.options.rosdistro))
 
         # Get a unique set of packages
-        self.catkin_packages = set(options.catkin_packages)
+        self.catkin_packages = None
+        if options.catkin_packages is not None:
+            self.catkin_packages = set(options.catkin_packages)
         self._rosdep_path = os.path.join(self.partdir, 'rosdep')
         self._compilers_path = os.path.join(self.partdir, 'compilers')
         self._catkin_path = os.path.join(self.partdir, 'catkin')
@@ -368,8 +375,13 @@ def pull(self):
             _handle_rosinstall_files(
                 wstool,  source_path, self.options.rosinstall_files)
 
-        # Make sure the package path exists before continuing
-        if self.catkin_packages and not os.path.exists(self._ros_package_path):
+        # Make sure the package path exists before continuing. We only care
+        # about doing this if there are actually packages to build, which is
+        # indicated both by self.catkin_packages being None as well as a
+        # non-empty list.
+        packages_to_build = (
+            self.catkin_packages is None or len(self.catkin_packages) > 0)
+        if packages_to_build and not os.path.exists(self._ros_package_path):
             raise FileNotFoundError(
                 'Unable to find package path: "{}"'.format(
                     self._ros_package_path))
@@ -676,7 +688,7 @@ def _use_in_snap_python(self):
 
     def _build_catkin_packages(self):
         # Nothing to do if no packages were specified
-        if not self.catkin_packages:
+        if self.catkin_packages is not None and len(self.catkin_packages) == 0:
             return
 
         catkincmd = ['catkin_make_isolated']
@@ -684,9 +696,10 @@ def _build_catkin_packages(self):
         # Install the package
         catkincmd.append('--install')
 
-        # Specify the packages to be built
-        catkincmd.append('--pkg')
-        catkincmd.extend(self.catkin_packages)
+        if self.catkin_packages:
+            # Specify the packages to be built
+            catkincmd.append('--pkg')
+            catkincmd.extend(self.catkin_packages)
 
         # Don't clutter the real ROS workspace-- use the Snapcraft build
         # directory
@@ -745,16 +758,22 @@ def _find_system_dependencies(catkin_packages, rosdep, catkin):
     """Find system dependencies for a given set of Catkin packages."""
 
     resolved_dependencies = {}
+    dependencies = set()
 
     logger.info('Determining system dependencies for Catkin packages...')
-    for package in catkin_packages:
-        # Query rosdep for the list of dependencies for this package
-        dependencies = rosdep.get_dependencies(package)
-
-        for dependency in dependencies:
-            _resolve_package_dependencies(
-                catkin_packages, dependency, catkin, rosdep,
-                resolved_dependencies)
+    if catkin_packages is not None:
+        for package in catkin_packages:
+            # Query rosdep for the list of dependencies for this package
+            dependencies |= rosdep.get_dependencies(package)
+    else:
+        # Rather than getting dependencies for an explicit list of packages,
+        # let's get the dependencies for the entire workspace.
+        dependencies |= rosdep.get_dependencies()
+
+    for dependency in dependencies:
+        _resolve_package_dependencies(
+            catkin_packages, dependency, catkin, rosdep,
+            resolved_dependencies)
 
     # We currently have nested dict structure of:
     #    dependency name -> package type -> package names
@@ -775,8 +794,8 @@ def _resolve_package_dependencies(catkin_packages, dependency, catkin, rosdep,
                                   resolved_dependencies):
     # No need to resolve this dependency if we know it's local, or if
     # we've already resolved it into a system dependency
-    if (dependency in catkin_packages or
-            dependency in resolved_dependencies):
+    if (dependency in resolved_dependencies or
+            (catkin_packages and dependency in catkin_packages)):
         return
 
     if _dependency_is_in_underlay(catkin, dependency):


Test patch:
diff --git a/snapcraft/tests/integration/snaps/ros-talker-listener/snap/snapcraft.yaml b/snapcraft/tests/integration/snaps/ros-talker-listener/snap/snapcraft.yaml
--- a/snapcraft/tests/integration/snaps/ros-talker-listener/snap/snapcraft.yaml
+++ b/snapcraft/tests/integration/snaps/ros-talker-listener/snap/snapcraft.yaml
@@ -13,7 +13,4 @@ parts:
   ros-project:
     plugin: catkin
     source: .
-    catkin-packages:
-      - talker
-      - listener
-    include-roscore: true
+    rosdistro: indigo
diff --git a/snapcraft/tests/unit/plugins/ros/test_rosdep.py b/snapcraft/tests/unit/plugins/ros/test_rosdep.py
--- a/snapcraft/tests/unit/plugins/ros/test_rosdep.py
+++ b/snapcraft/tests/unit/plugins/ros/test_rosdep.py
@@ -111,7 +111,7 @@ def test_get_dependencies(self):
         self.check_output_mock.return_value = b'foo\nbar\nbaz'
 
         self.assertThat(self.rosdep.get_dependencies('foo'),
-                        Equals(['foo', 'bar', 'baz']))
+                        Equals({'foo', 'bar', 'baz'}))
 
         self.check_output_mock.assert_called_with(['rosdep', 'keys', 'foo'],
                                                   env=mock.ANY)
@@ -119,7 +119,7 @@ def test_get_dependencies(self):
     def test_get_dependencies_no_dependencies(self):
         self.check_output_mock.return_value = b''
 
-        self.assertThat(self.rosdep.get_dependencies('foo'), Equals([]))
+        self.assertThat(self.rosdep.get_dependencies('foo'), Equals(set()))
 
     def test_get_dependencies_invalid_package(self):
         self.check_output_mock.side_effect = subprocess.CalledProcessError(
@@ -132,6 +132,15 @@ def test_get_dependencies_invalid_package(self):
         self.assertThat(str(raised),
                         Equals('Unable to find Catkin package "bar"'))
 
+    def test_get_dependencies_entire_workspace(self):
+        self.check_output_mock.return_value = b'foo\nbar\nbaz'
+
+        self.assertThat(self.rosdep.get_dependencies(),
+                        Equals({'foo', 'bar', 'baz'}))
+
+        self.check_output_mock.assert_called_with(
+            ['rosdep', 'keys', '-a', '-i'], env=mock.ANY)
+
     def test_resolve_dependency(self):
         self.check_output_mock.return_value = b'#apt\nmylib-dev'
 
diff --git a/snapcraft/tests/unit/plugins/test_catkin.py b/snapcraft/tests/unit/plugins/test_catkin.py
--- a/snapcraft/tests/unit/plugins/test_catkin.py
+++ b/snapcraft/tests/unit/plugins/test_catkin.py
@@ -171,22 +171,16 @@ def test_schema_catkin_packages(self):
 
         # Check catkin-packages property
         catkin_packages = schema['properties']['catkin-packages']
-        expected = ('type', 'default', 'minitems', 'uniqueItems', 'items')
+        expected = ('type', 'minitems', 'uniqueItems', 'items')
         self.assertThat(catkin_packages, HasLength(len(expected)))
         for prop in expected:
             self.assertThat(catkin_packages, Contains(prop))
         self.assertThat(catkin_packages['type'], Equals('array'))
-        self.assertThat(catkin_packages['default'], Equals([]))
         self.assertThat(catkin_packages['minitems'], Equals(1))
         self.assertTrue(catkin_packages['uniqueItems'])
         self.assertThat(catkin_packages['items'], Contains('type'))
         self.assertThat(catkin_packages['items']['type'], Equals('string'))
 
-        # Check required
-        self.assertTrue('catkin-packages' in schema['required'],
-                        'Expected "catkin-packages" to be included in '
-                        '"required"')
-
     def test_schema_source_space(self):
         schema = catkin.CatkinPlugin.schema()
 
@@ -464,6 +458,34 @@ def test_invalid_catkin_workspace_invalid_source_space(self):
             Equals('Unable to find package path: "{}"'.format(os.path.join(
                 plugin.sourcedir, self.properties.source_space))))
 
+    def test_invalid_catkin_workspace_invalid_source_space_build_all(self):
+        self.properties.source_space = 'foo'
+        self.properties.catkin_packages = None
+
+        # sourcedir is expected to be the root of the Catkin workspace. Since
+        # it does not contain a `src` folder and source_space wasn't
+        # specified, this should fail.
+        plugin = catkin.CatkinPlugin('test-part', self.properties,
+                                     self.project_options)
+        raised = self.assertRaises(FileNotFoundError, plugin.pull)
+
+        self.assertThat(
+            str(raised),
+            Equals('Unable to find package path: "{}"'.format(os.path.join(
+                plugin.sourcedir, self.properties.source_space))))
+
+    def test_invalid_catkin_workspace_invalid_source_no_packages(self):
+        """Test that an invalid source space is fine iff no packages."""
+        self.properties.source_space = 'foo'
+        self.properties.catkin_packages = []
+
+        plugin = catkin.CatkinPlugin('test-part', self.properties,
+                                     self.project_options)
+
+        # Normally pulling should fail, but since there are no packages to
+        # build, even an invalid workspace should be okay.
+        plugin.pull()
+
     def test_invalid_catkin_workspace_source_space_same_as_source(self):
         self.properties.source_space = '.'
 
@@ -1193,6 +1215,63 @@ def __eq__(self, args):
 
         finish_build_mock.assert_called_once_with()
 
+    @mock.patch('snapcraft.plugins.catkin.Compilers')
+    @mock.patch.object(catkin.CatkinPlugin, 'run')
+    @mock.patch.object(catkin.CatkinPlugin, '_run_in_bash')
+    @mock.patch.object(catkin.CatkinPlugin, 'run_output', return_value='foo')
+    @mock.patch.object(catkin.CatkinPlugin, '_prepare_build')
+    @mock.patch.object(catkin.CatkinPlugin, '_finish_build')
+    def test_build_all_packages(self, finish_build_mock, prepare_build_mock,
+                                run_output_mock, bashrun_mock, run_mock,
+                                compilers_mock):
+        self.properties.catkin_packages = None
+        plugin = catkin.CatkinPlugin('test-part', self.properties,
+                                     self.project_options)
+        os.makedirs(os.path.join(plugin.sourcedir, 'src'))
+
+        plugin.build()
+
+        prepare_build_mock.assert_called_once_with()
+
+        # Matching like this for order independence (otherwise it would be
+        # quite fragile)
+        build_attributes = self.build_attributes
+        catkin_cmake_args = self.catkin_cmake_args
+
+        class check_build_command():
+            def __eq__(self, args):
+                command = ' '.join(args)
+                if 'debug' in build_attributes:
+                    build_type_valid = re.match(
+                        '.*--cmake-args.*-DCMAKE_BUILD_TYPE=Debug', command)
+                else:
+                    build_type_valid = re.match(
+                        '.*--cmake-args.*-DCMAKE_BUILD_TYPE=Release', command)
+                args_valid = True
+                if catkin_cmake_args:
+                    expected_args = ' '.join(catkin_cmake_args)
+                    args_valid = re.match(
+                        '.*--cmake-args.*{}'.format(re.escape(expected_args)),
+                        command)
+                return (
+                    args_valid and
+                    build_type_valid and
+                    args[0] == 'catkin_make_isolated' and
+                    '--install' in command and
+                    '--directory {}'.format(plugin.builddir) in command and
+                    '--install-space {}'.format(plugin.rosdir) in command and
+                    '--source-space {}'.format(os.path.join(
+                        plugin.builddir,
+                        plugin.options.source_space)) in command)
+
+        bashrun_mock.assert_called_with(check_build_command(), env=mock.ANY)
+
+        self.assertFalse(
+            self.dependencies_mock.called,
+            'Dependencies should have been discovered in the pull() step')
+
+        finish_build_mock.assert_called_once_with()
+
 
 class FinishBuildTestCase(CatkinPluginBaseTestCase):
 
@@ -1354,7 +1433,7 @@ def setUp(self):
         super().setUp()
 
         self.rosdep_mock = mock.MagicMock()
-        self.rosdep_mock.get_dependencies.return_value = ['bar']
+        self.rosdep_mock.get_dependencies.return_value = {'bar'}
 
         self.catkin_mock = mock.MagicMock()
         exception = catkin.CatkinPackageNotFoundError('foo')
@@ -1373,6 +1452,19 @@ def test_find_system_dependencies_system_only(self):
         self.rosdep_mock.resolve_dependency.assert_called_once_with('bar')
         self.catkin_mock.find.assert_called_once_with('bar')
 
+    def test_find_system_dependencies_system_only_no_packages(self):
+        self.rosdep_mock.resolve_dependency.return_value = {
+            'apt': {'baz'},
+        }
+
+        self.assertThat(catkin._find_system_dependencies(
+            None, self.rosdep_mock, self.catkin_mock), Equals(
+            {'apt': {'baz'}}))
+
+        self.rosdep_mock.get_dependencies.assert_called_once_with()
+        self.rosdep_mock.resolve_dependency.assert_called_once_with('bar')
+        self.catkin_mock.find.assert_called_once_with('bar')
+
     def test_find_system_dependencies_local_only(self):
         self.assertThat(catkin._find_system_dependencies(
             {'foo', 'bar'}, self.rosdep_mock, self.catkin_mock),
@@ -1395,7 +1487,7 @@ def test_find_system_dependencies_satisfied_in_stage(self):
         self.rosdep_mock.resolve_dependency.assert_not_called()
 
     def test_find_system_dependencies_mixed(self):
-        self.rosdep_mock.get_dependencies.return_value = ['bar', 'baz', 'qux']
+        self.rosdep_mock.get_dependencies.return_value = {'bar', 'baz', 'qux'}
         self.rosdep_mock.resolve_dependency.return_value = {
             'apt': {'quux'},
         }