Repository: canonical/snapcraft

Issue:
Implement support for architecture in snapcraft.yaml

Implement support for architecture in snapcraft.yaml



Discussion:
Proposal has changed a bit, discussion is ongoing here: https://forum.snapcraft.io/t/proposal-architectures-keyword-for-snap-ci-systems/4675
Due to the redesign, this likely won't make it for this milestone, so we'll focus on docs instead.
Proposal has changed a bit, discussion is ongoing here: https://forum.snapcraft.io/t/proposal-architectures-keyword-for-snap-ci-systems/4675
Due to the redesign, this likely won't make it for this milestone, so we'll focus on docs instead.

Resolving patch:
diff --git a/snapcraft/internal/project_loader/_config.py b/snapcraft/internal/project_loader/_config.py
--- a/snapcraft/internal/project_loader/_config.py
+++ b/snapcraft/internal/project_loader/_config.py
@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import codecs
+import collections
 import logging
 import os
 import os.path
@@ -26,7 +27,7 @@
 from typing import Set  # noqa: F401
 
 
-from snapcraft import project
+from snapcraft import project, formatting_utils
 from snapcraft.project._project_info import ProjectInfo
 from snapcraft.internal import deprecations, remote_parts, states
 
@@ -71,6 +72,114 @@ def _validate_epoch(instance):
     return True
 
 
+@jsonschema.FormatChecker.cls_checks('architectures')
+def _validate_architectures(instance):
+    standalone_build_ons = collections.Counter()
+    build_ons = collections.Counter()
+    run_ons = collections.Counter()
+
+    saw_strings = False
+    saw_dicts = False
+
+    for item in instance:
+        # This could either be a dict or a string. In the latter case, the
+        # schema will take care of it. We just need to further validate the
+        # dict.
+        if isinstance(item, str):
+            saw_strings = True
+        elif isinstance(item, dict):
+            saw_dicts = True
+            build_on = _get_architectures_set(item, 'build-on')
+            build_ons.update(build_on)
+
+            # Add to the list of run-ons. However, if no run-on is specified,
+            # we know it's implicitly the value of build-on, so use that
+            # for validation instead.
+            run_on = _get_architectures_set(item, 'run-on')
+            if run_on:
+                run_ons.update(run_on)
+            else:
+                standalone_build_ons.update(build_on)
+
+    # Architectures can either be a list of strings, or a list of objects.
+    # Mixing the two forms is unsupported.
+    if saw_strings and saw_dicts:
+        raise jsonschema.exceptions.ValidationError(
+            'every item must either be a string or an object',
+            path=['architectures'], instance=instance)
+
+    # At this point, individual build-ons and run-ons have been validated,
+    # we just need to validate them across each other.
+
+    # First of all, if we have a `run-on: [all]` (or a standalone
+    # `build-on: [all]`) then we should only have one item in the instance,
+    # otherwise we know we'll have multiple snaps claiming they run on the same
+    # architectures (i.e. all and something else).
+    number_of_snaps = len(instance)
+    if 'all' in run_ons and number_of_snaps > 1:
+        raise jsonschema.exceptions.ValidationError(
+            "one of the items has 'all' in 'run-on', but there are {} "
+            "items: upon release they will conflict. 'all' should only be "
+            "used if there is a single item".format(
+                number_of_snaps),
+            path=['architectures'], instance=instance)
+    if 'all' in build_ons and number_of_snaps > 1:
+        raise jsonschema.exceptions.ValidationError(
+            "one of the items has 'all' in 'build-on', but there are {} "
+            "items: snapcraft doesn't know which one to use. 'all' should "
+            "only be used if there is a single item".format(
+                number_of_snaps),
+            path=['architectures'], instance=instance)
+
+    # We want to ensure that multiple `run-on`s (or standalone `build-on`s)
+    # don't incude the same arch, or they'll clash with each other when
+    # releasing.
+    all_run_ons = run_ons + standalone_build_ons
+    duplicates = {arch for (arch, count) in all_run_ons.items() if count > 1}
+    if duplicates:
+        raise jsonschema.exceptions.ValidationError(
+            'multiple items will build snaps that claim to run on {}'.format(
+                formatting_utils.humanize_list(duplicates, 'and')),
+            path=['architectures'], instance=instance)
+
+    # Finally, ensure that multiple `build-on`s don't include the same arch
+    # or Snapcraft has no way of knowing which one to use.
+    duplicates = {arch for (arch, count) in build_ons.items() if count > 1}
+    if duplicates:
+        raise jsonschema.exceptions.ValidationError(
+            "{} {} present in the 'build-on' of multiple items, which means "
+            "snapcraft doesn't know which 'run-on' to use when building on "
+            "{} {}".format(
+                formatting_utils.humanize_list(duplicates, 'and'),
+                formatting_utils.pluralize(duplicates, 'is', 'are'),
+                formatting_utils.pluralize(duplicates, 'that', 'those'),
+                formatting_utils.pluralize(
+                    duplicates, 'architecture', 'architectures')),
+            path=['architectures'], instance=instance)
+
+    return True
+
+
+def _get_architectures_set(item, name):
+    value = item.get(name, set())
+    if isinstance(value, str):
+        value_set = {value}
+    else:
+        value_set = set(value)
+
+    _validate_architectures_set(value_set, name)
+
+    return value_set
+
+
+def _validate_architectures_set(architectures_set, name):
+    if 'all' in architectures_set and len(architectures_set) > 1:
+        raise jsonschema.exceptions.ValidationError(
+            "'all' can only be used within {!r} by itself, "
+            "not with other architectures".format(name),
+            path=['architectures'], instance=architectures_set)
+
+
 class Config:
 
     @property
@@ -124,8 +233,8 @@ def __init__(self, project_options: project.Project=None) -> None:
                                  build_tools=self.build_tools,
                                  snapcraft_yaml=self.snapcraft_yaml_path)
 
-        if 'architectures' not in self.data:
-            self.data['architectures'] = [self._project_options.deb_arch]
+        self.data['architectures'] = _process_architectures(
+            self.data.get('architectures'), self._project_options.deb_arch)
 
     def get_metadata(self):
         return {'name': self.data['name'],
@@ -294,3 +403,50 @@ def _expand_filesets_for(step, properties):
             new_step_set.append(item)
 
     return new_step_set
+
+
+class _Architecture:
+    def __init__(self, *, build_on, run_on=None):
+        if isinstance(build_on, str):
+            self.build_on = [build_on]
+        else:
+            self.build_on = build_on
+
+        # If there is no run_on, it defaults to the value of build_on
+        if not run_on:
+            self.run_on = self.build_on
+        elif isinstance(run_on, str):
+            self.run_on = [run_on]
+        else:
+            self.run_on = run_on
+
+
+def _create_architecture_list(architectures, current_arch):
+    if not architectures:
+        return [_Architecture(build_on=[current_arch])]
+
+    build_architectures = []  # type: List[str]
+    architecture_list = []  # type: List[_Architecture]
+    for item in architectures:
+        if isinstance(item, str):
+            build_architectures.append(item)
+        if isinstance(item, dict):
+            architecture_list.append(_Architecture(
+                build_on=item.get('build-on'),
+                run_on=item.get('run-on')))
+
+    if build_architectures:
+        architecture_list.append(_Architecture(build_on=build_architectures))
+
+    return architecture_list
+
+
+def _process_architectures(architectures, current_arch):
+    architecture_list = _create_architecture_list(architectures, current_arch)
+
+    for architecture in architecture_list:
+        if (current_arch in architecture.build_on or
+                'all' in architecture.build_on):
+            return architecture.run_on
+
+    return [current_arch]
diff --git a/snapcraft/internal/project_loader/errors.py b/snapcraft/internal/project_loader/errors.py
--- a/snapcraft/internal/project_loader/errors.py
+++ b/snapcraft/internal/project_loader/errors.py
@@ -136,7 +136,11 @@ def _determine_cause(error):
             key = contextual_error.schema_path.popleft()
             if key not in contextual_messages:
                 contextual_messages[key] = []
-            contextual_messages[key].append(contextual_error.message)
+            message = contextual_error.message
+            if message:
+                # Sure it starts lower-case (not all messages do)
+                contextual_messages[key].append(
+                    message[0].lower() + message[1:])
 
         oneOf_messages = []  # type: List[str]
         for key, value in contextual_messages.items():


Test patch:
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
--- a/tests/integration/__init__.py
+++ b/tests/integration/__init__.py
@@ -214,6 +214,7 @@ def construct_yaml(self, name='test', version='0.1',
                        summary='Simple test snap',
                        description='Something something',
                        grade=None,
+                       architectures=None,
                        parts=dedent('''\
                            my-part:
                              plugin: nil
@@ -234,6 +235,8 @@ def construct_yaml(self, name='test', version='0.1',
             snapcraft_yaml['adopt-info'] = adopt_info
         if grade:
             snapcraft_yaml['grade'] = grade
+        if architectures:
+            snapcraft_yaml['architectures'] = architectures
 
         with open('snapcraft.yaml', 'w') as f:
             yaml.dump(snapcraft_yaml, f, default_flow_style=False)
diff --git a/tests/integration/general/test_architectures.py b/tests/integration/general/test_architectures.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/general/test_architectures.py
@@ -0,0 +1,59 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright (C) 2018 Canonical Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import yaml
+
+from testtools.matchers import Equals
+
+from tests import integration
+
+
+class ArchitecturesTestCase(integration.TestCase):
+
+    def test_simple_architecture(self):
+        self.construct_yaml(architectures=[self.deb_arch])
+
+        self.run_snapcraft('prime')
+
+        with open(os.path.join('prime', 'meta', 'snap.yaml')) as f:
+            y = yaml.load(f)
+
+        self.assertThat(y['architectures'], Equals([self.deb_arch]))
+
+    def test_build_and_run_on_architecture(self):
+        self.construct_yaml(architectures=[
+            {'build-on': [self.deb_arch], 'run-on': [self.deb_arch]},
+        ])
+
+        self.run_snapcraft('prime')
+
+        with open(os.path.join('prime', 'meta', 'snap.yaml')) as f:
+            y = yaml.load(f)
+
+        self.assertThat(y['architectures'], Equals([self.deb_arch]))
+
+    def test_default_architecture(self):
+        self.construct_yaml(architectures=[
+            {'build-on': ['other-arch']},
+        ])
+
+        self.run_snapcraft('prime')
+
+        with open(os.path.join('prime', 'meta', 'snap.yaml')) as f:
+            y = yaml.load(f)
+
+        self.assertThat(y['architectures'], Equals([self.deb_arch]))
diff --git a/tests/unit/commands/test_snap.py b/tests/unit/commands/test_snap.py
--- a/tests/unit/commands/test_snap.py
+++ b/tests/unit/commands/test_snap.py
@@ -43,7 +43,9 @@ class SnapCommandBaseTestCase(CommandBaseTestCase):
         version: 1.0
         summary: test snapping
         description: if snap is successful a snap package will be available
-        architectures: ['amd64']
+        architectures:
+          - build-on: all
+            run-on: 'amd64'
         type: {}
         confinement: strict
         grade: stable
diff --git a/tests/unit/project_loader/test_config.py b/tests/unit/project_loader/test_config.py
--- a/tests/unit/project_loader/test_config.py
+++ b/tests/unit/project_loader/test_config.py
@@ -515,7 +515,9 @@ def test_get_metadata(self):
 version: "1"
 summary: test
 description: nothing
-architectures: ['amd64']
+architectures:
+  - build-on: all
+    run-on: amd64
 confinement: strict
 grade: stable
 
@@ -1574,6 +1576,127 @@ def test_invalid_yaml_invalid_epochs(self):
             "followed by an optional asterisk\)")
 
 
+class ValidArchitecturesYamlTestCase(YamlBaseTestCase):
+
+    yaml_scenarios = [
+        ('none', {
+            'expected_amd64': ['amd64'],
+            'expected_i386': ['i386'],
+            'expected_armhf': ['armhf'],
+            'yaml': None,
+        }),
+        ('single string list', {
+            'expected_amd64': ['amd64'],
+            'expected_i386': ['i386'],
+            'expected_armhf': ['armhf'],
+            'yaml': '[amd64]',
+        }),
+        ('multiple string list', {
+            'expected_amd64': ['amd64', 'i386'],
+            'expected_i386': ['amd64', 'i386'],
+            'expected_armhf': ['armhf'],
+            'yaml': '[amd64, i386]',
+        }),
+        ('single object list', {
+            'expected_amd64': ['amd64'],
+            'expected_i386': ['i386'],
+            'expected_armhf': ['armhf'],
+            'yaml': dedent("""
+                - build-on: [amd64]
+                  run-on: [amd64]
+            """),
+        }),
+        ('multiple object list', {
+            'expected_amd64': ['amd64'],
+            'expected_i386': ['i386'],
+            'expected_armhf': ['armhf', 'arm64'],
+            'yaml': dedent("""
+                - build-on: [amd64]
+                  run-on: [amd64]
+                - build-on: [i386]
+                  run-on: [i386]
+                - build-on: [armhf]
+                  run-on: [armhf, arm64]
+            """),
+        }),
+        ('omit run-on', {
+            'expected_amd64': ['amd64'],
+            'expected_i386': ['i386'],
+            'expected_armhf': ['armhf'],
+            'yaml': dedent("""
+                - build-on: [amd64]
+            """),
+        }),
+        ('single build-on string, no list', {
+            'expected_amd64': ['amd64'],
+            'expected_i386': ['i386'],
+            'expected_armhf': ['armhf'],
+            'yaml': dedent("""
+                - build-on: amd64
+            """),
+        }),
+        ('build- and run-on string, no lists', {
+            'expected_amd64': ['amd64'],
+            'expected_i386': ['amd64'],
+            'expected_armhf': ['armhf'],
+            'yaml': dedent("""
+                - build-on: i386
+                  run-on: amd64
+            """),
+        }),
+        ('build on all', {
+            'expected_amd64': ['amd64'],
+            'expected_i386': ['amd64'],
+            'expected_armhf': ['amd64'],
+            'yaml': dedent("""
+                - build-on: [all]
+                  run-on: [amd64]
+            """),
+        }),
+        ('run on all', {
+            'expected_amd64': ['all'],
+            'expected_i386': ['i386'],
+            'expected_armhf': ['armhf'],
+            'yaml': dedent("""
+                - build-on: [amd64]
+                  run-on: [all]
+            """),
+        }),
+    ]
+
+    arch_scenarios = [
+        ('amd64', {'deb_arch': 'amd64'}),
+        ('i386', {'deb_arch': 'i386'}),
+        ('armhf', {'deb_arch': 'armhf'}),
+    ]
+
+    scenarios = multiply_scenarios(yaml_scenarios, arch_scenarios)
+
+    def test_architectures(self):
+        snippet = ''
+        if self.yaml:
+            snippet = 'architectures: {}'.format(self.yaml)
+        self.make_snapcraft_yaml(dedent("""\
+            name: test
+            version: "1"
+            summary: test
+            description: test
+            {}
+            parts:
+              my-part:
+                plugin: nil
+        """).format(snippet))
+
+        try:
+            c = _config.Config(
+                snapcraft.project.Project(target_deb_arch=self.deb_arch))
+
+            expected = getattr(self, 'expected_{}'.format(self.deb_arch))
+            self.assertThat(c.data['architectures'], Equals(expected))
+        except errors.YamlValidationError as e:
+            self.fail('Expected YAML to be valid, got an error: {}'.format(e))
+
+
 class InitTestCase(unit.TestCase):
 
     def test_config_raises_on_missing_snapcraft_yaml(self):
@@ -2544,6 +2667,108 @@ def test_invalid_part_names(self):
                         message=data)
 
 
+class InvalidArchitecturesTestCase(ValidationBaseTestCase):
+
+    scenarios = [
+        ('single string', {
+            'architectures': 'amd64',
+            'message': "'amd64' is not of type 'array'",
+        }),
+        ('unknown object properties', {
+            'architectures': [{'builds-on': ['amd64'], 'runs-on': ['amd64']}],
+            'message': "'build-on' is a required property and additional "
+                       "properties are not allowed",
+        }),
+        ('omit build-on', {
+            'architectures': [{'run-on': ['amd64']}],
+            'message': "'build-on' is a required property",
+        }),
+        ('build on all and others', {
+            'architectures': [{'build-on': ['amd64', 'all']}],
+            'message': "'all' can only be used within 'build-on' by itself, "
+                       "not with other architectures",
+        }),
+        ('run on all and others', {
+            'architectures': [
+                {'build-on': ['amd64'], 'run-on': ['amd64', 'all']}],
+            'message': "'all' can only be used within 'run-on' by itself, "
+                       "not with other architectures",
+        }),
+        ('run on all and more objects', {
+            'architectures': [
+                {'build-on': ['amd64'], 'run-on': ['all']},
+                {'build-on': ['i396'], 'run-on': ['i386']},
+            ],
+            'message': "one of the items has 'all' in 'run-on', but there are "
+                       "2 items: upon release they will conflict. 'all' "
+                       "should only be used if there is a single item",
+        }),
+        ('build on all and more objects', {
+            'architectures': [
+                {'build-on': ['all']},
+                {'build-on': ['i396']},
+            ],
+            'message': "one of the items has 'all' in 'build-on', but there "
+                       "are 2 items: snapcraft doesn't know which one to use. "
+                       "'all' should only be used if there is a single item",
+        }),
+        ('multiple builds run on same arch', {
+            'architectures': [
+                {'build-on': ['amd64'], 'run-on': ['amd64']},
+                {'build-on': ['i396'], 'run-on': ['amd64', 'i386']},
+            ],
+            'message': "multiple items will build snaps that claim to run on "
+                       "'amd64'",
+        }),
+        ('multiple builds run on same arch with implicit run-on', {
+            'architectures': [
+                {'build-on': ['amd64']},
+                {'build-on': ['i396'], 'run-on': ['amd64', 'i386']},
+            ],
+            'message': "multiple items will build snaps that claim to run on "
+                       "'amd64'",
+        }),
+        ('mixing forms', {
+            'architectures': [
+                'amd64',
+                {'build-on': ['i386'], 'run-on': ['amd64', 'i386']},
+            ],
+            'message': 'every item must either be a string or an object',
+        }),
+        ('build on all run on specific', {
+            'architectures': [
+                {'build-on': ['all'], 'run-on': ['amd64']},
+                {'build-on': ['all'], 'run-on': ['i386']},
+            ],
+            'message': "one of the items has 'all' in 'build-on', but there "
+                       "are 2 items: snapcraft doesn't know which one to use. "
+                       "'all' should only be used if there is a single item",
+        }),
+        ('build on overlap', {
+            'architectures': [
+                {'build-on': ['amd64', 'i386'], 'run-on': ['i386']},
+                {'build-on': ['amd64'], 'run-on': ['amd64']},
+            ],
+            'message': "'amd64' is present in the 'build-on' of multiple "
+                       "items, which means snapcraft doesn't know which "
+                       "'run-on' to use when building on that architecture",
+        }),
+    ]
+
+    def test_invalid_architectures(self):
+        data = self.data.copy()
+        data['architectures'] = self.architectures
+
+        raised = self.assertRaises(
+            errors.YamlValidationError,
+            project_loader.Validator(data).validate)
+
+        self.assertThat(
+            raised.message, MatchesRegex(
+                "The 'architectures.*?' property does not match the required "
+                "schema: {}".format(self.message)), message=data)
+
+
 class TestPluginLoadingProperties(unit.TestCase):
 
     scenarios = [
diff --git a/tests/unit/test_meta.py b/tests/unit/test_meta.py
--- a/tests/unit/test_meta.py
+++ b/tests/unit/test_meta.py
@@ -50,7 +50,7 @@ def setUp(self):
         super().setUp()
 
         self.config_data = {
-            'architectures': ['amd64'],
+            'architectures': [{'build-on': 'all', 'run-on': 'amd64'}],
             'name': 'my-package',
             'version': '1.0',
             'description': 'my description',
@@ -122,6 +122,22 @@ def test_create_meta(self):
 
         self.assertThat(y, Equals(expected))
 
+    def test_create_meta_default_architecture(self):
+        del self.config_data['architectures']
+
+        y = self.generate_meta_yaml()
+
+        expected = {'architectures': [self.project_options.deb_arch],
+                    'confinement': 'devmode',
+                    'grade': 'stable',
+                    'description': 'my description',
+                    'environment': {'GLOBAL': 'y'},
+                    'summary': 'my summary',
+                    'name': 'my-package',
+                    'version': '1.0'}
+
+        self.assertThat(y, Equals(expected))
+
     def test_create_meta_with_epoch(self):
         self.config_data['epoch'] = '1*'