Repository: canonical/snapcraft

Issue:
Add pre-stage/stage/post-stage
Estimate: 1 day

We currently have three scriptlets that cover the build step only: prepare, build, and install. We want scriptlets around all lifecycle steps: pull, build, stage, and prime. Note that introducing an install step is not part of this roadmap.

There are some complications here that I want to point out. The stage and prime steps are completely handled by the core right now. As soon as we change that and provide the ability to override them, we must account for the possibility that files in stage will be changed from what they were in install, etc. Which means the stage dir must flow to the prime dir, which it does not do today (the file list is retrieved from stage, but the files themselves come from the parts’ install dirs).

Note that the stanzas for pull and build can be completed without such a redesign.
Add pre-stage/stage/post-stage
Estimate: 1 day

We currently have three scriptlets that cover the build step only: prepare, build, and install. We want scriptlets around all lifecycle steps: pull, build, stage, and prime. Note that introducing an install step is not part of this roadmap.

There are some complications here that I want to point out. The stage and prime steps are completely handled by the core right now. As soon as we change that and provide the ability to override them, we must account for the possibility that files in stage will be changed from what they were in install, etc. Which means the stage dir must flow to the prime dir, which it does not do today (the file list is retrieved from stage, but the files themselves come from the parts’ install dirs).

Note that the stanzas for pull and build can be completed without such a redesign.


Discussion:

Resolving patch:
diff --git a/snapcraft/cli/snapcraftctl/_runner.py b/snapcraft/cli/snapcraftctl/_runner.py
--- a/snapcraft/cli/snapcraftctl/_runner.py
+++ b/snapcraft/cli/snapcraftctl/_runner.py
@@ -56,6 +56,12 @@ def build():
     _call_function('build')
 
 
+@run.command()
+def stage():
+    """Run the 'stage' step of the calling part's plugin"""
+    _call_function('stage')
+
+
 def _call_function(function_name, args=None):
     if not args:
         args = {}
diff --git a/snapcraft/internal/pluginhandler/__init__.py b/snapcraft/internal/pluginhandler/__init__.py
--- a/snapcraft/internal/pluginhandler/__init__.py
+++ b/snapcraft/internal/pluginhandler/__init__.py
@@ -102,9 +102,11 @@ def __init__(self, *, plugin, part_properties, project_options,
             part_properties=self._part_properties,
             sourcedir=self.plugin.sourcedir,
             builddir=self.plugin.build_basedir,
+            stagedir=self.stagedir,
             builtin_functions={
-                'build': self.plugin.build,
                 'pull': self._do_pull,
+                'build': self.plugin.build,
+                'stage': self._do_stage,
             })
 
         self._migrate_state_file()
@@ -489,6 +491,14 @@ def _organize(self):
     def stage(self, force=False):
         self.makedirs()
         self.notify_part_progress('Staging')
+        self._runner.stage()
+
+        # Only mark this step done if _do_stage() didn't run, in which case
+        # we have no directories or files to track.
+        if self.is_clean('stage'):
+            self.mark_stage_done(set(), set())
+
+    def _do_stage(self):
         snap_files, snap_dirs = self.migratable_fileset_for('stage')
 
         def fixup_func(file_path):
diff --git a/snapcraft/internal/pluginhandler/_runner.py b/snapcraft/internal/pluginhandler/_runner.py
--- a/snapcraft/internal/pluginhandler/_runner.py
+++ b/snapcraft/internal/pluginhandler/_runner.py
@@ -38,21 +38,24 @@ class Runner:
     # https://github.com/python/typing/issues/259 which is fixed in Python
     # 3.5.3.
     def __init__(self, *, part_properties: Dict[str, Any], sourcedir: str,
-                 builddir: str,
+                 builddir: str, stagedir: str,
                  builtin_functions: 'Dict[str, Callable[..., None]]') -> None:
         """Create a new Runner.
         :param dict part_properties: YAML properties set for this part.
         :param str sourcedir: The source directory for this part.
         :param str builddir: The build directory for this part.
+        :param str stagedir: The staging area.
         :param dict builtin_functions: Dict of builtin function names to
                                        actual callables.
         """
         self._sourcedir = sourcedir
         self._builddir = builddir
+        self._stagedir = stagedir
         self._builtin_functions = builtin_functions
 
         self._override_pull_scriptlet = part_properties.get('override-pull')
         self._override_build_scriptlet = part_properties.get('override-build')
+        self._override_stage_scriptlet = part_properties.get('override-stage')
 
         # These are all deprecated
         self._prepare_scriptlet = part_properties.get('prepare')
@@ -96,6 +99,13 @@ def install(self) -> None:
             self._run_scriptlet(
                 'install', self._install_scriptlet, self._builddir)
 
+    def stage(self) -> None:
+        """Run override-stage scriptlet."""
+        if self._override_stage_scriptlet:
+            self._run_scriptlet(
+                'override-stage', self._override_stage_scriptlet,
+                self._stagedir)
+
     def _run_scriptlet(self, scriptlet_name: str, scriptlet: str,
                        workdir: str) -> None:
         with tempfile.TemporaryDirectory() as tempdir:


Test patch:
diff --git a/tests/fixture_setup.py b/tests/fixture_setup.py
--- a/tests/fixture_setup.py
+++ b/tests/fixture_setup.py
@@ -584,9 +584,23 @@ def _setUp(self):
         self.check_output_mock.side_effect = self.check_output_side_effect()
         self.addCleanup(patcher.stop)
 
+        self._real_popen = subprocess.Popen
+
+        # Don't over-mock Popen, more things use it than just LXD.
+        def _fake_popen(*args, **kwargs):
+            if '/usr/lib/sftp-server' in args[0]:
+                return self._popen(args[0])
+            elif (args[0][:2] == ['lxc', 'exec'] and self.status and
+                    args[0][2] == self.name and args[0][8] == 'sshfs'):
+                self.files = ['foo', 'bar']
+                return self._popen(args[0])
+
+            # Fall back to the real deal
+            return self._real_popen(*args, **kwargs)
+
         patcher = mock.patch('subprocess.Popen')
         self.popen_mock = patcher.start()
-        self.popen_mock.side_effect = self.check_output_side_effect()
+        self.popen_mock.side_effect = _fake_popen
         self.addCleanup(patcher.stop)
 
         patcher = mock.patch('time.sleep', lambda _: None)
@@ -633,8 +647,6 @@ def call_effect(self, *args, **kwargs):
                 '"created_at":"test-created-at"}]').encode('utf-8')
         elif args[0][0] == 'sha384sum':
             return 'deadbeef {}'.format(args[0][1]).encode('utf-8')
-        elif '/usr/lib/sftp-server' in args[0]:
-            return self._popen(args[0])
         else:
             return ''.encode('utf-8')
 
@@ -664,9 +676,6 @@ def _lxc_exec(self, args):
             elif cmd == 'readlink':
                 if args[0][-1].endswith('/current'):
                     raise CalledProcessError(returncode=1, cmd=cmd)
-            elif cmd == 'sshfs':
-                self.files = ['foo', 'bar']
-                return self._popen(args[0])
             elif 'sha384sum' in args[0][-1]:
                 raise CalledProcessError(returncode=1, cmd=cmd)
 
diff --git a/tests/integration/general/test_scriptlets.py b/tests/integration/general/test_scriptlets.py
--- a/tests/integration/general/test_scriptlets.py
+++ b/tests/integration/general/test_scriptlets.py
@@ -17,7 +17,7 @@
 import multiprocessing
 import os
 
-from testtools.matchers import FileContains, FileExists
+from testtools.matchers import FileContains, FileExists, Not
 
 from tests import integration
 
@@ -88,3 +88,32 @@ def test_override_build(self):
             os.path.join(installdir, 'before-install'), FileExists())
         self.assertThat(
             os.path.join(installdir, 'after-install'), FileExists())
+
+    def test_override_stage(self):
+        self.run_snapcraft('stage', 'scriptlet-override-stage')
+
+        installdir = os.path.join(
+            self.parts_dir, 'override-stage-scriptlet-test', 'install')
+
+        # Assert that the before/after stage files aren't placed in the
+        # installdir, although the file is.
+        self.assertThat(os.path.join(installdir, 'file'), FileExists())
+        self.assertThat(
+            os.path.join(installdir, 'before-stage'), Not(FileExists()))
+        self.assertThat(
+            os.path.join(installdir, 'after-stage'), Not(FileExists()))
+
+        self.assertThat(os.path.join(self.stage_dir, 'file'), FileExists())
+        self.assertThat(
+            os.path.join(self.stage_dir, 'before-stage'), FileExists())
+        self.assertThat(
+            os.path.join(self.stage_dir, 'after-stage'), FileExists())
+
+        # Also assert that, while file2 was installed, it wasn't staged
+        self.assertThat(
+            os.path.join(
+                self.parts_dir, 'override-stage-do-nothing', 'install',
+                'file2'),
+            FileExists())
+        self.assertThat(
+            os.path.join(self.stage_dir, 'file2'), Not(FileExists()))
diff --git a/tests/integration/snaps/scriptlet-override-stage/part1/file b/tests/integration/snaps/scriptlet-override-stage/part1/file
new file mode 100644
--- /dev/null
+++ b/tests/integration/snaps/scriptlet-override-stage/part1/file
@@ -0,0 +1 @@
+I'm a nice file
diff --git a/tests/integration/snaps/scriptlet-override-stage/part2/file2 b/tests/integration/snaps/scriptlet-override-stage/part2/file2
new file mode 100644
--- /dev/null
+++ b/tests/integration/snaps/scriptlet-override-stage/part2/file2
@@ -0,0 +1 @@
+I'm a nicer file
diff --git a/tests/integration/snaps/scriptlet-override-stage/snap/snapcraft.yaml b/tests/integration/snaps/scriptlet-override-stage/snap/snapcraft.yaml
new file mode 100644
--- /dev/null
+++ b/tests/integration/snaps/scriptlet-override-stage/snap/snapcraft.yaml
@@ -0,0 +1,23 @@
+name: override-stage-scriptlet-test
+version: '0.1'
+summary: Runs the override-stage scriptlet for a part
+description: |
+  Runs the shell script defined in `override-stage` instead of plugin stage.
+grade: devel
+confinement: devmode
+
+parts:
+  override-stage-scriptlet-test:
+    plugin: dump
+    source: part1/
+    override-stage: |
+      touch before-stage
+      snapcraftctl stage
+      touch after-stage
+
+  override-stage-do-nothing:
+    plugin: dump
+    source: part2/
+    override-stage: |
+      # Completely skip stage
+      exit 0
diff --git a/tests/unit/pluginhandler/test_runner.py b/tests/unit/pluginhandler/test_runner.py
--- a/tests/unit/pluginhandler/test_runner.py
+++ b/tests/unit/pluginhandler/test_runner.py
@@ -38,16 +38,21 @@ def _fake_build():
     open(os.path.join('builddir', 'fake-build'), 'w').close()
 
 
+def _fake_stage():
+    open(os.path.join('stagedir', 'fake-stage'), 'w').close()
+
+
 class RunnerTestCase(unit.TestCase):
 
     def test_pull(self):
         os.mkdir('sourcedir')
 
         runner = _runner.Runner(
-                part_properties={'override-pull': 'touch pull'},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={})
+            part_properties={'override-pull': 'touch pull'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
 
         runner.pull()
 
@@ -57,10 +62,11 @@ def test_builtin_function_from_pull(self):
         os.mkdir('sourcedir')
 
         runner = _runner.Runner(
-                part_properties={'override-pull': 'snapcraftctl pull'},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={'pull': _fake_pull})
+            part_properties={'override-pull': 'snapcraftctl pull'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={'pull': _fake_pull})
 
         runner.pull()
 
@@ -70,10 +76,11 @@ def test_prepare(self):
         os.mkdir('builddir')
 
         runner = _runner.Runner(
-                part_properties={'prepare': 'touch prepare'},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={})
+            part_properties={'prepare': 'touch prepare'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
 
         runner.prepare()
 
@@ -83,10 +90,11 @@ def test_builtin_function_from_prepare(self):
         os.mkdir('builddir')
 
         runner = _runner.Runner(
-                part_properties={'prepare': 'snapcraftctl build'},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={'build': _fake_build})
+            part_properties={'prepare': 'snapcraftctl build'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={'build': _fake_build})
 
         runner.prepare()
 
@@ -98,12 +106,13 @@ def test_snapcraftctl_alias_if_snap(self):
         os.mkdir('builddir')
 
         runner = _runner.Runner(
-                    part_properties={
-                        'prepare': 'alias snapcraftctl > definition'
-                    },
-                    sourcedir='sourcedir',
-                    builddir='builddir',
-                    builtin_functions={})
+            part_properties={
+                'prepare': 'alias snapcraftctl > definition'
+            },
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
 
         with mock.patch('os.path.exists', return_value=True):
             runner.prepare()
@@ -115,14 +124,29 @@ def test_snapcraftctl_alias_if_snap(self):
             os.path.join('builddir', 'definition'),
             FileContains('snapcraftctl={!r}\n'.format(expected_snapcrafctl)))
 
+    def test_old_build(self):
+        os.mkdir('builddir')
+
+        runner = _runner.Runner(
+            part_properties={'build': 'touch build'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
+
+        runner.build()
+
+        self.assertThat(os.path.join('builddir', 'build'), FileExists())
+
     def test_build(self):
         os.mkdir('builddir')
 
         runner = _runner.Runner(
-                part_properties={'build': 'touch build'},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={})
+            part_properties={'override-build': 'touch build'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
 
         runner.build()
 
@@ -132,10 +156,11 @@ def test_builtin_function_from_build(self):
         os.mkdir('builddir')
 
         runner = _runner.Runner(
-                part_properties={'build': 'snapcraftctl build'},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={'build': _fake_build})
+            part_properties={'override-build': 'snapcraftctl build'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={'build': _fake_build})
 
         runner.build()
 
@@ -145,10 +170,11 @@ def test_install(self):
         os.mkdir('builddir')
 
         runner = _runner.Runner(
-                part_properties={'install': 'touch install'},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={})
+            part_properties={'install': 'touch install'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
 
         runner.install()
 
@@ -158,15 +184,44 @@ def test_builtin_function_from_install(self):
         os.mkdir('builddir')
 
         runner = _runner.Runner(
-                part_properties={'install': 'snapcraftctl build'},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={'build': _fake_build})
+            part_properties={'install': 'snapcraftctl build'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={'build': _fake_build})
 
         runner.install()
 
         self.assertThat(os.path.join('builddir', 'fake-build'), FileExists())
 
+    def test_stage(self):
+        os.mkdir('stagedir')
+
+        runner = _runner.Runner(
+            part_properties={'override-stage': 'touch stage'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
+
+        runner.stage()
+
+        self.assertThat(os.path.join('stagedir', 'stage'), FileExists())
+
+    def test_builtin_function_from_stage(self):
+        os.mkdir('stagedir')
+
+        runner = _runner.Runner(
+            part_properties={'override-stage': 'snapcraftctl stage'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={'stage': _fake_stage})
+
+        runner.stage()
+
+        self.assertThat(os.path.join('stagedir', 'fake-stage'), FileExists())
+
 
 class RunnerFailureTestCase(unit.TestCase):
 
@@ -179,10 +234,11 @@ def test_failure_on_last_script_command_results_in_failure(self):
         """)
 
         runner = _runner.Runner(
-                part_properties={'build': script},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={})
+            part_properties={'build': script},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
 
         self.assertRaises(errors.ScriptletRunError, runner.build)
 
@@ -195,10 +251,11 @@ def test_failure_to_execute_mid_script_results_in_failure(self):
         """)
 
         runner = _runner.Runner(
-                part_properties={'build': script},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={})
+            part_properties={'build': script},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
 
         self.assertRaises(errors.ScriptletRunError, runner.build)
 
@@ -206,10 +263,11 @@ def test_snapcraftctl_no_alias_if_not_snap(self):
         os.mkdir('builddir')
 
         runner = _runner.Runner(
-                part_properties={'build': 'alias snapcraftctl 2> /dev/null'},
-                sourcedir='sourcedir',
-                builddir='builddir',
-                builtin_functions={})
+            part_properties={'build': 'alias snapcraftctl 2> /dev/null'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            builtin_functions={})
 
         self.assertRaises(errors.ScriptletRunError, runner.build)
 
@@ -229,6 +287,7 @@ def test_prepare_deprecation(self):
             part_properties={'prepare': 'foo'},
             sourcedir='sourcedir',
             builddir='builddir',
+            stagedir='stagedir',
             builtin_functions={})
 
         self.assertThat(self.fake_logger.output, Contains(
@@ -243,6 +302,7 @@ def test_build_deprecation(self):
             part_properties={'build': 'foo'},
             sourcedir='sourcedir',
             builddir='builddir',
+            stagedir='stagedir',
             builtin_functions={})
 
         self.assertThat(self.fake_logger.output, Contains(
@@ -257,6 +317,7 @@ def test_install_deprecation(self):
             part_properties={'install': 'foo'},
             sourcedir='sourcedir',
             builddir='builddir',
+            stagedir='stagedir',
             builtin_functions={})
 
         self.assertThat(self.fake_logger.output, Contains(
diff --git a/tests/unit/project_loader/test_config.py b/tests/unit/project_loader/test_config.py
--- a/tests/unit/project_loader/test_config.py
+++ b/tests/unit/project_loader/test_config.py
@@ -2322,6 +2322,10 @@ class OldConflictsWithNewScriptletTestCase(ValidationTestCase):
             'new_keyword': 'override-build',
             'new_value': ['test-override-build'],
         }),
+        ('override-stage', {
+            'new_keyword': 'override-stage',
+            'new_value': ['test-override-stage'],
+        }),
     ]
 
     scenarios = multiply_scenarios(