Repository: canonical/snapcraft

Issue:
Add pre-prime/prime/post-prime
Estimate: 1 day

We currently have three scriptlets that cover the build step only: prepare, build, and install. We want scriptlets around all lifecycle steps: pull, build, stage, and prime. Note that introducing an install step is not part of this roadmap.

There are some complications here that I want to point out. The stage and prime steps are completely handled by the core right now. As soon as we change that and provide the ability to override them, we must account for the possibility that files in stage will be changed from what they were in install, etc. Which means the stage dir must flow to the prime dir, which it does not do today (the file list is retrieved from stage, but the files themselves come from the parts’ install dirs).

Note that the stanzas for pull and build can be completed without such a redesign.
Add pre-prime/prime/post-prime
Estimate: 1 day

We currently have three scriptlets that cover the build step only: prepare, build, and install. We want scriptlets around all lifecycle steps: pull, build, stage, and prime. Note that introducing an install step is not part of this roadmap.

There are some complications here that I want to point out. The stage and prime steps are completely handled by the core right now. As soon as we change that and provide the ability to override them, we must account for the possibility that files in stage will be changed from what they were in install, etc. Which means the stage dir must flow to the prime dir, which it does not do today (the file list is retrieved from stage, but the files themselves come from the parts’ install dirs).

Note that the stanzas for pull and build can be completed without such a redesign.


Discussion:

Resolving patch:
diff --git a/snapcraft/cli/snapcraftctl/_runner.py b/snapcraft/cli/snapcraftctl/_runner.py
--- a/snapcraft/cli/snapcraftctl/_runner.py
+++ b/snapcraft/cli/snapcraftctl/_runner.py
@@ -46,22 +46,28 @@ def run(debug):
 
 @run.command()
 def pull():
-    """Run the 'pull' step of the calling part's plugin"""
+    """Run the 'pull' step of the calling part's lifecycle"""
     _call_function('pull')
 
 
 @run.command()
 def build():
-    """Run the 'build' step of the calling part's plugin"""
+    """Run the 'build' step of the calling part's lifecycle"""
     _call_function('build')
 
 
 @run.command()
 def stage():
-    """Run the 'stage' step of the calling part's plugin"""
+    """Run the 'stage' step of the calling part's lifecycle"""
     _call_function('stage')
 
 
+@run.command()
+def prime():
+    """Run the 'prime' step of the calling part's lifecycle"""
+    _call_function('prime')
+
+
 def _call_function(function_name, args=None):
     if not args:
         args = {}
diff --git a/snapcraft/internal/pluginhandler/__init__.py b/snapcraft/internal/pluginhandler/__init__.py
--- a/snapcraft/internal/pluginhandler/__init__.py
+++ b/snapcraft/internal/pluginhandler/__init__.py
@@ -103,10 +103,12 @@ def __init__(self, *, plugin, part_properties, project_options,
             sourcedir=self.plugin.sourcedir,
             builddir=self.plugin.build_basedir,
             stagedir=self.stagedir,
+            primedir=self.primedir,
             builtin_functions={
                 'pull': self._do_pull,
                 'build': self.plugin.build,
                 'stage': self._do_stage,
+                'prime': self._do_prime,
             })
 
         self._migrate_state_file()
@@ -543,6 +545,14 @@ def clean_stage(self, project_staged_state, hint=''):
     def prime(self, force=False) -> None:
         self.makedirs()
         self.notify_part_progress('Priming')
+        self._runner.prime()
+
+        # Only mark this step done if _do_prime() didn't run, in which case
+        # we have no files, directories, or dependency paths to track.
+        if self.is_clean('prime'):
+            self.mark_prime_done(set(), set(), set())
+
+    def _do_prime(self) -> None:
         snap_files, snap_dirs = self.migratable_fileset_for('prime')
         _migrate_files(snap_files, snap_dirs, self.stagedir, self.primedir)
 
diff --git a/snapcraft/internal/pluginhandler/_runner.py b/snapcraft/internal/pluginhandler/_runner.py
--- a/snapcraft/internal/pluginhandler/_runner.py
+++ b/snapcraft/internal/pluginhandler/_runner.py
@@ -38,24 +38,27 @@ class Runner:
     # https://github.com/python/typing/issues/259 which is fixed in Python
     # 3.5.3.
     def __init__(self, *, part_properties: Dict[str, Any], sourcedir: str,
-                 builddir: str, stagedir: str,
+                 builddir: str, stagedir: str, primedir: str,
                  builtin_functions: 'Dict[str, Callable[..., None]]') -> None:
         """Create a new Runner.
         :param dict part_properties: YAML properties set for this part.
         :param str sourcedir: The source directory for this part.
         :param str builddir: The build directory for this part.
         :param str stagedir: The staging area.
+        :param str primedir: The priming area.
         :param dict builtin_functions: Dict of builtin function names to
                                        actual callables.
         """
         self._sourcedir = sourcedir
         self._builddir = builddir
         self._stagedir = stagedir
+        self._primedir = primedir
         self._builtin_functions = builtin_functions
 
         self._override_pull_scriptlet = part_properties.get('override-pull')
         self._override_build_scriptlet = part_properties.get('override-build')
         self._override_stage_scriptlet = part_properties.get('override-stage')
+        self._override_prime_scriptlet = part_properties.get('override-prime')
 
         # These are all deprecated
         self._prepare_scriptlet = part_properties.get('prepare')
@@ -106,6 +109,13 @@ def stage(self) -> None:
                 'override-stage', self._override_stage_scriptlet,
                 self._stagedir)
 
+    def prime(self) -> None:
+        """Run override-prime scriptlet."""
+        if self._override_prime_scriptlet:
+            self._run_scriptlet(
+                'override-prime', self._override_prime_scriptlet,
+                self._primedir)
+
     def _run_scriptlet(self, scriptlet_name: str, scriptlet: str,
                        workdir: str) -> None:
         with tempfile.TemporaryDirectory() as tempdir:


Test patch:
diff --git a/tests/integration/general/test_scriptlets.py b/tests/integration/general/test_scriptlets.py
--- a/tests/integration/general/test_scriptlets.py
+++ b/tests/integration/general/test_scriptlets.py
@@ -117,3 +117,42 @@ def test_override_stage(self):
             FileExists())
         self.assertThat(
             os.path.join(self.stage_dir, 'file2'), Not(FileExists()))
+
+    def test_override_prime(self):
+        self.run_snapcraft('prime', 'scriptlet-override-prime')
+
+        installdir = os.path.join(
+            self.parts_dir, 'override-prime-scriptlet-test', 'install')
+
+        # Assert that the before/after prime files aren't placed in the
+        # installdir, although the file is.
+        self.assertThat(os.path.join(installdir, 'file'), FileExists())
+        self.assertThat(
+            os.path.join(installdir, 'before-prime'), Not(FileExists()))
+        self.assertThat(
+            os.path.join(installdir, 'after-prime'), Not(FileExists()))
+
+        # Assert that the before/after prime files aren't placed in the
+        # stagedir, although the file is.
+        self.assertThat(os.path.join(self.stage_dir, 'file'), FileExists())
+        self.assertThat(
+            os.path.join(self.stage_dir, 'before-prime'), Not(FileExists()))
+        self.assertThat(
+            os.path.join(self.stage_dir, 'after-prime'), Not(FileExists()))
+
+        self.assertThat(os.path.join(self.prime_dir, 'file'), FileExists())
+        self.assertThat(
+            os.path.join(self.prime_dir, 'before-prime'), FileExists())
+        self.assertThat(
+            os.path.join(self.prime_dir, 'after-prime'), FileExists())
+
+        # Also assert that, while file2 was installed and staged, it wasn't
+        # primed
+        self.assertThat(
+            os.path.join(
+                self.parts_dir, 'override-prime-do-nothing', 'install',
+                'file2'),
+            FileExists())
+        self.assertThat(os.path.join(self.stage_dir, 'file2'), FileExists())
+        self.assertThat(
+            os.path.join(self.prime_dir, 'file2'), Not(FileExists()))
diff --git a/tests/integration/snaps/scriptlet-override-prime/part1/file b/tests/integration/snaps/scriptlet-override-prime/part1/file
new file mode 100644
--- /dev/null
+++ b/tests/integration/snaps/scriptlet-override-prime/part1/file
@@ -0,0 +1 @@
+I'm a nice file
diff --git a/tests/integration/snaps/scriptlet-override-prime/part2/file2 b/tests/integration/snaps/scriptlet-override-prime/part2/file2
new file mode 100644
--- /dev/null
+++ b/tests/integration/snaps/scriptlet-override-prime/part2/file2
@@ -0,0 +1 @@
+I'm a nicer file
diff --git a/tests/integration/snaps/scriptlet-override-prime/snap/snapcraft.yaml b/tests/integration/snaps/scriptlet-override-prime/snap/snapcraft.yaml
new file mode 100644
--- /dev/null
+++ b/tests/integration/snaps/scriptlet-override-prime/snap/snapcraft.yaml
@@ -0,0 +1,23 @@
+name: override-prime-scriptlet-test
+version: '0.1'
+summary: Runs the override-prime scriptlet for a part
+description: |
+  Runs the shell script defined in `override-prime` instead of normal prime.
+grade: devel
+confinement: devmode
+
+parts:
+  override-prime-scriptlet-test:
+    plugin: dump
+    source: part1/
+    override-prime: |
+      touch before-prime
+      snapcraftctl prime
+      touch after-prime
+
+  override-prime-do-nothing:
+    plugin: dump
+    source: part2/
+    override-prime: |
+      # Completely skip prime
+      exit 0
diff --git a/tests/unit/pluginhandler/test_runner.py b/tests/unit/pluginhandler/test_runner.py
--- a/tests/unit/pluginhandler/test_runner.py
+++ b/tests/unit/pluginhandler/test_runner.py
@@ -42,6 +42,10 @@ def _fake_stage():
     open(os.path.join('stagedir', 'fake-stage'), 'w').close()
 
 
+def _fake_prime():
+    open(os.path.join('primedir', 'fake-prime'), 'w').close()
+
+
 class RunnerTestCase(unit.TestCase):
 
     def test_pull(self):
@@ -52,6 +56,7 @@ def test_pull(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         runner.pull()
@@ -66,6 +71,7 @@ def test_builtin_function_from_pull(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={'pull': _fake_pull})
 
         runner.pull()
@@ -80,6 +86,7 @@ def test_prepare(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         runner.prepare()
@@ -94,6 +101,7 @@ def test_builtin_function_from_prepare(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={'build': _fake_build})
 
         runner.prepare()
@@ -112,6 +120,7 @@ def test_snapcraftctl_alias_if_snap(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         with mock.patch('os.path.exists', return_value=True):
@@ -132,6 +141,7 @@ def test_old_build(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         runner.build()
@@ -146,6 +156,7 @@ def test_build(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         runner.build()
@@ -160,6 +171,7 @@ def test_builtin_function_from_build(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={'build': _fake_build})
 
         runner.build()
@@ -174,6 +186,7 @@ def test_install(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         runner.install()
@@ -188,6 +201,7 @@ def test_builtin_function_from_install(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={'build': _fake_build})
 
         runner.install()
@@ -202,6 +216,7 @@ def test_stage(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         runner.stage()
@@ -216,12 +231,43 @@ def test_builtin_function_from_stage(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={'stage': _fake_stage})
 
         runner.stage()
 
         self.assertThat(os.path.join('stagedir', 'fake-stage'), FileExists())
 
+    def test_prime(self):
+        os.mkdir('primedir')
+
+        runner = _runner.Runner(
+            part_properties={'override-prime': 'touch prime'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            primedir='primedir',
+            builtin_functions={})
+
+        runner.prime()
+
+        self.assertThat(os.path.join('primedir', 'prime'), FileExists())
+
+    def test_builtin_function_from_prime(self):
+        os.mkdir('primedir')
+
+        runner = _runner.Runner(
+            part_properties={'override-prime': 'snapcraftctl prime'},
+            sourcedir='sourcedir',
+            builddir='builddir',
+            stagedir='stagedir',
+            primedir='primedir',
+            builtin_functions={'prime': _fake_prime})
+
+        runner.prime()
+
+        self.assertThat(os.path.join('primedir', 'fake-prime'), FileExists())
+
 
 class RunnerFailureTestCase(unit.TestCase):
 
@@ -238,6 +284,7 @@ def test_failure_on_last_script_command_results_in_failure(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         self.assertRaises(errors.ScriptletRunError, runner.build)
@@ -255,6 +302,7 @@ def test_failure_to_execute_mid_script_results_in_failure(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         self.assertRaises(errors.ScriptletRunError, runner.build)
@@ -267,6 +315,7 @@ def test_snapcraftctl_no_alias_if_not_snap(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         self.assertRaises(errors.ScriptletRunError, runner.build)
@@ -288,6 +337,7 @@ def test_prepare_deprecation(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         self.assertThat(self.fake_logger.output, Contains(
@@ -303,6 +353,7 @@ def test_build_deprecation(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         self.assertThat(self.fake_logger.output, Contains(
@@ -318,6 +369,7 @@ def test_install_deprecation(self):
             sourcedir='sourcedir',
             builddir='builddir',
             stagedir='stagedir',
+            primedir='primedir',
             builtin_functions={})
 
         self.assertThat(self.fake_logger.output, Contains(
diff --git a/tests/unit/project_loader/test_config.py b/tests/unit/project_loader/test_config.py
--- a/tests/unit/project_loader/test_config.py
+++ b/tests/unit/project_loader/test_config.py
@@ -2326,6 +2326,10 @@ class OldConflictsWithNewScriptletTestCase(ValidationTestCase):
             'new_keyword': 'override-stage',
             'new_value': ['test-override-stage'],
         }),
+        ('override-prime', {
+            'new_keyword': 'override-prime',
+            'new_value': ['test-override-prime'],
+        }),
     ]
 
     scenarios = multiply_scenarios(