Repository: conan-io/conan

Issue:
Re-run system_requirements()
I am using the following scenario:

- I am running builds of our software on CI (Jenkins) which rely on some Conan packages
- The packages feature `system_requirements()` to install some system packages
- The builds run inside Docker container
- The Docker container mount a volume to which all Conan data is cached between CI builds (to save bandwidth and speed things up in general)

This is what happens:

- The first CI build is fine
  - Conan package is not installed
  - it gets downloaded
  - `system_requirements()` install system packages inside the Docker container
  - whole build runs fine
- On subsequent builds it fails
  - Conan package is already installed (because it is inside the mounted volume)
  - Conan does not invoke `system_requirements()`
  - System packages are missing in the Docker container
  - the build fails

Is there any way to re-run `system_requirements()` for an already installed package?



Discussion:
No, `system_requirements()` is run when it's needed to build the package or when it's downloaded, but if it's already there it's not run. this is a very special case where the cache is shared between different systems (docker and host).
  
Maybe we could think of a special config to force it, but it feels like a corner case, or an explicit way to call it, like local commands.
  
Thanks for your answer! Yes it is indeed a corner case. For the moment I will rethink my setup..
Well, I can provide you an easy workaround, that might work for your CI: Just remove ``system_reqs.txt`` files from your conan local cache. They are the ones representing that those system requirements have been installed.
Please note that currently (Conan 1.7.4) there are **directories** called `system_reqs` which have to be removed (**not** files `system_reqs.txt`) as a workaround.

But even when those directories are removed `system_requirements()` is not run again. @memsharded Is there anything more I have to do?
Let's take a look for the following release.
Thanks @lasote. I guess `_handle_system_requirements` should be called in `_handle_node_cache` too, maybe here: https://github.com/conan-io/conan/blob/719d66d1843a5180b665e81249fb34316844a34e/conans/client/installer.py#L306-L309

Resolving patch:
diff --git a/conans/client/installer.py b/conans/client/installer.py
--- a/conans/client/installer.py
+++ b/conans/client/installer.py
@@ -58,7 +58,8 @@ def __init__(self, conan_file, package_reference, client_cache, output, plugin_m
         self._plugin_manager = plugin_manager
 
         new_id = build_id(self._conan_file)
-        self.build_reference = PackageReference(self._conan_ref, new_id) if new_id else package_reference
+        self.build_reference = PackageReference(self._conan_ref,
+                                                new_id) if new_id else package_reference
         self.build_folder = self._client_cache.build(self.build_reference,
                                                      self._conan_file.short_paths)
         self.package_folder = self._client_cache.package(self._package_reference,
@@ -72,9 +73,6 @@ def prepare_build(self):
             return
 
         # build_id is not caching the build folder, so actually rebuild the package
-        _handle_system_requirements(self._conan_file, self._package_reference,
-                                    self._client_cache, self._out)
-
         export_folder = self._client_cache.export(self._conan_ref)
         export_source_folder = self._client_cache.export_sources(self._conan_ref,
                                                                  self._conan_file.short_paths)
@@ -277,7 +275,7 @@ def install(self, deps_graph, keep_build=False):
     def _build(self, nodes_by_level, deps_graph, keep_build, root_node):
         inverse_levels = {n: i for i, level in enumerate(deps_graph.inverse_levels()) for n in level}
 
-        processed_package_references = set()
+        processed_package_refs = set()
         for level in nodes_by_level:
             for node in level:
                 conan_ref, conan_file = node.conan_ref, node.conanfile
@@ -296,15 +294,16 @@ def _build(self, nodes_by_level, deps_graph, keep_build, root_node):
                 if workspace_package:
                     self._handle_node_workspace(node, workspace_package, inverse_levels, deps_graph)
                 else:
-                    self._handle_node_cache(node, package_id, keep_build, processed_package_references)
+                    package_ref = PackageReference(conan_ref, package_id)
+                    _handle_system_requirements(conan_file, package_ref, self._client_cache, output)
+                    self._handle_node_cache(node, package_ref, keep_build, processed_package_refs)
 
         # Finally, propagate information to root node (conan_ref=None)
         self._propagate_info(root_node, inverse_levels, deps_graph, self._out)
 
-    def _handle_node_cache(self, node, package_id, keep_build, processed_package_references):
+    def _handle_node_cache(self, node, package_ref, keep_build, processed_package_references):
         conan_ref, conan_file = node.conan_ref, node.conanfile
         output = ScopedOutput(str(conan_ref), self._out)
-        package_ref = PackageReference(conan_ref, package_id)
         package_folder = self._client_cache.package(package_ref, conan_file.short_paths)
 
         with self._client_cache.package_lock(package_ref):
@@ -314,7 +313,8 @@ def _handle_node_cache(self, node, package_id, keep_build, processed_package_ref
                 if node.binary == BINARY_BUILD:
                     self._build_package(node, package_ref, output, keep_build)
                 elif node.binary in (BINARY_UPDATE, BINARY_DOWNLOAD):
-                    self._download_package(conan_file, package_ref, output, package_folder, node.binary_remote)
+                    self._remote_manager.get_package(package_ref, package_folder,
+                                                     node.binary_remote, output, self._recorder)
                     if node.binary_remote != node.remote:
                         self._registry.set_ref(conan_ref, node.binary_remote.name)
                 elif node.binary == BINARY_CACHE:
@@ -355,11 +355,6 @@ def _handle_node_workspace(self, node, workspace_package, inverse_levels, deps_g
         copied_files = run_imports(conan_file, build_folder, output)
         report_copied_files(copied_files, output)
 
-    def _download_package(self, conan_file, package_reference, output, package_folder, remote):
-        self._remote_manager.get_package(package_reference, package_folder,
-                                         remote, output, self._recorder)
-        _handle_system_requirements(conan_file, package_reference, self._client_cache, output)
-
     def _build_package(self, node, package_ref, output, keep_build):
         conan_ref, conan_file = node.conan_ref, node.conanfile
 


Test patch:
diff --git a/conans/test/integration/system_reqs_test.py b/conans/test/integration/system_reqs_test.py
--- a/conans/test/integration/system_reqs_test.py
+++ b/conans/test/integration/system_reqs_test.py
@@ -23,6 +23,23 @@ def system_requirements(self):
 
 class SystemReqsTest(unittest.TestCase):
 
+    def force_system_reqs_rerun_test(self):
+        client = TestClient()
+        files = {'conanfile.py': base_conanfile.replace("%GLOBAL%", "")}
+        client.save(files)
+        client.run("create . user/channel")
+        self.assertIn("*+Running system requirements+*", client.user_io.out)
+        client.run("install Test/0.1@user/channel")
+        self.assertNotIn("*+Running system requirements+*", client.user_io.out)
+        ref = ConanFileReference.loads("Test/0.1@user/channel")
+        pfs = client.client_cache.packages(ref)
+        pid = os.listdir(pfs)[0]
+        reqs_file = client.client_cache.system_reqs_package(PackageReference(ref, pid))
+        os.unlink(reqs_file)
+        client.run("install Test/0.1@user/channel")
+        self.assertIn("*+Running system requirements+*", client.user_io.out)
+        self.assertTrue(os.path.exists(reqs_file))
+
     def local_system_requirements_test(self):
         client = TestClient()
         files = {'conanfile.py': base_conanfile.replace("%GLOBAL%", "")}