Repository: railwayapp/nixpacks
Pull request #644 (2022-10-31 19:50:33)

Consistent Ruby Caching
This PR fixes addresses a few things related to Ruby and Docker layer caching
- Install Ruby in a single Docker layer
- Noop dockerfile generation for a phase if it doesn't do anything
- Use `BTree(Map/Set)` to get determinstic sorting of phases when topological sorting phases

A fresh build of the [bullet_train](https://github.com/bullet-train-co/bullet_train) repo

![image](https://user-images.githubusercontent.com/3044853/199097650-8a212bff-e0f0-408b-a199-abafd2fd44c2.png)

The commands for the next build are executed in the same order and take advantage of the docker layer cache (0.0s on installing ruby)

![image](https://user-images.githubusercontent.com/3044853/199097753-c6fa6f7a-d8f7-40ba-b2dd-462099c24e44.png)

Fixes https://github.com/railwayapp/nixpacks/issues/641

flybayer: excellent! :)

Patch:
diff --git a/src/nixpacks/builder/docker/dockerfile_generation.rs b/src/nixpacks/builder/docker/dockerfile_generation.rs
index 76a2a30..c6e090c 100644
--- a/src/nixpacks/builder/docker/dockerfile_generation.rs
+++ b/src/nixpacks/builder/docker/dockerfile_generation.rs
@@ -320,6 +320,7 @@ impl DockerfileGenerator for StartPhase {
             None => {
                 formatdoc! {"
                   # start
+                  COPY . /app
                   {}
                 ",
                 start_cmd}
@@ -338,6 +339,10 @@ impl DockerfileGenerator for Phase {
         _output: &OutputDir,
         file_server_config: Option<FileServerConfig>,
     ) -> Result<String> {
+        if !self.runs_docker_commands() {
+            return Ok(format!("# {} phase\n# noop\n", self.get_name()));
+        }
+
         let phase = self;
 
         let cache_key = if !options.no_cache && !env.is_config_variable_truthy("NO_CACHE") {
diff --git a/src/nixpacks/images.rs b/src/nixpacks/images.rs
index 7d6eb73..d9063fc 100644
--- a/src/nixpacks/images.rs
+++ b/src/nixpacks/images.rs
@@ -1,2 +1,2 @@
-pub const DEFAULT_BASE_IMAGE: &str = "ghcr.io/railwayapp/nixpacks:debian-1666656625";
+pub const DEFAULT_BASE_IMAGE: &str = "ghcr.io/railwayapp/nixpacks:debian-1667261211";
 pub const DEBIAN_SLIM_IMAGE: &str = "debian:bullseye-slim";
diff --git a/src/nixpacks/plan/phase.rs b/src/nixpacks/plan/phase.rs
index 360eb10..1b81112 100644
--- a/src/nixpacks/plan/phase.rs
+++ b/src/nixpacks/plan/phase.rs
@@ -116,11 +116,18 @@ impl Phase {
         }
     }
 
+    /// Whether or not the phase uses Nix in any way
     pub fn uses_nix(&self) -> bool {
         !self.nix_pkgs.clone().unwrap_or_default().is_empty()
             || !self.nix_libs.clone().unwrap_or_default().is_empty()
     }
 
+    /// Whether or not the phase runs any docker commands
+    pub fn runs_docker_commands(&self) -> bool {
+        !self.cmds.clone().unwrap_or_default().is_empty()
+            || !self.paths.clone().unwrap_or_default().is_empty()
+    }
+
     pub fn depends_on_phase<S: Into<String>>(&mut self, name: S) {
         self.depends_on = Some(add_to_option_vec(self.depends_on.clone(), name.into()));
     }
diff --git a/src/nixpacks/plan/topological_sort.rs b/src/nixpacks/plan/topological_sort.rs
index 81c3323..f93853d 100644
--- a/src/nixpacks/plan/topological_sort.rs
+++ b/src/nixpacks/plan/topological_sort.rs
@@ -1,6 +1,5 @@
-use std::collections::{HashMap, HashSet};
-
 use anyhow::{bail, Result};
+use std::collections::{BTreeMap, BTreeSet};
 
 pub trait TopItem {
     fn get_name(&self) -> String;
@@ -11,7 +10,10 @@ pub fn topological_sort<T>(items: Vec<T>) -> Result<Vec<T>>
 where
     T: Clone + TopItem,
 {
-    let mut lookup = HashMap::<String, T>::new();
+    let mut items = items;
+    items.sort_by_cached_key(TopItem::get_name);
+
+    let mut lookup = BTreeMap::<String, T>::new();
     for item in items.clone() {
         if lookup.contains_key(&item.get_name()) {
             bail!("Multiple items with the same name: {}", item.get_name());
@@ -22,11 +24,11 @@ where
 
     // Reference of name -> dependent [items]
     let mut adj_list = items.iter().fold(
-        HashMap::<String, HashSet<String>>::new(),
+        BTreeMap::<String, BTreeSet<String>>::new(),
         |mut acc, item| {
             let n = item.get_name();
             if !acc.contains_key(&n) {
-                acc.insert(n.clone(), HashSet::new());
+                acc.insert(n.clone(), BTreeSet::new());
             }
 
             item.get_dependencies().iter().for_each(|dep| {
@@ -49,13 +51,13 @@ where
                     .count(),
             )
         })
-        .collect::<HashMap<String, usize>>();
+        .collect::<BTreeMap<String, usize>>();
 
     let mut result: Vec<T> = Vec::new();
 
     while !indegree.is_empty() {
         // Get the items with no dependencies
-        let (no_deps, new_indegree): (HashMap<String, usize>, HashMap<String, usize>) = indegree
+        let (no_deps, new_indegree): (BTreeMap<String, usize>, BTreeMap<String, usize>) = indegree
             .into_iter()
             .partition(|(_name, indgree)| *indgree == 0);
 
diff --git a/src/providers/ruby.rs b/src/providers/ruby.rs
index adbe190..04a6d4e 100644
--- a/src/providers/ruby.rs
+++ b/src/providers/ruby.rs
@@ -80,13 +80,16 @@ impl RubyProvider {
             setup.add_apt_pkgs(vec![String::from("libicu-dev")]);
         }
 
-        setup.add_cmd(
-            "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh".to_string(),
-        );
+        setup.add_cmd(format!(
+            "curl -sSL https://get.rvm.io | bash -s stable \
+            && . /etc/profile.d/rvm.sh \
+            && rvm install {ruby_version} \
+            && rvm --default use {ruby_version} \
+            && gem install {bundler_version}",
+            ruby_version = self.get_ruby_version(app)?,
+            bundler_version = self.get_bundler_version(app)
+        ));
 
-        setup.add_cmd(format!("rvm install {}", self.get_ruby_version(app)?));
-        setup.add_cmd(format!("rvm --default use {}", self.get_ruby_version(app)?));
-        setup.add_cmd(format!("gem install {}", self.get_bundler_version(app)));
         setup.add_cmd("echo 'source /usr/local/rvm/scripts/rvm' >> /root/.profile".to_string());
 
         Ok(Some(setup))


Test patch:
diff --git a/tests/snapshots/generate_plan_tests__ruby.snap b/tests/snapshots/generate_plan_tests__ruby.snap
index c60653d..32e5287 100644
--- a/tests/snapshots/generate_plan_tests__ruby.snap
+++ b/tests/snapshots/generate_plan_tests__ruby.snap
@@ -35,10 +35,7 @@ expression: plan
         "procps"
       ],
       "cmds": [
-        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh",
-        "rvm install ruby-3.1.2",
-        "rvm --default use ruby-3.1.2",
-        "gem install bundler:2.3.7",
+        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh && rvm install ruby-3.1.2 && rvm --default use ruby-3.1.2 && gem install bundler:2.3.7",
         "echo 'source /usr/local/rvm/scripts/rvm' >> /root/.profile"
       ],
       "onlyIncludeFiles": []
diff --git a/tests/snapshots/generate_plan_tests__ruby_execjs.snap b/tests/snapshots/generate_plan_tests__ruby_execjs.snap
index 0614030..6b2f3a5 100644
--- a/tests/snapshots/generate_plan_tests__ruby_execjs.snap
+++ b/tests/snapshots/generate_plan_tests__ruby_execjs.snap
@@ -58,10 +58,7 @@ expression: plan
         "procps"
       ],
       "cmds": [
-        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh",
-        "rvm install ruby-3.1.2",
-        "rvm --default use ruby-3.1.2",
-        "gem install bundler:2.3.7",
+        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh && rvm install ruby-3.1.2 && rvm --default use ruby-3.1.2 && gem install bundler:2.3.7",
         "echo 'source /usr/local/rvm/scripts/rvm' >> /root/.profile"
       ],
       "onlyIncludeFiles": []
diff --git a/tests/snapshots/generate_plan_tests__ruby_local_deps.snap b/tests/snapshots/generate_plan_tests__ruby_local_deps.snap
index 3f8acad..a8ac2f8 100644
--- a/tests/snapshots/generate_plan_tests__ruby_local_deps.snap
+++ b/tests/snapshots/generate_plan_tests__ruby_local_deps.snap
@@ -35,10 +35,7 @@ expression: plan
         "procps"
       ],
       "cmds": [
-        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh",
-        "rvm install ruby-3.1.2",
-        "rvm --default use ruby-3.1.2",
-        "gem install bundler:2.3.7",
+        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh && rvm install ruby-3.1.2 && rvm --default use ruby-3.1.2 && gem install bundler:2.3.7",
         "echo 'source /usr/local/rvm/scripts/rvm' >> /root/.profile"
       ],
       "onlyIncludeFiles": []
diff --git a/tests/snapshots/generate_plan_tests__ruby_rails_postgres.snap b/tests/snapshots/generate_plan_tests__ruby_rails_postgres.snap
index a59c8ad..bbd5465 100644
--- a/tests/snapshots/generate_plan_tests__ruby_rails_postgres.snap
+++ b/tests/snapshots/generate_plan_tests__ruby_rails_postgres.snap
@@ -46,10 +46,7 @@ expression: plan
         "libpq-dev"
       ],
       "cmds": [
-        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh",
-        "rvm install 3.1.2",
-        "rvm --default use 3.1.2",
-        "gem install bundler:2.3.7",
+        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh && rvm install 3.1.2 && rvm --default use 3.1.2 && gem install bundler:2.3.7",
         "echo 'source /usr/local/rvm/scripts/rvm' >> /root/.profile"
       ],
       "onlyIncludeFiles": []
diff --git a/tests/snapshots/generate_plan_tests__ruby_sinatra.snap b/tests/snapshots/generate_plan_tests__ruby_sinatra.snap
index 5e36ab7..d2ff165 100644
--- a/tests/snapshots/generate_plan_tests__ruby_sinatra.snap
+++ b/tests/snapshots/generate_plan_tests__ruby_sinatra.snap
@@ -35,10 +35,7 @@ expression: plan
         "procps"
       ],
       "cmds": [
-        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh",
-        "rvm install ruby-3.1.2",
-        "rvm --default use ruby-3.1.2",
-        "gem install bundler:2.3.7",
+        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh && rvm install ruby-3.1.2 && rvm --default use ruby-3.1.2 && gem install bundler:2.3.7",
         "echo 'source /usr/local/rvm/scripts/rvm' >> /root/.profile"
       ],
       "onlyIncludeFiles": []
diff --git a/tests/snapshots/generate_plan_tests__ruby_with_node.snap b/tests/snapshots/generate_plan_tests__ruby_with_node.snap
index 4eda5ce..b09673c 100644
--- a/tests/snapshots/generate_plan_tests__ruby_with_node.snap
+++ b/tests/snapshots/generate_plan_tests__ruby_with_node.snap
@@ -61,10 +61,7 @@ expression: plan
         "procps"
       ],
       "cmds": [
-        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh",
-        "rvm install ruby-3.1.2",
-        "rvm --default use ruby-3.1.2",
-        "gem install bundler:2.3.7",
+        "curl -sSL https://get.rvm.io | bash -s stable && . /etc/profile.d/rvm.sh && rvm install ruby-3.1.2 && rvm --default use ruby-3.1.2 && gem install bundler:2.3.7",
         "echo 'source /usr/local/rvm/scripts/rvm' >> /root/.profile"
       ],
       "onlyIncludeFiles": []