Repository: railwayapp/nixpacks
Pull request #1116 (2024-06-09 16:30:12)

feat: asdf support
Follow-up on https://github.com/railwayapp/nixpacks/pull/1062

> Instead of asserting here, could we instead just use the major/minor versions and warn that specifying the exact patch version is not supported

This is fixed!


github-actions[bot]: This pull request is stale because it has been open 10 days with no activity. Remove stale label or comment or this will be closed in 5 days
iloveitaly: Friendly reminder on this pull request! Please provide an update or this will be closed soon.
github-actions[bot]: This pull request is stale because it has been open 10 days with no activity. Remove stale label or comment or this will be closed in 5 days

Patch:
diff --git a/examples/python-asdf-poetry/.tool-versions b/examples/python-asdf-poetry/.tool-versions
new file mode 100644
index 0000000..8a8ea6f
--- /dev/null
+++ b/examples/python-asdf-poetry/.tool-versions
@@ -0,0 +1,2 @@
+python 3.12.3
+poetry 1.8.2
\ No newline at end of file
diff --git a/examples/python-asdf-poetry/main.py b/examples/python-asdf-poetry/main.py
new file mode 100644
index 0000000..109925d
--- /dev/null
+++ b/examples/python-asdf-poetry/main.py
@@ -0,0 +1,9 @@
+import sys
+import subprocess
+
+print(sys.version)
+
+poetry_version = subprocess.run(
+    ["poetry", "--version"], capture_output=True, text=True
+).stdout.strip()
+print(poetry_version)
diff --git a/examples/python-asdf-poetry/poetry.lock b/examples/python-asdf-poetry/poetry.lock
new file mode 100644
index 0000000..20aad95
--- /dev/null
+++ b/examples/python-asdf-poetry/poetry.lock
@@ -0,0 +1,129 @@
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+
+[[package]]
+name = "atomicwrites"
+version = "1.4.1"
+description = "Atomic file writes."
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+    {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"},
+]
+
+[[package]]
+name = "attrs"
+version = "23.2.0"
+description = "Classes Without Boilerplate"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"},
+    {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"},
+]
+
+[package.extras]
+cov = ["attrs[tests]", "coverage[toml] (>=5.3)"]
+dev = ["attrs[tests]", "pre-commit"]
+docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"]
+tests = ["attrs[tests-no-zope]", "zope-interface"]
+tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"]
+tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"]
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+description = "Cross-platform colored terminal text."
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+files = [
+    {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
+    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
+]
+
+[[package]]
+name = "more-itertools"
+version = "10.2.0"
+description = "More routines for operating on iterables, beyond itertools"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "more-itertools-10.2.0.tar.gz", hash = "sha256:8fccb480c43d3e99a00087634c06dd02b0d50fbf088b380de5a41a015ec239e1"},
+    {file = "more_itertools-10.2.0-py3-none-any.whl", hash = "sha256:686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684"},
+]
+
+[[package]]
+name = "packaging"
+version = "24.0"
+description = "Core utilities for Python packages"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"},
+    {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"},
+]
+
+[[package]]
+name = "pluggy"
+version = "0.13.1"
+description = "plugin and hook calling mechanisms for python"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+    {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
+    {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
+]
+
+[package.extras]
+dev = ["pre-commit", "tox"]
+
+[[package]]
+name = "py"
+version = "1.11.0"
+description = "library with cross-python path, ini-parsing, io, code, log facilities"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+files = [
+    {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
+    {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
+]
+
+[[package]]
+name = "pytest"
+version = "5.4.3"
+description = "pytest: simple powerful testing with Python"
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
+    {file = "pytest-5.4.3.tar.gz", hash = "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"},
+]
+
+[package.dependencies]
+atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""}
+attrs = ">=17.4.0"
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+more-itertools = ">=4.0.0"
+packaging = "*"
+pluggy = ">=0.12,<1.0"
+py = ">=1.5.0"
+wcwidth = "*"
+
+[package.extras]
+checkqa-mypy = ["mypy (==v0.761)"]
+testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
+
+[[package]]
+name = "wcwidth"
+version = "0.2.13"
+description = "Measures the displayed width of unicode strings in a terminal"
+optional = false
+python-versions = "*"
+files = [
+    {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"},
+    {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"},
+]
+
+[metadata]
+lock-version = "2.0"
+python-versions = "^3.10"
+content-hash = "491a37bfc34281cb87606e36384453db66e94ff9c309c7a34be6f9380c5e9782"
diff --git a/examples/python-asdf-poetry/pyproject.toml b/examples/python-asdf-poetry/pyproject.toml
new file mode 100644
index 0000000..d78db5e
--- /dev/null
+++ b/examples/python-asdf-poetry/pyproject.toml
@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "python-asdf-poetry"
+version = "0.1.0"
+description = ""
+authors = ["iloveitaly"]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+
+[tool.poetry.dev-dependencies]
+pytest = "^5.2"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
diff --git a/src/nixpacks/asdf.rs b/src/nixpacks/asdf.rs
new file mode 100644
index 0000000..d7a2e1d
--- /dev/null
+++ b/src/nixpacks/asdf.rs
@@ -0,0 +1,45 @@
+use regex::Regex;
+use std::collections::HashMap;
+
+pub fn parse_tool_versions_content(file_content: &str) -> HashMap<String, String> {
+    let re = Regex::new(r"\s+").unwrap();
+    file_content
+        .lines()
+        .map(str::trim)
+        .filter(|line| !line.starts_with('#'))
+        .filter_map(|line| {
+            let parts: Vec<&str> = re.splitn(line, 2).collect();
+            if parts.len() == 2 {
+                Some((parts[0].to_string(), parts[1].to_string()))
+            } else {
+                None
+            }
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_tool_versions_content() {
+        let file_content = "\
+# This is a comment
+python 3.10.4
+ poetry  1.7.1
+# exact ruby version required for homebrew
+ruby\t3.1.4
+direnv  2.32.3 ";
+        let versions = parse_tool_versions_content(file_content);
+
+        let expected = HashMap::from([
+            ("direnv".to_string(), "2.32.3".to_string()),
+            ("python".to_string(), "3.10.4".to_string()),
+            ("ruby".to_string(), "3.1.4".to_string()),
+            ("poetry".to_string(), "1.7.1".to_string()),
+        ]);
+
+        assert_eq!(versions, expected);
+    }
+}
diff --git a/src/nixpacks/mod.rs b/src/nixpacks/mod.rs
index dd34516..318dc79 100644
--- a/src/nixpacks/mod.rs
+++ b/src/nixpacks/mod.rs
@@ -1,4 +1,5 @@
 pub mod app;
+pub mod asdf;
 pub mod builder;
 pub mod environment;
 mod files;
diff --git a/src/providers/python.rs b/src/providers/python.rs
index 00aca4b..7bb643e 100644
--- a/src/providers/python.rs
+++ b/src/providers/python.rs
@@ -2,6 +2,7 @@ use crate::{
     chain,
     nixpacks::{
         app::App,
+        asdf::parse_tool_versions_content,
         environment::{Environment, EnvironmentVariables},
         plan::{
             phase::{Phase, StartPhase},
@@ -73,12 +74,24 @@ impl Provider for PythonProvider {
         plan.add_variables(PythonProvider::default_python_environment_variables());
 
         if app.includes_file("poetry.lock") {
+            let mut version = POETRY_VERSION.to_string();
+
+            if app.includes_file(".tool-versions") {
+                let file_content = &app.read_file(".tool-versions")?;
+
+                if let Some(poetry_version) =
+                    PythonProvider::parse_tool_versions_poetry_version(file_content)?
+                {
+                    println!("Using poetry version from .tool-versions: {poetry_version}");
+                    version = poetry_version;
+                }
+            }
+
             plan.add_variables(EnvironmentVariables::from([(
                 "NIXPACKS_POETRY_VERSION".to_string(),
-                POETRY_VERSION.to_string(),
+                version,
             )]));
         }
-
         if app.includes_file("pdm.lock") {
             plan.add_variables(EnvironmentVariables::from([(
                 "NIXPACKS_PDM_VERSION".to_string(),
@@ -302,6 +315,32 @@ impl PythonProvider {
             .map(|m| m.get(2).unwrap().as_str().to_string()))
     }
 
+    fn parse_tool_versions_python_version(file_content: &str) -> Result<Option<String>> {
+        let asdf_versions = parse_tool_versions_content(file_content);
+
+        // the python version can only specify a major.minor version right now, and not a patch version
+        // however, in asdf a patch version is specified, so we need to strip it
+        Ok(asdf_versions.get("python").map(|s| {
+            let parts: Vec<&str> = s.split('.').collect();
+
+            if parts.len() == 3 {
+                // this is the expected result, but will be unexpected to users
+                println!("Patch version detected in .tool-versions, but not supported in nixpkgs.");
+            } else if parts.len() == 2 {
+                println!("Expected a version string in the format x.y.z from .tool-versions");
+            } else {
+                println!("Could not find a version string in the format x.y.z or x.y from .tool-versions");
+            }
+
+            format!("{}.{}", parts[0], parts[1])
+        }))
+    }
+
+    fn parse_tool_versions_poetry_version(file_content: &str) -> Result<Option<String>> {
+        let asdf_versions = parse_tool_versions_content(file_content);
+        Ok(asdf_versions.get("poetry").cloned())
+    }
+
     fn default_python_environment_variables() -> EnvironmentVariables {
         let python_variables = vec![
             ("PYTHONFAULTHANDLER", "1"),
@@ -342,6 +381,9 @@ impl PythonProvider {
         } else if app.includes_file("Pipfile") {
             let file_content = &app.read_file("Pipfile")?;
             custom_version = PythonProvider::parse_pipfile_python_version(file_content)?;
+        } else if app.includes_file(".tool-versions") {
+            let file_content = &app.read_file(".tool-versions")?;
+            custom_version = PythonProvider::parse_tool_versions_python_version(file_content)?;
         }
 
         // If it's still none, return default


Test patch:
diff --git a/tests/docker_run_tests.rs b/tests/docker_run_tests.rs
index a7f915d..7ed1c8a 100644
--- a/tests/docker_run_tests.rs
+++ b/tests/docker_run_tests.rs
@@ -692,6 +692,15 @@ async fn test_python_2() {
     assert!(output.contains("Hello from Python 2"));
 }
 
+#[tokio::test]
+async fn test_python_asdf_poetry() {
+    let name = simple_build("./examples/python-asdf-poetry").await;
+    let output = run_image(&name, None).await;
+
+    assert!(output.contains("3.12.3"), "{}", output);
+    assert!(output.contains("Poetry (version 1.8.2)"), "{}", output);
+}
+
 #[tokio::test]
 async fn test_django() {
     // Create the network
diff --git a/tests/snapshots/generate_plan_tests__python_asdf_poetry.snap b/tests/snapshots/generate_plan_tests__python_asdf_poetry.snap
new file mode 100644
index 0000000..a1af3f3
--- /dev/null
+++ b/tests/snapshots/generate_plan_tests__python_asdf_poetry.snap
@@ -0,0 +1,52 @@
+---
+source: tests/generate_plan_tests.rs
+expression: plan
+---
+{
+  "providers": [],
+  "buildImage": "[build_image]",
+  "variables": {
+    "NIXPACKS_METADATA": "python,poetry",
+    "NIXPACKS_POETRY_VERSION": "1.8.2",
+    "PIP_DEFAULT_TIMEOUT": "100",
+    "PIP_DISABLE_PIP_VERSION_CHECK": "1",
+    "PIP_NO_CACHE_DIR": "1",
+    "PYTHONDONTWRITEBYTECODE": "1",
+    "PYTHONFAULTHANDLER": "1",
+    "PYTHONHASHSEED": "random",
+    "PYTHONUNBUFFERED": "1"
+  },
+  "phases": {
+    "install": {
+      "name": "install",
+      "dependsOn": [
+        "setup"
+      ],
+      "cmds": [
+        "python -m venv --copies /opt/venv && . /opt/venv/bin/activate && pip install poetry==$NIXPACKS_POETRY_VERSION && poetry install --no-dev --no-interaction --no-ansi"
+      ],
+      "cacheDirectories": [
+        "/root/.cache/pip"
+      ],
+      "paths": [
+        "/opt/venv/bin"
+      ]
+    },
+    "setup": {
+      "name": "setup",
+      "nixPkgs": [
+        "python312",
+        "gcc"
+      ],
+      "nixLibs": [
+        "zlib",
+        "stdenv.cc.cc.lib"
+      ],
+      "nixOverlays": [],
+      "nixpkgsArchive": "[archive]"
+    }
+  },
+  "start": {
+    "cmd": "python main.py"
+  }
+}