Repository: conda/conda-build

Issue:
Add a load_pyproject_toml_data() template function
The Bokeh project currently uses the `install_requires` of a `setup.py` to specify the single source of truth for runtime dependencies. We use the existing `load_setup_py_data()` function that conda-build supplies to pull those dependencies into conda recipes automatically. 

We would like to move build information in to a `pyproject.toml` file, but also still keep ability to automatically pull information in to our recipes without manually duplicating it. I propose adding a `load_pyproject_toml_data()` alongside the existing `load_setup_py_data()`. 

Note: I did also find https://github.com/conda/conda-build/issues/3507 which seems possibly related, but I think the proposal here is narrower in scope, and aligned to a different use-case. 




Discussion:
Sounds good, but I don't think anyone will have time to do this. A PR would be appreciated though.
I'm not sure I'd be able to sign the CLA
Ok thanks Brian. Is it worth asking, just so we know for the future whether it is allowed?
@mingwandroid We would have to execute the Corporate CLA mentioned in para 4, not the individual CLA at https://conda.io/en/latest/contributing.html#conda-contributor-license-agreement Can you provide the CCLA for me to send to NV legal?
ping @mingwandroid any thoughts on the above?
Hi Brian.
My health has deteriorated and i don't think I'll get round to this or much else. Ping @chenghlee 
Take care of yourself!
> @mingwandroid We would have to execute the Corporate CLA mentioned in para 4, not the individual CLA at https://conda.io/en/latest/contributing.html#conda-contributor-license-agreement Can you provide the CCLA for me to send to NV legal?

Hey Bryan, I'll try to find this document for you.
@bryevdv: here's a [direct link](https://anaconda.na2.echosign.com/public/esignWidget?wid=CBFCIBAA3AAABLblqZhAzPD8JNseGxXBNSY55tQ9ZE65FLEa2B-rUNAEYEjfxph-KGE3uSbu0fvR5cTRRCnY*&hosted=false) to the updated CLA that covers both individuals and organizations.
I'll pass it along to NV legal, I can't guarantee any specific outcome form them and I also don't know when I might be able to work on this personally.


Resolving patch:
diff --git a/conda_build/config.py b/conda_build/config.py
--- a/conda_build/config.py
+++ b/conda_build/config.py
@@ -780,12 +780,6 @@ def work_dir(self):
         """Where the source for the build is extracted/copied to."""
         path = join(self.build_folder, 'work')
         _ensure_dir(path)
-        # if os.path.isdir(path):
-        #     lst = [fn for fn in os.listdir(path) if not fn.startswith('.')]
-        #     if len(lst) == 1:
-        #         dir_path = join(path, lst[0])
-        #         if os.path.isdir(dir_path):
-        #             return dir_path
         return path
 
     @property
diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py
--- a/conda_build/jinja_context.py
+++ b/conda_build/jinja_context.py
@@ -1,11 +1,17 @@
 from functools import partial
+from io import StringIO
 import json
 import os
+import pathlib
 import re
 import time
 import datetime
+from typing import IO, Any, Optional
+from warnings import warn
 
 import jinja2
+import toml
+import yaml
 
 from .conda_interface import PY3
 from .environ import get_dict as get_environ
@@ -18,6 +24,9 @@
 from . import _load_setup_py_data
 
 
+log = get_logger(__name__)
+
+
 class UndefinedNeverFail(jinja2.Undefined):
     """
     A class for Undefined jinja variables.
@@ -131,7 +140,6 @@ def load_setup_py_data(m, setup_file='setup.py', from_recipe_dir=False, recipe_d
             pass
         except ImportError as e:
             if permit_undefined_jinja:
-                log = get_logger(__name__)
                 log.debug("Reading setup.py failed due to missing modules.  This is probably OK, "
                           "since it may succeed in later passes.  Watch for incomplete recipe "
                           "info, though.")
@@ -145,9 +153,11 @@ def load_setup_py_data(m, setup_file='setup.py', from_recipe_dir=False, recipe_d
 
 def load_setuptools(m, setup_file='setup.py', from_recipe_dir=False, recipe_dir=None,
                     permit_undefined_jinja=True):
-    log = get_logger(__name__)
-    log.warn("Deprecation notice: the load_setuptools function has been renamed to "
-             "load_setup_py_data.  load_setuptools will be removed in a future release.")
+    warn(
+        "conda_build.jinja_context.load_setuptools is pending deprecation in a future release. "
+        "Use conda_build.jinja_context.load_setup_py_data instead.",
+        PendingDeprecationWarning,
+    )
     return load_setup_py_data(m, setup_file=setup_file, from_recipe_dir=from_recipe_dir,
                               recipe_dir=recipe_dir, permit_undefined_jinja=permit_undefined_jinja)
 
@@ -159,42 +169,36 @@ def load_npm():
         return json.load(pkg)
 
 
-def load_file_regex(config, load_file, regex_pattern, from_recipe_dir=False,
-                    recipe_dir=None, permit_undefined_jinja=True):
-    match = False
-    log = get_logger(__name__)
-
-    cd_to_work = False
+def _find_file(file_name: str, from_recipe_dir: bool, recipe_dir: str, config) -> str:
+    """ Get the path to the given file which may be in the work_dir
+    or in the recipe_dir.
 
-    if from_recipe_dir and recipe_dir:
-        load_file = os.path.abspath(os.path.join(recipe_dir, load_file))
+    Note, the returned file name may not exist.
+    """
+    if os.path.isabs(file_name):
+        path = file_name
+    elif from_recipe_dir and recipe_dir:
+        path = os.path.abspath(os.path.join(recipe_dir, file_name))
     elif os.path.exists(config.work_dir):
-        cd_to_work = True
-        cwd = os.getcwd()
-        os.chdir(config.work_dir)
-        if not os.path.isabs(load_file):
-            load_file = os.path.join(config.work_dir, load_file)
-    else:
-        message = ("Did not find {} file in manually specified location, and source "
-                  "not downloaded yet.".format(load_file))
-        if permit_undefined_jinja:
-            log.debug(message)
-            return {}
-        else:
-            raise RuntimeError(message)
+        path = os.path.join(config.work_dir, file_name)
 
-    if os.path.isfile(load_file):
-        with open(load_file) as lfile:
-            match = re.search(regex_pattern, lfile.read())
-    else:
-        if not permit_undefined_jinja:
-            raise TypeError(f'{load_file} is not a file that can be read')
+    if not os.path.isfile(path):
+        raise FileNotFoundError(f"No such file: {file_name!r}")
+    return path
 
-    # Reset the working directory
-    if cd_to_work:
-        os.chdir(cwd)
 
-    return match if match else None
+def load_file_regex(config, load_file, regex_pattern, from_recipe_dir=False,
+                    recipe_dir=None, permit_undefined_jinja=True):
+    try:
+        load_file = _find_file(load_file, from_recipe_dir, recipe_dir, config)
+    except FileNotFoundError as e:
+        if permit_undefined_jinja:
+            log.debug(e)
+            return None
+        raise
+    else:
+        with open(load_file) as lfile:
+            return re.search(regex_pattern, lfile.read())
 
 
 cached_env_dependencies = {}
@@ -493,6 +497,59 @@ def resolved_packages(m, env, permit_undefined_jinja=False,
     return package_names
 
 
+_file_parsers = {
+    "json": json.load,
+    "yaml": yaml.safe_load,
+    "yml": yaml.safe_load,
+    "toml": toml.load,
+}
+
+
+def _load_data(stream: IO, fmt: str, *args, **kwargs) -> Any:
+    try:
+        load = _file_parsers[fmt.lower()]
+    except KeyError:
+        raise ValueError(
+            f"Unknown file format: {fmt}. Allowed formats: {list(_file_parsers.keys())}"
+        )
+    else:
+        return load(stream, *args, **kwargs)
+
+
+def load_file_data(filename: str, fmt: Optional[str] = None, *args, config=None,
+                   from_recipe_dir=False, recipe_dir=None, permit_undefined_jinja=True,
+                   **kwargs):
+    """Loads a file and returns the parsed data.
+
+    For example to load file data from a JSON file, you can use any of:
+
+        load_file_data("my_file.json")
+        load_file_data("my_json_file_without_ext", "json")
+    """
+    try:
+        file_path = _find_file(filename, from_recipe_dir, recipe_dir, config)
+    except FileNotFoundError as e:
+        if permit_undefined_jinja:
+            log.debug(e)
+            return {}
+        raise
+    else:
+        with open(file_path) as f:
+            return _load_data(f, fmt or pathlib.Path(filename).suffix.lstrip("."), *args, **kwargs)
+
+
+def load_str_data(string: str, fmt: str, *args, **kwargs):
+    """Parses data from a string
+
+    For example to load string data, you can use any of:
+
+        load_str_data("[{"name": "foo", "val": "bar"}, {"name": "biz", "val": "buz"}]", "json")
+        load_str_data("[tool.poetry]\\nname = 'foo'", "toml")
+        load_str_data("name: foo\\nval: bar", "yaml")
+    """
+    return _load_data(StringIO(string), fmt, *args, **kwargs)
+
+
 def context_processor(initial_metadata, recipe_dir, config, permit_undefined_jinja,
                       allow_no_other_outputs=False, bypass_env_check=False, skip_build_id=False,
                       variant=None):
@@ -516,6 +573,9 @@ def context_processor(initial_metadata, recipe_dir, config, permit_undefined_jin
         load_npm=load_npm,
         load_file_regex=partial(load_file_regex, config=config, recipe_dir=recipe_dir,
                                 permit_undefined_jinja=permit_undefined_jinja),
+        load_file_data=partial(load_file_data, config=config, recipe_dir=recipe_dir,
+                               permit_undefined_jinja=permit_undefined_jinja),
+        load_str_data=load_str_data,
         installed=get_installed_packages(os.path.join(config.host_prefix, 'conda-meta')),
         pin_compatible=partial(pin_compatible, initial_metadata,
                                permit_undefined_jinja=permit_undefined_jinja,
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -18,6 +18,7 @@
     "beautifulsoup4",
     "chardet",
     "pytz",
+    "toml",
     "tqdm",
     "psutil",
     "six",


Test patch:
diff --git a/tests/requirements.txt b/tests/requirements.txt
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -15,3 +15,4 @@ pytest
 pytest-cov
 pytest-mock
 python-libarchive-c
+toml
diff --git a/tests/test-recipes/metadata/jinja_load_toml/meta.yaml b/tests/test-recipes/metadata/jinja_load_toml/meta.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/metadata/jinja_load_toml/meta.yaml
@@ -0,0 +1,30 @@
+{% set pyproject = load_file_data("pyproject.toml", from_recipe_dir=True) %}
+{% set poetry = pyproject.get("tool").get("poetry") %}
+{% set deps = poetry.get("dependencies") %}
+
+package:
+  name: {{ poetry.get("name") }}
+  version: {{ poetry.get("version") }}
+
+source:
+  path: ../../test-package
+
+requirements:
+  build:
+    - python >={{ deps.get("python").lstrip("^") }}
+    - pip
+
+  run:
+{% for dep in deps -%}
+  {% if deps[dep] == "*" %}
+    - {{ dep }}
+  {% elif deps[dep][0] == "^" %}
+    - {{ dep }} >={{ deps[dep].lstrip("^") }}
+  {% else %}
+    - {{ dep }} =={{ deps[dep] }}
+  {% endif %}
+{% endfor %}
+
+build:
+  script: {{ PYTHON }} -m pip install --no-deps --ignore-installed .
+  noarch: python
\ No newline at end of file
diff --git a/tests/test-recipes/metadata/jinja_load_toml/pyproject.toml b/tests/test-recipes/metadata/jinja_load_toml/pyproject.toml
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/metadata/jinja_load_toml/pyproject.toml
@@ -0,0 +1,14 @@
+[tool.poetry]
+name = "test-jinja-load-toml"
+version = "1.test"
+description = "A test project to make sure conda-build can read toml into its meta.yaml"
+
+[tool.poetry.dependencies]
+python = "^3.6"
+tqdm = "*"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
diff --git a/tests/test-recipes/metadata/jinja_load_yaml/environment.yml b/tests/test-recipes/metadata/jinja_load_yaml/environment.yml
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/metadata/jinja_load_yaml/environment.yml
@@ -0,0 +1,7 @@
+name: foo
+channels:
+  - defaults
+dependencies:
+  - python=3.10
+  - tqdm
+prefix: /home/abraham/.conda/envs/foo
diff --git a/tests/test-recipes/metadata/jinja_load_yaml/meta.yaml b/tests/test-recipes/metadata/jinja_load_yaml/meta.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/metadata/jinja_load_yaml/meta.yaml
@@ -0,0 +1,23 @@
+{% set env = load_file_data("environment.yml", "yaml", from_recipe_dir=True) %}
+{% set deps = env.get("dependencies", []) %}
+
+package:
+  name: {{ env.get("name") }}
+  version: 1.2.3
+
+source:
+  path: ../../test-package
+
+requirements:
+  build:
+    - {{ deps[0].replace("=", " =") }}
+    - pip
+
+  run:
+  {% for dep in deps -%}
+    - {{ dep.replace("=", " =") }}
+  {% endfor %}
+
+build:
+  script: {{ PYTHON }} -m pip install --no-deps --ignore-installed .
+  noarch: python
\ No newline at end of file
diff --git a/tests/test_api_build.py b/tests/test_api_build.py
--- a/tests/test_api_build.py
+++ b/tests/test_api_build.py
@@ -503,13 +503,6 @@ def test_compileall_compiles_all_good_files(testing_workdir, testing_config):
     assert not package_has_file(output, add_mangling(bad_file))
 
 
-@pytest.mark.sanity
-@pytest.mark.serial
-def test_render_setup_py_old_funcname(testing_workdir, testing_config, caplog):
-    api.build(os.path.join(metadata_dir, "_source_setuptools"), config=testing_config)
-    assert "Deprecation notice: the load_setuptools function has been renamed to " in caplog.text
-
-
 @pytest.mark.sanity
 @pytest.mark.skipif(not on_win, reason="only Windows is insane enough to have backslashes in paths")
 def test_backslash_in_always_include_files_path(testing_config):
diff --git a/tests/test_jinja_context.py b/tests/test_jinja_context.py
--- a/tests/test_jinja_context.py
+++ b/tests/test_jinja_context.py
@@ -149,3 +149,17 @@ def test_load_setup_py_data_from_setup_cfg(testing_metadata, tmpdir):
     assert setuptools_data['name'] == 'name_from_setup_cfg'
     assert setuptools_data['version'] == 'version_from_setup_cfg'
     assert setuptools_data['extras_require'] == {'extra': ['extra_package']}
+
+
+@pytest.mark.parametrize("filename,fmt,data,expected", [
+    ("file.json", None, '{"a": 1}', {"a": 1}),
+    ("json_file", "json", '{"a": 1}', {"a": 1}),
+    ("file.toml", None, '[tbl]\na = 1', {"tbl": {"a": 1}}),
+    ("toml_file", "toml", '[tbl]\na = 1', {"tbl": {"a": 1}}),
+    ("file.yaml", None, 'a: 1\nb:\n  - c: 2', {"a": 1, "b": [{"c": 2}]}),
+])
+def test_load_file_data(tmpdir, filename, fmt, data, expected, testing_metadata):
+    f = tmpdir.join(filename)
+    f.write(data)
+    fn = str(f)
+    assert jinja_context.load_file_data(fn, fmt, config=testing_metadata.config) == expected