Repository: conda/conda-build

Issue:
load_setup_py_data/custom variable acts strangely in conda build 3
Hello, I've been trying to debug this for a bit, but I'm afraid I've gotten stuck.

Some background: 
We store our recipe in the repo we're trying to build and we use versioneer to version everything. We've got a notion of a "release train" where every package in our system is in lock-step. To make our lives easier we use `load_setup_py_data` in our template and just place that version (or a release) wherever it is needed.

However, with conda build 3 we're seeing the following on both linux and os-x:
```
Undefined Jinja2 variables remain (['get', 'get']).  Please enable source downloading and try again.
```
[full log on our Concourse CI instance here](https://busywork.qiime2.org/teams/main/pipelines/master-distribution/jobs/build-qiime2/builds/31)

This happens during the test phase of the build. What is kind of irritating is that when you look at the built conda package, the rendered `meta.yaml` file looks fine, no Jinja2 variables are found anywhere and they've all been resolved to the correct strings.

When I remove the `{{ version }}` variable from the recipe and make it a static string, the test phase finishes correctly, but obviously this defeats the purpose. 

Here is our [conda recipe](https://github.com/qiime2/qiime2/blob/master/ci/recipe/meta.yaml).

Is there a more correct way to handle this in conda build 3 or did we run into a regression of some sort?

Thanks!


Discussion:
Wow, that's a really cool set of concourse pipelines you have there.  I'm able to reproduce this locally, but I haven't isolated the cause yet.  I'll look into it for the next bugfix release.  Next day or two.

Side note: pinning python has never worked well:

https://github.com/qiime2/qiime2/blob/master/ci/recipe/meta.yaml#L20

If you remove the pinning there, but instead configure your CI system either by setting CONDA_PY=35 or using conda_build_config.yaml (documented at https://conda.io/docs/building/variants.html), I think you'll do better.  I'll get back you on this get thing being unresolved.
Thanks @msarahan! We'll probably just pin our conda-build to 2 for the moment then.

Re: python pinning, we're looking into getting Python 3.6 support soon, and so conda build 3's variants look awesome for this. Excited to start using them!

Resolving patch:
diff --git a/conda_build/environ.py b/conda_build/environ.py
--- a/conda_build/environ.py
+++ b/conda_build/environ.py
@@ -633,7 +633,7 @@ def package_specs(self):
         return specs
 
 
-spec_needing_star_re = re.compile("([0-9a-zA-Z\.]+\s+)([0-9a-zA-Z\.]+)(\s+[0-9a-zA-Z\.]+)?")
+spec_needing_star_re = re.compile("([0-9a-zA-Z\.]+\s+)([0-9a-zA-Z\.\+]+)(\s+[0-9a-zA-Z\.\_]+)?")
 spec_ver_needing_star_re = re.compile("^([0-9a-zA-Z\.]+)$")
 
 
diff --git a/conda_build/utils.py b/conda_build/utils.py
--- a/conda_build/utils.py
+++ b/conda_build/utils.py
@@ -851,6 +851,21 @@ def __hash__(self):
         return hash(json.dumps(self, sort_keys=True))
 
 
+def represent_hashabledict(dumper, data):
+    value = []
+
+    for item_key, item_value in data.items():
+        node_key = dumper.represent_data(item_key)
+        node_value = dumper.represent_data(item_value)
+
+        value.append((node_key, node_value))
+
+    return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value)
+
+
+yaml.add_representer(HashableDict, represent_hashabledict)
+
+
 # http://stackoverflow.com/a/10743550/1170370
 @contextlib.contextmanager
 def capture():
@@ -922,9 +937,13 @@ def conda_43():
     return LooseVersion(conda_version) >= LooseVersion('4.3')
 
 
-def _increment(version):
+def _increment(version, alpha_ver):
     try:
-        last_version = str(int(version) + 1)
+        if alpha_ver:
+            suffix = 'a'
+        else:
+            suffix = '.0a0'
+        last_version = str(int(version) + 1) + suffix
     except ValueError:
         last_version = chr(ord(version) + 1)
     return last_version
@@ -942,11 +961,17 @@ def apply_pin_expressions(version, min_pin='x.x.x.x.x.x.x', max_pin='x'):
         else:
             flat_list.append(item)
     versions = ['', '']
+    # first idx is lower bound pin; second is upper bound pin.
+    #    pin value is number of places to pin.
     for p_idx, pin in enumerate(pins):
         if pin:
+            # flat_list is the blown-out representation of the version
             for v_idx, v in enumerate(flat_list[:pin]):
+                # upper bound pin
                 if p_idx == 1 and v_idx == pin - 1:
-                    v = _increment(v)
+                    # is the last place an alphabetic character?  OpenSSL, JPEG
+                    alpha_ver = str(flat_list[min(pin, len(flat_list) - 1)]).isalpha()
+                    v = _increment(v, alpha_ver)
                 versions[p_idx] += str(v)
                 if v_idx != nesting_position:
                     versions[p_idx] += '.'


Test patch:
diff --git a/tests/test_api_render.py b/tests/test_api_render.py
--- a/tests/test_api_render.py
+++ b/tests/test_api_render.py
@@ -105,7 +105,7 @@ def test_output_without_jinja_does_not_download(mock_source, testing_workdir, te
 def test_pin_compatible_semver(testing_config):
     recipe_dir = os.path.join(metadata_dir, '_pin_compatible')
     metadata = api.render(recipe_dir, config=testing_config)[0][0]
-    assert 'numpy  >=1.11.2,<2' in metadata.get_value('requirements/run')
+    assert 'numpy  >=1.11.2,<2.0a0' in metadata.get_value('requirements/run')
     # not terribly important, but might be nice for continuity's sake
     #    This is broken right now, because compound pins like we do here have never been supported
     #    in the build string.
diff --git a/tests/test_jinja_context.py b/tests/test_jinja_context.py
--- a/tests/test_jinja_context.py
+++ b/tests/test_jinja_context.py
@@ -8,7 +8,7 @@ def test_pin_default(testing_metadata, mocker):
     get_env_dependencies = mocker.patch.object(jinja_context, 'get_env_dependencies')
     get_env_dependencies.return_value = ['test 1.2.3'], [], None
     pin = jinja_context.pin_compatible(testing_metadata, 'test')
-    assert pin == 'test  >=1.2.3,<2'
+    assert pin == 'test  >=1.2.3,<2.0a0'
 
 
 def test_pin_compatible_exact(testing_metadata, mocker):
@@ -22,7 +22,7 @@ def test_pin_jpeg_style_default(testing_metadata, mocker):
     get_env_dependencies = mocker.patch.object(jinja_context, 'get_env_dependencies')
     get_env_dependencies.return_value = ['jpeg 9d 0'], [], None
     pin = jinja_context.pin_compatible(testing_metadata, 'jpeg')
-    assert pin == 'jpeg  >=9d,<10'
+    assert pin == 'jpeg  >=9d,<10a'
 
 
 def test_pin_jpeg_style_minor(testing_metadata, mocker):
@@ -36,7 +36,7 @@ def test_pin_openssl_style_bugfix(testing_metadata, mocker):
     get_env_dependencies = mocker.patch.object(jinja_context, 'get_env_dependencies')
     get_env_dependencies.return_value = ['openssl 1.0.2j 0'], [], None
     pin = jinja_context.pin_compatible(testing_metadata, 'openssl', max_pin='x.x.x')
-    assert pin == 'openssl  >=1.0.2j,<1.0.3'
+    assert pin == 'openssl  >=1.0.2j,<1.0.3a'
     pin = jinja_context.pin_compatible(testing_metadata, 'openssl', max_pin='x.x.x.x')
     assert pin == 'openssl  >=1.0.2j,<1.0.2k'
 
@@ -45,7 +45,7 @@ def test_pin_major_minor(testing_metadata, mocker):
     get_env_dependencies = mocker.patch.object(jinja_context, 'get_env_dependencies')
     get_env_dependencies.return_value = ['test 1.2.3'], [], None
     pin = jinja_context.pin_compatible(testing_metadata, 'test', max_pin='x.x')
-    assert pin == 'test  >=1.2.3,<1.3'
+    assert pin == 'test  >=1.2.3,<1.3.0a0'
 
 
 def test_pin_upper_bound(testing_metadata, mocker):
@@ -58,15 +58,15 @@ def test_pin_upper_bound(testing_metadata, mocker):
 def test_pin_lower_bound(testing_metadata, mocker):
     get_env_dependencies = mocker.patch.object(jinja_context, 'get_env_dependencies')
     get_env_dependencies.return_value = ['test 1.2.3'], [], None
-    pin = jinja_context.pin_compatible(testing_metadata, 'test', lower_bound=1.0, upper_bound="3.0")
-    assert pin == 'test  >=1.0,<3.0'
+    pin = jinja_context.pin_compatible(testing_metadata, 'test', lower_bound=1.0)
+    assert pin == 'test  >=1.0,<2.0a0'
 
 
 def test_pin_none_min(testing_metadata, mocker):
     get_env_dependencies = mocker.patch.object(jinja_context, 'get_env_dependencies')
     get_env_dependencies.return_value = ['test 1.2.3'], [], None
     pin = jinja_context.pin_compatible(testing_metadata, 'test', min_pin=None)
-    assert pin == 'test  <2'
+    assert pin == 'test  <2.0a0'
 
 
 def test_pin_none_max(testing_metadata, mocker):