Repository: conda/conda-build

Issue:
Test circular relationships between outputs
Outputs can have "delayed" circular relationships, like:

```
outputs:
    - name: a
      run_exports:
          - {{ pin_subpackage('b', exact=True) }}
    - name: b
       requirements:
           - {{ pin_subpackage('a', exact=True) }}
```

We need to test that some downstream package using 'a' as a build dependency ends up pinning 'b' as a runtime dependency with an exact pin.


Discussion:

Resolving patch:
diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py
--- a/conda_build/jinja_context.py
+++ b/conda_build/jinja_context.py
@@ -307,7 +307,6 @@ def pin_subpackage(metadata, subpackage_name, min_pin='x.x.x.x.x.x', max_pin='x'
     For example, given a compiler package, allow it to specify either a compatible or exact
     pinning on the runtime package that is also created by the compiler package recipe
     """
-    global subpackage_cache
     if not hasattr(metadata, 'other_outputs'):
         if allow_no_other_outputs:
             pin = subpackage_name
@@ -316,13 +315,33 @@ def pin_subpackage(metadata, subpackage_name, min_pin='x.x.x.x.x.x', max_pin='x'
                              "order to allow pinning to them.  It's not here.")
     else:
         key = (subpackage_name, HashableDict(metadata.config.variant))
-        if key not in subpackage_cache:
-            pin = pin_subpackage_against_outputs(key, metadata.other_outputs, min_pin, max_pin,
-                                                 exact, permit_undefined_jinja)
-            if pin != subpackage_name:
-                subpackage_cache[key] = pin
-        else:
-            pin = subpackage_cache[key]
+        # two ways to match:
+        #    1. only one other output named the same as the subpackage_name from the key
+        #    2. whole key matches (both subpackage name and variant)
+        keys = list(metadata.other_outputs.keys())
+        matching_package_keys = [k for k in keys if k[0] == subpackage_name]
+        if len(matching_package_keys) == 1:
+            key = matching_package_keys[0]
+
+        pin = pin_subpackage_against_outputs(key, metadata.other_outputs, min_pin, max_pin,
+                                                exact, permit_undefined_jinja)
+        if pin != subpackage_name and exact:
+            assert len(pin.split(' ')) == 3
+            # find index of this package in list of other outputs
+
+            # test that no outputs earlier in the list of other outputs have this
+            #     package pinned exactly in their requirements/run or build/run_exports sections
+            subpackage_index = list(metadata.other_outputs.keys()).index(key)
+            for _, m in list(metadata.other_outputs.values())[:subpackage_index]:
+                deps = m.get_value('requirements/run') + m.get_value('build/run_exports')
+                for dep in deps:
+                    if (dep.split()[0] == subpackage_name and
+                            len(dep.split()) == 3 and
+                            dep != pin):
+                        # raise an error, because this indicates a cyclical dependency
+                        raise ValueError("Infinite loop in subpackages. Exact pins in dependencies "
+                                    "that contribute to the hash often cause this. Can you "
+                                    "change one or more exact pins to version bound constraints?")
     return pin
 
 
diff --git a/conda_build/metadata.py b/conda_build/metadata.py
--- a/conda_build/metadata.py
+++ b/conda_build/metadata.py
@@ -254,11 +254,12 @@ def parse(data, config, path=None):
 falses = {'n', 'no', 'false', 'off'}
 
 default_structs = {
-    'source/patches': list,
     'build/entry_points': list,
+    'build/features': list,
+    'source/patches': list,
     'build/script': list,
     'build/script_env': list,
-    'build/features': list,
+    'build/run_exports': list,
     'build/track_features': list,
     'requirements/build': list,
     'requirements/host': list,
@@ -369,7 +370,7 @@ def _git_clean(source_meta):
               'detect_binary_files_with_prefix', 'skip_compile_pyc', 'rpaths',
               'script_env', 'always_include_files', 'skip', 'msvc_compiler',
               'pin_depends', 'include_recipe',  # pin_depends is experimental still
-              'preferred_env', 'preferred_env_executable_paths',
+              'preferred_env', 'preferred_env_executable_paths', 'run_exports',
               ],
     'requirements': ['build', 'host', 'run', 'conflicts', 'run_constrained'],
     'app': ['entry', 'icon', 'summary', 'type', 'cli_opts',
@@ -537,6 +538,45 @@ def get_output_dicts_from_metadata(metadata):
     return outputs
 
 
+def finalize_outputs_pass(base_metadata, iteration, render_order=None, outputs=None,
+                          permit_unsatisfiable_variants=True):
+    from .render import finalize_metadata
+    outputs = OrderedDict()
+    for output_d, metadata in render_order.values():
+        try:
+            log = utils.get_logger(__name__)
+            log.info("Attempting to finalize metadata for {}".format(metadata.name()))
+            # this is weird, but I think necessary. We should reparse
+            #    the top-level recipe to get all of our dependencies
+            #    fixed up.
+            if not hasattr(base_metadata, 'other_outputs'):
+                base_metadata.other_outputs = OrderedDict()
+            base_metadata.other_outputs.update(outputs)
+            om = base_metadata.copy()
+            om.config.variant = metadata.config.variant
+            if not om.final:
+                om.parse_until_resolved()
+            # get the new output_d from the reparsed top-level metadata, so that we have any
+            #    exact subpackage version/hash info
+            recipe_outputs = get_output_dicts_from_metadata(om)
+            output_d = get_updated_output_dict_from_reparsed_metadata(output_d,
+                                                                        recipe_outputs)
+            metadata = om.get_output_metadata(output_d)
+            fm = finalize_metadata(metadata)
+            if not output_d.get('type') or output_d.get('type') == 'conda':
+                outputs[(fm.name(), HashableDict(fm.config.variant))] = (output_d, fm)
+        except DependencyNeedsBuildingError as e:
+            if not permit_unsatisfiable_variants:
+                raise
+            else:
+                log = utils.get_logger(__name__)
+                log.warn("Could not finalize metadata due to missing dependencies: "
+                            "{}".format(e.packages))
+                outputs[(metadata.name(), HashableDict(metadata.config.variant))] = (
+                    output_d, metadata)
+    return outputs
+
+
 def get_updated_output_dict_from_reparsed_metadata(original_dict, new_outputs):
     output_d = original_dict
     if 'name' in original_dict:
@@ -1417,7 +1457,6 @@ def get_output_metadata(self, output):
 
     def get_output_metadata_set(self, permit_undefined_jinja=False,
                                 permit_unsatisfiable_variants=True):
-        from .render import finalize_metadata
         out_metadata_map = {}
 
         for variant in (self.config.variants if hasattr(self.config, 'variants')
@@ -1436,51 +1475,44 @@ def get_output_metadata_set(self, permit_undefined_jinja=False,
                     raise
                 out_metadata_map = {}
 
+        # format here is {output_dict: metadata_object}
         render_order = toposort(out_metadata_map, phase='build')
+
+        conda_packages = OrderedDict()
         non_conda_packages = []
-        outputs = OrderedDict()
+        for output_d, m in render_order.items():
+            if not output_d.get('type') or output_d['type'] == 'conda':
+                conda_packages[m.name(), HashableDict(m.config.variant)] = (output_d, m)
+            else:
+                # for wheels and other non-conda packages, just append them at the end.
+                #    no deduplication with hashes currently.
+                # hard part about including any part of output_d
+                #    outside of this func is that it is harder to
+                #    obtain an exact match elsewhere
+                non_conda_packages.append((output_d, m))
 
         # early stages don't need to do the finalization.  Skip it until the later stages
         #     when we need it.
-        if permit_undefined_jinja:
-            outputs = render_order
-        else:
-            # variant is already rolled in from above
-            for output_d, metadata in render_order.items():
-                try:
-                    log = utils.get_logger(__name__)
-                    log.info("Attempting to finalize metadata for {}".format(metadata.name()))
-                    # this is weird, but I think necessary. We should reparse
-                    #    the top-level recipe to get all of our dependencies
-                    #    fixed up.
-                    self.other_outputs = outputs
-                    om = self.copy()
-                    om.config.variant = metadata.config.variant
-                    if not om.final:
-                        om.parse_until_resolved()
-                    # get the new output_d from the reparsed top-level metadata, so that we have any
-                    #    exact subpackage version/hash info
-                    recipe_outputs = get_output_dicts_from_metadata(om)
-                    output_d = get_updated_output_dict_from_reparsed_metadata(output_d,
-                                                                              recipe_outputs)
-                    metadata = om.get_output_metadata(output_d)
-                    fm = finalize_metadata(metadata)
-                    if not output_d.get('type') or output_d.get('type') == 'conda':
-                        outputs[(fm.name(), HashableDict(fm.config.variant))] = (output_d, fm)
-                    else:
-                        # for wheels and other non-conda packages, just append them at the end.
-                        #    no deduplication with hashes currently.
-                        # hard part about including any part of output_d
-                        #    outside of this func is that it is harder to
-                        #    obtain an exact match elsewhere
-                        non_conda_packages.append((output_d, fm))
-
-                except DependencyNeedsBuildingError as e:
-                    if not permit_unsatisfiable_variants:
-                        raise
-                    else:
-                        log = utils.get_logger(__name__)
-                        log.warn("Could not finalize metadata due to missing dependencies: "
-                                    "{}".format(e.packages))
-                        outputs[(metadata.name(), HashableDict(variant))] = (output_d, metadata)
-        return list(outputs.values()) + non_conda_packages
+        if not permit_undefined_jinja:
+            # The loop above gives us enough info to determine the build order.  After that,
+            #    we can "finalize" the metadata (fill in version pins) for the build metadata.
+            #    This does not, however, account for circular dependencies
+            #    where a runtime exact dependency pin on a downstream build
+            #    subpackage in an upstream subpackage changes the upstream's
+            #    hash, which then changes the downstream package's hash, which
+            #    then recurses infinitely
+
+            # In general, given upstream a and downstream b, b can depend on a
+            # exactly, but a can only have version constraints on b at run or
+            # run_exports, not exact=True
+
+            # 3 passes here:
+            #    1. fill in fully-resolved build-time dependencies
+            #    2. fill in fully-resolved run-time dependencies.  Note that circular dependencies
+            #       are allowed, but you can't have exact=True for circular run-time dependencies
+            #    3. finally, everything should be filled in and done.
+            for i in range(3):
+                conda_packages = finalize_outputs_pass(self, i, conda_packages,
+                                        permit_unsatisfiable_variants=permit_unsatisfiable_variants)
+                self.other_outputs = conda_packages
+        return list(conda_packages.values()) + non_conda_packages
diff --git a/conda_build/render.py b/conda_build/render.py
--- a/conda_build/render.py
+++ b/conda_build/render.py
@@ -155,6 +155,9 @@ def get_upstream_pins(m, actions, index):
                 elif os.path.isfile(pkg_file):
                     extra_specs = utils.package_has_file(pkg_file, 'info/run_exports')
                     if extra_specs:
+                        # exclude packages pinning themselves (makes no sense)
+                        extra_specs = [spec for spec in extra_specs
+                                       if not spec.startswith(pkg_dist.rsplit('-', 2)[0])]
                         additional_specs.extend(extra_specs.splitlines())
                     break
                 elif utils.conda_43():


Test patch:
diff --git a/tests/test-recipes/split-packages/_intradependencies_circular/meta.yaml b/tests/test-recipes/split-packages/_intradependencies_circular/meta.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/split-packages/_intradependencies_circular/meta.yaml
@@ -0,0 +1,17 @@
+# What's supposed to happen here is:
+#   1. a gets rendered.  It has an unresolved run_exports entry for b (just b, without the pin)
+#   2. b gets rendered.  It has an exact reference to a.  This in turn will change b - and the loop goes on.
+#      Conda-build should not loop.
+
+package:
+  name: circular_intradependencies
+  version: 1.0
+
+outputs:
+  - name: a
+    run_exports:
+      - {{ pin_subpackage('b', exact=True) }}
+  - name: b
+    requirements:
+      build:
+        - {{ pin_subpackage('a', exact=True) }}
diff --git a/tests/test_subpackages.py b/tests/test_subpackages.py
--- a/tests/test_subpackages.py
+++ b/tests/test_subpackages.py
@@ -194,9 +194,18 @@ def test_toplevel_entry_points_do_not_apply_to_subpackages(testing_config):
             assert not utils.package_has_file(out, '{}/{}{}'.format(script_dir, 'top2', ext))
         elif fn.startswith('test_split_package_entry_points'):
             # python commands will make sure that these are available.
-            # assert utils.package_has_file(out, '{}/{}{}'.format(script_dir, 'top1', ext))
-            # assert utils.package_has_file(out, '{}/{}{}'.format(script_dir, 'top2', ext))
+            assert utils.package_has_file(out, '{}/{}{}'.format(script_dir, 'top1', ext))
+            assert utils.package_has_file(out, '{}/{}{}'.format(script_dir, 'top2', ext))
             assert not utils.package_has_file(out, '{}/{}{}'.format(script_dir, 'pkg1', ext))
             assert not utils.package_has_file(out, '{}/{}{}'.format(script_dir, 'pkg2', ext))
         else:
             raise ValueError("Didn't see any of the 3 expected filenames.  Filename was {}".format(fn))
+
+
+def test_cyclical_exact_subpackage_pins_raises_error(testing_config):
+    recipe_dir = os.path.join(subpackage_dir, '_intradependencies_circular')
+    with pytest.raises(ValueError) as e:
+        api.build(recipe_dir, config=testing_config)
+    assert e.value.args[0] == ("Infinite loop in subpackages. Exact pins in "
+            "dependencies that contribute to the hash often cause this. Can "
+            "you change one or more exact pins to version bound constraints?")