Repository: conda/conda-build

Issue:
Download of source packages is done before skip-existing
This behaviour changed between CB2 and 3: in CB2 the source download is done after deciding whether we can skip the build while in CB3 not.. That wastes a lot of space and computing time. Could that be changed back?


Discussion:
Conda-build should only be downloading when it is necessary either to render or to build.  If you are not using git jinja, for example, it should not be downloading.  If you can provide a recipe or procedure that reproduces the problem, I'll be happy to look into it.

Resolving patch:
diff --git a/conda_build/build.py b/conda_build/build.py
--- a/conda_build/build.py
+++ b/conda_build/build.py
@@ -867,6 +867,23 @@ def build(m, post=None, need_source_download=True, need_reparse_in_env=False, bu
         env["CONDA_PATH_BACKUP"] = os.environ["CONDA_PATH_BACKUP"]
 
     if post in [False, None]:
+        output_metas = expand_outputs([(m, need_source_download, need_reparse_in_env)])
+
+        if m.config.skip_existing:
+            # TODO: should we check both host and build envs?  These are the same, except when
+            #    cross compiling.
+            package_locations = [is_package_built(om, 'host') for _, om in output_metas]
+            if all(package_locations):
+                print("Packages for ", m.path or m.name(),
+                        "are already built in {0}, skipping.".format(package_locations))
+                return default_return
+            else:
+                package_locations = [bldpkg_path(om) for _, om in output_metas]
+        else:
+            package_locations = [bldpkg_path(om) for _, om in output_metas]
+
+        print("BUILD START:", [os.path.basename(pkg) for pkg in package_locations])
+
         specs = [ms.spec for ms in m.ms_depends('build')]
         if any(out.get('type') == 'wheel' for out in m.meta.get('outputs', [])):
             specs.extend(['pip', 'wheel'])
@@ -896,7 +913,9 @@ def build(m, post=None, need_source_download=True, need_reparse_in_env=False, bu
         # installed files at packaging-time.
         host_ms_deps = None
         build_ms_deps = None
-        test_run_ms_deps = m.get_value('test/requires', []) + m.get_value('requirements/run', [])
+
+        for env in ('build', 'host'):
+            utils.insert_variant_versions(m, env)
 
         if (m.config.host_subdir != m.config.build_subdir and
                 m.config.host_subdir != "noarch"):
@@ -936,6 +955,9 @@ def build(m, post=None, need_source_download=True, need_reparse_in_env=False, bu
 
         try:
             if not notest:
+                utils.insert_variant_versions(m, 'run')
+                test_run_ms_deps = m.get_value('test/requires', []) + \
+                                   m.get_value('requirements/run', [])
                 # make sure test deps are available before taking time to create build env
                 environ.get_install_actions(m.config.test_prefix,
                                             tuple(test_run_ms_deps), 'test',
@@ -1004,23 +1026,6 @@ def build(m, post=None, need_source_download=True, need_reparse_in_env=False, bu
             f.write(u'\n'.join(sorted(list(files1))))
             f.write(u'\n')
 
-        output_metas = expand_outputs([(m, need_source_download, need_reparse_in_env)])
-
-        if m.config.skip_existing:
-            # TODO: should we check both host and build envs?  These are the same, except when
-            #    cross compiling.
-            package_locations = [is_package_built(om, 'host') for _, om in output_metas]
-            if all(package_locations):
-                print("Packages for ", m.path or m.name(),
-                        "are already built in {0}, skipping.".format(package_locations))
-                return default_return
-            else:
-                package_locations = [bldpkg_path(om) for _, om in output_metas]
-        else:
-            package_locations = [bldpkg_path(om) for _, om in output_metas]
-
-        print("BUILD START:", [os.path.basename(pkg) for pkg in package_locations])
-
         # Use script from recipe?
         script = utils.ensure_list(m.get_value('build/script', None))
         if script:
diff --git a/conda_build/render.py b/conda_build/render.py
--- a/conda_build/render.py
+++ b/conda_build/render.py
@@ -89,7 +89,7 @@ def get_env_dependencies(m, env, variant, exclude_pattern=None,
             for key, value in variant.items():
                 # for sake of comparison, ignore dashes and underscores
                 if (dash_or_under.sub("", key) == dash_or_under.sub("", spec_name) and
-                        not re.search(r'%s\s+[0-9a-zA-Z\_\.]' % spec_name, spec)):
+                        not re.search(r'%s\s+[0-9a-zA-Z\_\.\<\>\=\*]' % spec_name, spec)):
                     dependencies.append(" ".join((spec_name, value)))
         elif exclude_pattern.match(spec):
             pass_through_deps.append(spec)
@@ -236,7 +236,6 @@ def get_upstream_pins(m, actions, env):
             else:
                 raise RuntimeError("Didn't find expected package {} in package cache ({})"
                                     .format(pkg_dist, _pkgs_dirs))
-
     return additional_specs
 
 


Test patch:
diff --git a/tests/test-recipes/metadata/empty_sections/meta.yaml b/tests/test-recipes/metadata/empty_sections/meta.yaml
--- a/tests/test-recipes/metadata/empty_sections/meta.yaml
+++ b/tests/test-recipes/metadata/empty_sections/meta.yaml
@@ -2,6 +2,9 @@ package:
   name: empty_sections
   version: 0.0
 
+source:
+  path: ../../../../../conda_build_test_recipe
+
 build:
 
 requirements:
diff --git a/tests/test_cli.py b/tests/test_cli.py
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -481,15 +481,18 @@ def test_conda_py_no_period(testing_workdir, testing_metadata, monkeypatch):
 
 
 @pytest.mark.serial
-def test_build_skip_existing(testing_workdir, capfd):
+def test_build_skip_existing(testing_workdir, capfd, mocker):
     # build the recipe first
     empty_sections = os.path.join(metadata_dir, "empty_sections")
     args = ['--no-anaconda-upload', empty_sections]
     main_build.execute(args)
     args.insert(0, '--skip-existing')
+    import conda_build.source
+    provide = mocker.patch.object(conda_build.source, 'provide')
     main_build.execute(args)
+    provide.assert_not_called()
     output, error = capfd.readouterr()
-    assert "are already built" in output
+    assert ("are already built" in output or "are already built" in error)
 
 
 @pytest.mark.serial