Repository: conda/conda-build

Issue:
Outputs with inner loop over python remove earlier artifacts
xgboost recipe currently failing because packages disappear:

https://github.com/conda-forge/xgboost-feedstock
help pls!
looks like the CI services couldn't upload the mutex packages
https://circleci.com/gh/conda-forge/xgboost-feedstock/115?utm_campaign=vcs-integration-link&utm_medium=referral&utm_source=github-build-link



	


Discussion:

Resolving patch:
diff --git a/conda_build/build.py b/conda_build/build.py
--- a/conda_build/build.py
+++ b/conda_build/build.py
@@ -1252,7 +1252,8 @@ def build(m, stats, post=None, need_source_download=True, need_reparse_in_env=Fa
             return default_return
 
         print("BUILD START:", [os.path.basename(pkg) for pkg in package_locations])
-        environ.remove_existing_packages(m.config.bldpkgs_dir, package_locations, m.config)
+        environ.remove_existing_packages(m.config.bldpkgs_dir,
+                [pkg for pkg in package_locations if pkg not in built_packages], m.config)
 
         specs = [ms.spec for ms in m.ms_depends('build')]
         if any(out.get('type') == 'wheel' for out in m.meta.get('outputs', [])):


Test patch:
diff --git a/tests/test-recipes/metadata/_cmake_generator/meta.yaml b/tests/test-recipes/metadata/_cmake_generator/meta.yaml
--- a/tests/test-recipes/metadata/_cmake_generator/meta.yaml
+++ b/tests/test-recipes/metadata/_cmake_generator/meta.yaml
@@ -5,3 +5,4 @@ package:
 requirements:
   build:
     - cmake
+    - {{ compiler("c") }}
diff --git a/tests/test-recipes/split-packages/_xgboost_example/conda_build_config.yaml b/tests/test-recipes/split-packages/_xgboost_example/conda_build_config.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/split-packages/_xgboost_example/conda_build_config.yaml
@@ -0,0 +1,8 @@
+python:
+  - 2.7
+  - 3.6
+r_version:
+  - 3.5.0
+r_implementation:
+  - 'r-base'
+  - 'mro-base'  # [not osx]
diff --git a/tests/test-recipes/split-packages/_xgboost_example/meta.yaml b/tests/test-recipes/split-packages/_xgboost_example/meta.yaml
new file mode 100644
--- /dev/null
+++ b/tests/test-recipes/split-packages/_xgboost_example/meta.yaml
@@ -0,0 +1,100 @@
+{% set name = "xgboost" %}
+{% set version = "0.80" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+build:
+  number: 1
+  skip: true  # [win or linux32]
+
+requirements:
+  build:
+    - make
+
+outputs:
+  - name: libxgboost
+    requirements:
+      build:
+        - make
+
+  - name: _py-xgboost-mutex
+    version: 2.0
+    build:
+      string: cpu_0
+
+  - name: py-xgboost
+    requirements:
+      host:
+        - {{ pin_subpackage('libxgboost', exact=True) }}
+        - python
+        - setuptools
+        - pip
+      run:
+        - {{ pin_subpackage('libxgboost', exact=True) }}
+        - {{ pin_subpackage('_py-xgboost-mutex', exact=True) }}
+        - python
+
+  - name: py-xgboost-cpu
+    requirements:
+      host:
+        - python
+      run:
+        - python
+        - {{ pin_subpackage('py-xgboost', exact=True) }}
+
+  - name: xgboost
+    requirements:
+      host:
+        - python
+      run:
+        - {{ pin_subpackage('py-xgboost', exact=True) }}
+
+  - name: _r-xgboost-mutex
+    version: 2.0
+    build:
+      string: cpu_0
+
+  - name: r-xgboost
+    build:
+      rpaths:
+        - lib/R/lib
+    requirements:
+      host:
+        - {{ pin_subpackage('libxgboost', exact=True) }}
+        - {{ r_implementation }}
+      run:
+        - {{ pin_subpackage('libxgboost', exact=True) }}
+        - {{ pin_subpackage('_r-xgboost-mutex', exact=True) }}
+        - llvm-openmp >=4.0.1    # [osx]
+        - {{ r_implementation }}
+
+  - name: r-xgboost-cpu
+    requirements:
+      host:
+        - {{ r_implementation }}
+      run:
+        - {{ r_implementation }}
+        - {{ pin_subpackage('r-xgboost', exact=True) }}
+
+about:
+  home: https://github.com/dmlc/xgboost
+  license: Apache-2.0
+  summary: |
+    Scalable, Portable and Distributed Gradient Boosting (GBDT, GBRT or GBM) Library, for
+    Python, R, Java, Scala, C++ and more. Runs on single machine, Hadoop, Spark, Flink
+    and DataFlow
+  description: |
+    XGBoost is an optimized distributed gradient boosting library designed to be highly efficient,
+    flexible and portable. It implements machine learning algorithms under the Gradient Boosting
+    framework. XGBoost provides a parallel tree boosting (also known as GBDT, GBM) that solve many
+    data science problems in a fast and accurate way. The same code runs on major distributed
+    environment (Hadoop, SGE, MPI) and can solve problems beyond billions of examples.
+  doc_url: https://xgboost.readthedocs.io/
+  dev_url: https://github.com/dmlc/xgboost/
+
+extra:
+  recipe-maintainers:
+    - beckermr
+    - aldanor
diff --git a/tests/test_subpackages.py b/tests/test_subpackages.py
--- a/tests/test_subpackages.py
+++ b/tests/test_subpackages.py
@@ -1,3 +1,4 @@
+from glob import glob
 import json
 import os
 import pytest
@@ -278,6 +279,7 @@ def test_strong_run_exports_from_build_applies_to_host(testing_config):
     api.build(recipe, config=testing_config)
 
 
+@pytest.mark.serial
 @pytest.mark.parametrize("recipe", ('_line_up_python_compiled_libs',
                                     '_line_up_python_compiled_libs_top_level_same_name_output'))
 def test_python_line_up_with_compiled_lib(recipe, testing_config):
@@ -328,3 +330,11 @@ def test_inherit_build_number(testing_config):
     for m, _, _ in ms:
         assert 'number' in m.meta['build'], "build number was not inherited at all"
         assert int(m.meta['build']['number']) == 1, "build number should have been inherited as '1'"
+
+
+def test_loops_do_not_remove_earlier_packages(testing_config):
+    recipe = os.path.join(subpackage_dir, '_xgboost_example')
+    output_files = api.get_output_file_paths(recipe, config=testing_config)
+
+    api.build(recipe, config=testing_config)
+    assert len(output_files) == len(glob(os.path.join(testing_config.croot, testing_config.host_subdir, "*.tar.bz2")))
diff --git a/tests/test_variants.py b/tests/test_variants.py
--- a/tests/test_variants.py
+++ b/tests/test_variants.py
@@ -354,6 +354,7 @@ def test_target_platform_looping(testing_config):
     assert len(outputs) == 2
 
 
+@pytest.mark.serial
 def test_numpy_used_variable_looping(testing_config):
     outputs = api.get_output_file_paths(os.path.join(recipe_dir, 'numpy_used'),
                                    platform='win', arch='64')
@@ -407,6 +408,7 @@ def test_exclusive_config_file(testing_workdir):
     assert variant['abc'] == '123'
 
 
+@pytest.mark.serial
 def test_inner_python_loop_with_output(testing_config):
     outputs = api.get_output_file_paths(os.path.join(recipe_dir, 'test_python_as_subpackage_loop'),
                                         config=testing_config)