Repository: conda/conda-build

Issue:
all builds fail on Fedora due to environment corruption
On Fedora 22, conda build fails when trying to run build.sh, like this:

```
/bin/bash: scl: line 1: syntax error: unexpected end of file
Command failed: /bin/bash -x -e /home/hp/checkout/conda-project/conda.recipe/build.sh
```

The issue appears to be that the environment variable called `BASH_FUNC_scl()` (the parens are in the env var name!) gets its value corrupted. `BASH_FUNC_module()` also exists and has the same issue.

Here is the output of printenv:

```
BASH_FUNC_scl()=() {  local CMD=$1;
 if [ "$CMD" = "load" -o "$CMD" = "unload" ]; then
 eval "module $@";
 else
 /usr/bin/scl "$@";
 fi
}
BASH_FUNC_module()=() {  eval `/usr/bin/modulecmd bash $*`
}
```

If I add debug prints to conda_build/build.py, the value of `BASH_FUNC_scl()` is `() {  local CMD`

The code in environ.py doesn't look prepared for newline or "=" to be in the value of an env var, but in this case both of them are.

On Linux a possible fix would be to use `env --null` then `split('\x00')` instead of newline, and then only special-case the _first_ '=' (treat any subsequent '=' as part of the value), maybe using `var.split('=', 1)`.  However, I wouldn't be surprised if BSD lacks `--null`. Then again, BSD probably doesn't set env vars to stuff containing '=' and newline.



Discussion:
Thanks for the great bug report and suggestions.  That's a tough problem.  OS X does not seem to have --null.

Perhaps I should just ignore any variables that have unmatched brackets before a newline?  I really don't want to write a super-duper parser here.

nutty possibility, maybe replace "env" with something like `python -c 'import json; print(json.dumps(environ))'` ? didn't test that but you get the idea ... 


Resolving patch:
diff --git a/conda_build/build.py b/conda_build/build.py
--- a/conda_build/build.py
+++ b/conda_build/build.py
@@ -30,7 +30,6 @@
 from conda_build import __version__
 from conda_build import environ, source, tarcheck
 from conda_build.config import config
-from conda_build.environ import activate_env, deactivate_env
 from conda_build.render import parse_or_try_download, output_yaml, bldpkg_path
 from conda_build.scripts import create_entry_points, prepend_bin_path
 from conda_build.post import (post_process, post_build,
@@ -75,13 +74,13 @@ def create_post_scripts(m):
     Create scripts to run after build step
     '''
     recipe_dir = m.path
-    ext = '.bat' if sys.platform == 'win32' else '.sh'
+    ext = '.bat' if on_win else '.sh'
     for tp in 'pre-link', 'post-link', 'pre-unlink':
         src = join(recipe_dir, tp + ext)
         if not isfile(src):
             continue
         dst_dir = join(config.build_prefix,
-                       'Scripts' if sys.platform == 'win32' else 'bin')
+                       'Scripts' if on_win else 'bin')
         if not isdir(dst_dir):
             os.makedirs(dst_dir, int('755', 8))
         dst = join(dst_dir, '.%s-%s%s' % (m.name(), tp, ext))
@@ -126,7 +125,7 @@ def have_prefix_files(files):
 
         mode = 'binary' if mm.find(b'\x00') != -1 else 'text'
         if mode == 'text':
-            if sys.platform != 'win32' and mm.find(prefix_bytes) != -1:
+            if not on_win and mm.find(prefix_bytes) != -1:
                 # Use the placeholder for maximal backwards compatibility, and
                 # to minimize the occurrences of usernames appearing in built
                 # packages.
@@ -252,7 +251,7 @@ def create_info_files(m, files, include_recipe=True):
                 d[key] = value
         json.dump(d, fo, indent=2, sort_keys=True)
 
-    if sys.platform == 'win32':
+    if on_win:
         # make sure we use '/' path separators in metadata
         files = [_f.replace('\\', '/') for _f in files]
 
@@ -279,7 +278,7 @@ def create_info_files(m, files, include_recipe=True):
 
     if files_with_prefix and not m.get_value('build/noarch_python'):
         auto_detect = m.get_value('build/detect_binary_files_with_prefix')
-        if sys.platform == 'win32':
+        if on_win:
             # Paths on Windows can contain spaces, so we need to quote the
             # paths. Fortunately they can't contain quotes, so we don't have
             # to worry about nested quotes.
@@ -408,7 +407,8 @@ def rm_pkgs_cache(dist):
 
 
 def build(m, post=None, include_recipe=True, keep_old_work=False,
-          need_source_download=True, verbose=True, dirty=False):
+          need_source_download=True, verbose=True, dirty=False,
+          activate=True):
     '''
     Build the package with the specified metadata.
 
@@ -468,13 +468,6 @@ def build(m, post=None, include_recipe=True, keep_old_work=False,
             create_env(config.build_prefix,
                     [ms.spec for ms in m.ms_depends('build')])
 
-            # modify os.environ to have our new environment be active
-            #    This is important for setting variables from activate.d and deactivate.d
-            # this modifies os.environ in place, and provides a diff of variables to actually
-            #    include in the build environment later
-            # Keeping all environment variables would defeat some isolation that we want.
-            env_diff = activate_env(config.build_prefix)
-
             if need_source_download:
                 # Execute any commands fetching the source (e.g., git) in the _build environment.
                 # This makes it possible to provide source fetchers (eg. git, hg, svn) as build
@@ -528,34 +521,42 @@ def build(m, post=None, include_recipe=True, keep_old_work=False,
                 if isinstance(script, list):
                     script = '\n'.join(script)
 
-            if sys.platform == 'win32':
+            if on_win:
                 build_file = join(m.path, 'bld.bat')
                 if script:
                     build_file = join(source.get_dir(), 'bld.bat')
                     with open(join(source.get_dir(), 'bld.bat'), 'w') as bf:
                         bf.write(script)
                 import conda_build.windows as windows
-                windows.build(m, build_file, dirty=dirty, env_diff=env_diff)
+                windows.build(m, build_file, dirty=dirty, activate=activate)
             else:
-                env = environ.get_dict(m, dirty=dirty)
-                env.update(env_diff)
-
                 build_file = join(m.path, 'build.sh')
 
-                if script:
-                    build_file = join(source.get_dir(), 'conda_build.sh')
-                    with open(build_file, 'w') as bf:
-                        bf.write(script)
-                    os.chmod(build_file, 0o766)
-
-                if isfile(build_file):
-                    cmd = [shell_path, '-x', '-e', build_file]
-
-                    _check_call(cmd, env=env, cwd=src_dir)
-
-        # this is necessary to return our current os.environ to normal.
-        #    We are discarding the list of changed values.
-        deactivate_env()
+                # There is no sense in trying to run an empty build script.
+                if isfile(build_file) or script:
+                    env = environ.get_dict(m, dirty=dirty)
+                    work_file = join(source.get_dir(), 'conda_build.sh')
+                    if script:
+                        with open(work_file, 'w') as bf:
+                            bf.write(script)
+                    if activate:
+                        if isfile(build_file):
+                            data = open(build_file).read()
+                        else:
+                            data = open(work_file).read()
+                        with open(work_file, 'w') as bf:
+                            bf.write("source activate {build_prefix}\n".format(
+                                build_prefix=config.build_prefix))
+                            bf.write(data)
+                    else:
+                        if not isfile(work_file):
+                            shutil.copy(build_file, work_file)
+                    os.chmod(work_file, 0o766)
+
+                    if isfile(work_file):
+                        cmd = [shell_path, '-x', '-e', work_file]
+
+                        _check_call(cmd, env=env, cwd=src_dir)
 
         if post in [True, None]:
             if post:
@@ -627,7 +628,7 @@ def order(f):
             shutil.rmtree(old_WORK_DIR, ignore_errors=True)
 
 
-def test(m, move_broken=True):
+def test(m, move_broken=True, activate=True):
     '''
     Execute any test scripts for the given package.
 
@@ -690,67 +691,60 @@ def test(m, move_broken=True):
             specs += ['lua %s*' % environ.get_lua_ver()]
 
         create_env(config.test_prefix, specs)
-        env_diff = activate_env(config.test_prefix)
 
         env = dict(os.environ)
         env.update(environ.get_dict(m, prefix=config.test_prefix))
-        env.update(env_diff)
 
-        # prepend bin (or Scripts) directory
-        env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True)
+        if not activate:
+            # prepend bin (or Scripts) directory
+            env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True)
+
+            if on_win:
+                env['PATH'] = config.test_prefix + os.pathsep + env['PATH']
 
-        if sys.platform == 'win32':
-            env['PATH'] = config.test_prefix + os.pathsep + env['PATH']
         for varname in 'CONDA_PY', 'CONDA_NPY', 'CONDA_PERL', 'CONDA_LUA':
             env[varname] = str(getattr(config, varname) or '')
         env['PREFIX'] = config.test_prefix
 
         # Python 2 Windows requires that envs variables be string, not unicode
         env = {str(key): str(value) for key, value in env.items()}
-        if py_files:
-            try:
-                subprocess.check_call([config.test_python, '-s',
-                                    join(tmp_dir, 'run_test.py')],
-                                    env=env, cwd=tmp_dir)
-            except subprocess.CalledProcessError:
-                tests_failed(m, move_broken=move_broken)
-
-        if pl_files:
-            try:
-                subprocess.check_call([config.test_perl,
-                                    join(tmp_dir, 'run_test.pl')],
-                                    env=env, cwd=tmp_dir)
-            except subprocess.CalledProcessError:
-                tests_failed(m, move_broken=move_broken)
-
-        if lua_files:
-            try:
-                subprocess.check_call([config.test_lua,
-                                    join(tmp_dir, 'run_test.lua')],
-                                    env=env, cwd=tmp_dir)
-            except subprocess.CalledProcessError:
-                tests_failed(m)
-
-        if shell_files:
-            if sys.platform == 'win32':
-                test_file = join(tmp_dir, 'run_test.bat')
-                cmd = [os.environ['COMSPEC'], '/c', 'call', test_file]
-                try:
-                    subprocess.check_call(cmd, env=env, cwd=tmp_dir)
-                except subprocess.CalledProcessError:
-                    tests_failed(m, move_broken=move_broken)
-            else:
-                test_file = join(tmp_dir, 'run_test.sh')
-                # TODO: Run the test/commands here instead of in run_test.py
-                cmd = [shell_path, '-x', '-e', test_file]
-                try:
-                    subprocess.check_call(cmd, env=env, cwd=tmp_dir)
-                except subprocess.CalledProcessError:
-                    tests_failed(m, move_broken=move_broken)
-
-        # this is necessary to return our current os.environ to normal.
-        #    We are discarding the list of changed values.
-        deactivate_env()
+        suffix = "bat" if on_win else "sh"
+        test_script = join(tmp_dir, "conda_test_runner.{suffix}".format(suffix=suffix))
+
+        with open(test_script, 'w') as tf:
+            if activate:
+                tf.write("{source}activate _test\n".format(source="" if on_win else "source "))
+            if py_files:
+                tf.write("{python} -s {test_file}\n".format(
+                    python=config.test_python,
+                    test_file=join(tmp_dir, 'run_test.py')))
+
+            if pl_files:
+                tf.write("{perl} {test_file}\n".format(
+                    python=config.test_perl,
+                    test_file=join(tmp_dir, 'run_test.pl')))
+
+            if lua_files:
+                tf.write("{lua} {test_file}\n".format(
+                    python=config.test_perl,
+                    test_file=join(tmp_dir, 'run_test.lua')))
+
+            if shell_files:
+                test_file = join(tmp_dir, 'run_test.' + suffix)
+                if on_win:
+                    tf.write("call {test_file}\n".format(test_file=test_file))
+                else:
+                    # TODO: Run the test/commands here instead of in run_test.py
+                    tf.write("{shell_path} -x -e {test_file}\n".format(shell_path=shell_path,
+                                                                       test_file=test_file))
+        if on_win:
+            cmd = [env["COMSPEC"], "/d", "/c", test_script]
+        else:
+            cmd = [shell_path, '-x', '-e', test_script]
+        try:
+            subprocess.check_call(cmd, env=env, cwd=tmp_dir)
+        except subprocess.CalledProcessError:
+            tests_failed(m, move_broken=move_broken)
 
     print("TEST END:", m.dist())
 
diff --git a/conda_build/environ.py b/conda_build/environ.py
--- a/conda_build/environ.py
+++ b/conda_build/environ.py
@@ -10,7 +10,8 @@
 import subprocess
 
 import conda.config as cc
-from conda.compat import text_type, PY3
+# noqa here because PY3 is used only on windows, and trips up flake8 otherwise.
+from conda.compat import text_type, PY3  # noqa
 
 from conda_build import external
 from conda_build import source
@@ -376,70 +377,6 @@ def system_vars(env_dict, prefix):
     return d
 
 
-# http://code.activestate.com/recipes/576644-diff-two-dictionaries/#c9
-def _dict_diff(d1, d2):
-    """Shows entries that have changed or been added in d2 relative to d1"""
-    both = set(d1.keys()) & set(d2.keys())
-    diff = {k: d2[k] for k in both if d1[k] != d2[k]}
-    diff.update({k: d2[k] for k in set(d2.keys()) - both})
-    return diff
-
-
-def _set_environ_from_subprocess_values(vars):
-    """
-    Vars is an unprocessed string of envrionment variable output, such as from ```set```
-    on Windows, or ```env``` elsewhere
-    """
-    start_environ = os.environ
-    vars = vars.split("\n")
-    modified_environ = {var.split("=")[0].strip(): var.split("=")[1].strip()
-                        for var in vars if "=" in var}
-
-    # the diff is the only part we'll set for the actual build environment - mind you,
-    #   the current process is not the actual build environment.  That is always a native
-    #   shell subprocess.
-    diff = _dict_diff(start_environ, modified_environ)
-
-    # modify the current process with the activated/deactivated values
-    for key, value in modified_environ.items():
-        os.environ[key] = value
-
-    return diff
-
-
-def activate_or_deactivate_env(action, env_name_or_path=""):
-    """
-    Strategy is to open a subprocess, run activate, record the variables,
-    then apply them in our process
-
-    action should be "activate" or "deactivate"
-    env_name_or_path should only be provided for activation.
-    """
-    if sys.platform == "win32":
-        cmd = '"{0}\\Scripts\\{1}.bat" "{2}" && set'.format(sys.prefix,
-                                                            action,
-                                                            env_name_or_path)
-        if PY3:
-            # this method simplifies dealing with str vs bytestring on Py3.
-            vars = subprocess.getoutput(cmd).replace("\r\n", "\n")
-        else:
-            vars = subprocess.check_output(cmd).replace("\r\n", "\n")
-    else:
-        cmd = "source {0}/bin/{1} {2} && env".format(sys.prefix, action, env_name_or_path)
-        vars = subprocess.check_output(["bash", "-c", cmd], env=os.environ)
-    if PY3 and hasattr(vars, "decode"):
-        vars = vars.decode("UTF-8")
-    return _set_environ_from_subprocess_values(vars)
-
-
-def activate_env(env_name_or_path):
-    return activate_or_deactivate_env("activate", env_name_or_path)
-
-
-def deactivate_env():
-    return activate_or_deactivate_env("deactivate")
-
-
 if __name__ == '__main__':
     e = get_dict()
     for k in sorted(e):
diff --git a/conda_build/windows.py b/conda_build/windows.py
--- a/conda_build/windows.py
+++ b/conda_build/windows.py
@@ -160,7 +160,7 @@ def build_vcvarsall_cmd(cmd, arch=arch_selector):
         # Visual Studio 14 or otherwise
         msvc_env_lines.append(build_vcvarsall_cmd(vcvarsall_vs_path))
 
-    return '\n'.join(msvc_env_lines)
+    return '\n'.join(msvc_env_lines) + '\n'
 
 
 def kill_processes(process_names=["msbuild.exe"]):
@@ -186,12 +186,8 @@ def kill_processes(process_names=["msbuild.exe"]):
             continue
 
 
-def build(m, bld_bat, dirty=False, env_diff=None):
-    if not env_diff:
-        env_diff = {}
+def build(m, bld_bat, dirty=False, activate=True):
     env = environ.get_dict(m, dirty=dirty)
-    # env_diff contains activation info, among any other env variables set by activation scripts
-    env.update(env_diff)
 
     for name in 'BIN', 'INC', 'LIB':
         path = env['LIBRARY_' + name]
@@ -210,6 +206,8 @@ def build(m, bld_bat, dirty=False, env_diff=None):
             fo.write("set INCLUDE={};%INCLUDE%\n".format(env["LIBRARY_INC"]))
             fo.write("set LIB={};%LIB%\n".format(env["LIBRARY_LIB"]))
             fo.write(msvc_env_cmd(bits=cc.bits, override=m.get_value('build/msvc_compiler', None)))
+            if activate:
+                fo.write("call activate _build\n")
             fo.write('\n')
             fo.write("REM ===== end generated header =====\n")
             fo.write(data)


Test patch:
diff --git a/tests/test-recipes/metadata/empty_sections/meta.yaml b/tests/test-recipes/metadata/empty_sections/meta.yaml
--- a/tests/test-recipes/metadata/empty_sections/meta.yaml
+++ b/tests/test-recipes/metadata/empty_sections/meta.yaml
@@ -2,9 +2,7 @@ package:
   name: empty_sections
   version: 0.0
 
-
 build:
 
 requirements:
   build:
-