Repository: conda/conda-build

Issue:
Conda build fails on automated build attempt
I am able to build g2pkg on my local mac, but when I try to move it to a remote machine, I run into the error shown below:
- /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/bin/conda build g2pkg
- source activate /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_
  ++ [[ -n 3.2.53(1)-release ]]
  ++ _SCRIPT_LOCATION=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin/activate
  ++ _SHELL=bash
  +++ dirname /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin/activate
  ++ _CONDA_DIR=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin
  ++ '[' 1 -gt 1 ']'
  ++ case "$(uname -s)" in
  +++ uname -s
  ++ EXT=
  ++ [[ -n 3.2.53(1)-release ]]
  +++ basename /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/conda-bld/work/conda_build.sh
  ++ [[ conda_build.sh == \a\c\t\i\v\a\t\e ]]
  ++ '[' 1 -eq 0 ']'
  ++ args=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_
  ++ /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin/conda ..checkenv bash /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_
  ++ ((  0 != 0  ))
  ++ source /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin/deactivate
  +++ [[ -n 3.2.53(1)-release ]]
  +++ _SCRIPT_LOCATION=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin/deactivate
  +++ _SHELL=bash
  ++++ dirname /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin/deactivate
  +++ _CONDA_DIR=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin
  +++ case "$(uname -s)" in
  ++++ uname -s
  +++ EXT=
  +++ [[ 1 > 0 ]]
  +++ key=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_
  +++ case $key in
  +++ shift
  +++ [[ 0 > 0 ]]
  +++ [[ -n 3.2.53(1)-release ]]
  ++++ basename /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/conda-bld/work/conda_build.sh
  +++ [[ conda_build.sh == \d\e\a\c\t\i\v\a\t\e ]]
  +++ [[ -z '' ]]
  +++ [[ -n 3.2.53(1)-release ]]
  ++++ basename /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/conda-bld/work/conda_build.sh
  +++ [[ conda_build.sh == \d\e\a\c\t\i\v\a\t\e ]]
  +++ return 0
  +++ /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin/conda ..activate bash /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_
  ++ _NEW_PART=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin
  ++ ((  0 == 0  ))
  ++ export CONDA_PATH_BACKUP=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin:/Users/jenkins/bin:/usr/local/bin:/opt/local/bin:/bin:/usr/bin:/usr/sbin:/sbin
  ++ CONDA_PATH_BACKUP=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin:/Users/jenkins/bin:/usr/local/bin:/opt/local/bin:/bin:/usr/bin:/usr/sbin:/sbin
  ++ export CONDA_PS1_BACKUP=
  ++ CONDA_PS1_BACKUP=
  ++ export PATH=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin:/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin:/Users/jenkins/bin:/usr/local/bin:/opt/local/bin:/bin:/usr/bin:/usr/sbin:/sbin
  ++ PATH=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin:/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin:/Users/jenkins/bin:/usr/local/bin:/opt/local/bin:/bin:/usr/bin:/usr/sbin:/sbin
  ++ [[ '' == _/_ ]]
  ++ export CONDA_DEFAULT_ENV=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_
  ++ CONDA_DEFAULT_ENV=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_
  ++ firstpath=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin
  +++ echo /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin
  +++ sed 's|/bin$||'
  ++ export CONDA_PREFIX=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_
  +++ /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin/conda ..changeps1
  ++ '[' 1 = 1 ']'
  +++ grep -q CONDA_DEFAULT_ENV
  ++ export 'PS1=(/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_) '
  ++ PS1='(/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_) '
  ++ _CONDA_D=/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/etc/conda/activate.d
  ++ [[ -d /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/etc/conda/activate.d ]]
  ++ unset CONDA_PATH
  ++ [[ -n 3.2.53(1)-release ]]
  ++ hash -r
- echo 'running build.sh'
  running build.sh
- mkdir -p /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin
- cp /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/GSAS-II/install/g2pkg/src/bootstrap.py /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs/_build_placehold_/bin/gsas2-install.py
  An unexpected error has occurred, please consider sending the
  following traceback to the conda GitHub issue tracker at:
  
  https://github.com/conda/conda-build/issues

Include the output of the command 'conda info' in your report.

Removing old build environment
BUILD START: g2load-0.1.8-0
    (actual version deferred until further download or env creation)
Copying /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/GSAS-II/install/g2pkg/src to /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/conda-bld/work
BUILD START: g2load-0.1.8-0
Package: g2load-0.1.8-0
source tree in: /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/conda-bld/work
compiling .pyc files...
Traceback (most recent call last):
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/bin/conda-build", line 5, in <module>
    sys.exit(main())
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/lib/python2.7/site-packages/conda_build/main_build.py", line 152, in main
    args_func(args, p)
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/lib/python2.7/site-packages/conda_build/main_build.py", line 415, in args_func
    args.func(args, p)
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/lib/python2.7/site-packages/conda_build/main_build.py", line 358, in execute
    debug=args.debug)
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/lib/python2.7/site-packages/conda_build/build.py", line 646, in build
    preserve_egg_dir=bool(m.get_value('build/preserve_egg_dir')))
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/lib/python2.7/site-packages/conda_build/post.py", line 182, in post_process
    compile_missing_pyc(files)
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/lib/python2.7/site-packages/conda_build/post.py", line 174, in compile_missing_pyc
    call([python_exe, '-Wi', '-m', 'py_compile'] + compile_files, cwd=cwd)
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/lib/python2.7/subprocess.py", line 523, in call
    return Popen(_popenargs, *_kwargs).wait()
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/lib/python2.7/subprocess.py", line 711, in **init**
    errread, errwrite)
  File "/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/lib/python2.7/subprocess.py", line 1343, in _execute_child
    raise child_exception
OSError: [Errno 2] No such file or directory
- /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/bin/conda info
  Current conda install:
  
  ```
       platform : osx-64
  conda version : 4.1.11
  ```
  
  conda-env version : 2.5.1
  conda-build version : 1.21.11+0.g5b44ab3.dirty
       python version : 2.7.12.final.0
     requests version : 2.10.0
     root environment : /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2  (writable)
  default environment : /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2
     envs directories : /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/envs
        package cache : /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/pkgs
         channel URLs : https://repo.continuum.io/pkgs/free/osx-64/
                        https://repo.continuum.io/pkgs/free/noarch/
                        https://repo.continuum.io/pkgs/pro/osx-64/
                        https://repo.continuum.io/pkgs/pro/noarch/
          config file : None
         offline mode : False
    is foreign system : False
- /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2/bin/conda list
  # packages in environment at /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/conda2:
  # 
  
  anaconda-client           1.5.1                    py27_0  
  clyent                    1.2.2                    py27_0  
  conda                     4.1.11                   py27_0  
  conda-build               1.21.11                  py27_0  
  conda-env                 2.5.1                    py27_0  
  constructor               1.3.1                    py27_0  
  jinja2                    2.8                      py27_1  
  libconda                  4.0.0                    py27_0  
  markupsafe                0.23                     py27_2  
  openssl                   1.0.2h                        1  
  pip                       8.1.2                    py27_0  
  pycosat                   0.6.1                    py27_1  
  pycrypto                  2.6.1                    py27_4  
  python                    2.7.12                        1  
  python-dateutil           2.5.3                    py27_0  
  pytz                      2016.6.1                 py27_0  
  pyyaml                    3.11                     py27_4  
  readline                  6.2                           2  
  requests                  2.10.0                   py27_0  
  ruamel_yaml               0.11.14                  py27_0  
  setuptools                23.0.0                   py27_0  
  six                       1.10.0                   py27_0  
  sqlite                    3.13.0                        0  
  tk                        8.5.18                        0  
  wheel                     0.29.0                   py27_0  
  yaml                      0.1.6                         0  
  zlib                      1.2.8                         3  
- ls -lR /Users/jenkins/workspace/GSAS-II-Mac64-Constructor/GSAS-II/install/g2pkg
  total 40
  -rw-r--r--  1 jenkins  staff  230 Aug  7 17:18 bld.bat
  -rw-r--r--  1 jenkins  staff  118 Aug  7 17:18 build.sh
  -rw-r--r--  1 jenkins  staff  178 Aug  7 17:18 meta.yaml
  -rw-r--r--  1 jenkins  staff  288 Aug  7 17:18 post-link.bat
  -rw-r--r--  1 jenkins  staff  227 Aug  7 17:18 post-link.sh
  drwxr-xr-x  4 jenkins  staff  136 Aug  7 17:18 src

/Users/jenkins/workspace/GSAS-II-Mac64-Constructor/GSAS-II/install/g2pkg/src:
total 16
-rw-r--r--  1 jenkins  staff  7084 Aug  7 17:18 bootstrap.py



Discussion:
Just to be clear - are all of the versions the same on both of your systems?  Conda-build 1.21.11 in both places?

oops. No. This fails in conda-build-1.21.11 on both machines. 

It succeeds with conda-build 1.19.2, but fails with both conda-build 1.20.3 and conda-build 1.21.11. 

In case it is of help, the source is in https://subversion.xray.aps.anl.gov/pyGSAS/install (g2pkg)

Thanks.  Having the source available should help a lot.  I'll see what I can find.


Resolving patch:
diff --git a/conda_build/__init__.py b/conda_build/__init__.py
--- a/conda_build/__init__.py
+++ b/conda_build/__init__.py
@@ -20,4 +20,3 @@
     'sign',
     'skeleton',
 ]
-
diff --git a/conda_build/post.py b/conda_build/post.py
--- a/conda_build/post.py
+++ b/conda_build/post.py
@@ -1,28 +1,28 @@
 from __future__ import absolute_import, division, print_function
 
+from collections import defaultdict
+from glob import glob
+import io
 import locale
+import mmap
 import re
 import os
-import sys
-import stat
-from glob import glob
 from os.path import (basename, dirname, join, splitext, isdir, isfile, exists,
                      islink, realpath, relpath, normpath)
+import stat
+from subprocess import call
+import sys
 try:
     from os import readlink
 except ImportError:
     readlink = False
-import io
-from subprocess import call
-from collections import defaultdict
-import mmap
 
 from conda_build.config import config
 from conda_build import external
 from conda_build import environ
 from conda_build import utils
 from conda_build import source
-from conda.compat import lchmod, PY3
+from conda.compat import lchmod
 from conda.misc import walk_prefix
 from conda.utils import md5_file
 
@@ -168,12 +168,24 @@ def coerce_pycache_to_old_style(files, cwd):
 
 
 def compile_missing_pyc(files, cwd=config.build_prefix, python_exe=config.build_python):
-    compile_files = [f for f in files if f.endswith('.py') and f + 'c' not in files]
+    compile_files = []
+    for fn in files:
+        # omit files in Library/bin, Scripts, and the root prefix - they are not generally imported
+        if sys.platform == 'win32':
+            if any([fn.lower().startswith(start) for start in ['library/bin', 'library\\bin',
+                                                               'scripts']]):
+                continue
+        else:
+            if fn.startswith('bin'):
+                continue
+        if fn.endswith(".py"):
+            compile_files.append(fn)
+
     if compile_files:
         print('compiling .pyc files...')
-        call([python_exe, '-Wi', '-m', 'py_compile'] + compile_files, cwd=cwd)
-        if PY3:
-            coerce_pycache_to_old_style(compile_files, cwd=cwd)
+        for f in compile_files:
+            call([python_exe, '-Wi', '-m', 'py_compile', f], cwd=cwd)
+        coerce_pycache_to_old_style(compile_files, cwd=cwd)
 
 
 def post_process(files, preserve_egg_dir=False):


Test patch:
diff --git a/tests/test-recipes/metadata/jinja2_build_str_template_only/bld.bat b/tests/test-recipes/metadata/jinja2_build_str_template_only/bld.bat
deleted file mode 100644
--- a/tests/test-recipes/metadata/jinja2_build_str_template_only/bld.bat
+++ /dev/null
@@ -1,10 +0,0 @@
-set PATH
-
-rem check that GIT_* tags are present
-for %%i in (GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH) DO (
-  if defined %%i (
-      echo %%i
-  ) else (
-    exit 1
-  )
-)
diff --git a/tests/test-recipes/metadata/jinja2_build_str_template_only/build.sh b/tests/test-recipes/metadata/jinja2_build_str_template_only/build.sh
deleted file mode 100644
--- a/tests/test-recipes/metadata/jinja2_build_str_template_only/build.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-# check if GIT_* variables are defined
-for i in GIT_DESCRIBE_TAG GIT_DESCRIBE_NUMBER GIT_DESCRIBE_HASH GIT_FULL_HASH
-do
-  if [ -n "eval $i" ]; then
-    eval echo \$$i
-  else
-    exit 1
-  fi
-done
diff --git a/tests/test-recipes/metadata/jinja2_build_str_template_only/meta.yaml b/tests/test-recipes/metadata/jinja2_build_str_template_only/meta.yaml
deleted file mode 100644
--- a/tests/test-recipes/metadata/jinja2_build_str_template_only/meta.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-package:
-  name: conda-build-test-source-git-jinja2-2
-  version: 1.0
-
-source:
-  git_url: ../../../../
-
-build:
-  string: {{ PKG_BUILDNUM }}_g{{ GIT_FULL_HASH[:7] }}
-
-requirements:
-  build:
-    # To test the conda_build version
-    - python
-    - git