Repository: conda/conda-build

Issue:
can't run conda-build from central install without write access 
For a centrally installed conda, where users don't have write access, they can't run conda-build.

Running 
```
conda-build recipe-dir
```
ends with
```
OSError: [Errno 13] Permission denied: '/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv'
```
while running 
```
conda-build --no-locking /reg/g/psdm/sw/conda/manage/recipes/external/szip
```
ends with
```
...
number of files: 8
Fixing permissions
Making absolute symlink lib/libsz.so.2 -> libsz.so.2.0.0 relative
Making absolute symlink lib/libsz.so -> libsz.so.2.0.0 relative
patchelf: file: /reg/neh/home/davidsch/conda-bld/szip_1490722924710/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placeh/lib/libsz.so.2.0.0
    setting rpath to: $ORIGIN/.
Fixing permissions
Detected hard-coded path in text file lib/libsz.la
/reg/neh/home/davidsch/conda-bld/linux-64/szip-2.1-102.tar.bz2
updating index in: /reg/neh/home/davidsch/conda-bld
Traceback (most recent call last):
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/bin/conda-build", line 6, in <module>
    sys.exit(conda_build.cli.main_build.main())
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 334, in main
    execute(sys.argv[1:])
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 325, in execute
    noverify=args.no_verify)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/api.py", line 97, in build
    need_source_download=need_source_download, config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1502, in build_tree
    config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1145, in build
    built_package = bundlers[output_dict.get('type', 'conda')](output_dict, m, config, env)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 940, in bundle_conda
    update_index(os.path.dirname(output_folder), config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/index.py", line 145, in update_index
    write_repodata(repodata, dir_path, lock=lock, config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/index.py", line 43, in write_repodata
    with try_acquire_locks(locks, config.timeout):
UnboundLocalError: local variable 'locks' referenced before assignment
(ana-1.2.4) psanaphi104: ~/rel/anarel-test $ 
```
My version of conda and conda-build is
```
 conda info
Current conda install:

               platform : linux-64
          conda version : 4.3.14
       conda is private : False
      conda-env version : 4.3.14
    conda-build version : 2.1.8
         python version : 2.7.12.final.0
       requests version : 2.12.4
       root environment : /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7  (read only)
    default environment : /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/envs/ana-1.2.4
       envs directories : /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/envs
                          /reg/neh/home/davidsch/.conda/envs
          package cache : /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/pkgs
                          /reg/neh/home/davidsch/.conda/pkgs
           channel URLs : file:///reg/g/psdm/sw/conda/channels/system-rhel7/linux-64
                          file:///reg/g/psdm/sw/conda/channels/system-rhel7/noarch
                          file:///reg/g/psdm/sw/conda/channels/psana-rhel7/linux-64
                          file:///reg/g/psdm/sw/conda/channels/psana-rhel7/noarch
                          file:///reg/g/psdm/sw/conda/channels/external-rhel7/linux-64
                          file:///reg/g/psdm/sw/conda/channels/external-rhel7/noarch
                          https://repo.continuum.io/pkgs/free/linux-64
                          https://repo.continuum.io/pkgs/free/noarch
                          https://repo.continuum.io/pkgs/r/linux-64
                          https://repo.continuum.io/pkgs/r/noarch
                          https://repo.continuum.io/pkgs/pro/linux-64
                          https://repo.continuum.io/pkgs/pro/noarch
                          https://conda.anaconda.org/conda-forge/linux-64
                          https://conda.anaconda.org/conda-forge/noarch
                          https://conda.anaconda.org/kalefranz/linux-64
                          https://conda.anaconda.org/kalefranz/noarch
                          https://conda.anaconda.org/jjhelmus/linux-64
                          https://conda.anaconda.org/jjhelmus/noarch
                          file:///reg/g/psdm/sw/conda/channels/testing-rhel7/linux-64
                          file:///reg/g/psdm/sw/conda/channels/testing-rhel7/noarch
            config file : /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/.condarc
           offline mode : False
             user-agent : conda/4.3.14 requests/2.12.4 CPython/2.7.12 Linux/3.19.8-1.el7.x86_64 Red Hat Enterprise Linux Server/7.2 glibc/2.17
                UID:GID : 12658:1111

```


Discussion:
I realized that I had done something to try to work around this, namely giving everybody read/write access to the `locks` subdirectory of the conda installation. Conda-build is able to create the lock files, but it is at the end when it tries to acquire it, that it fails, here is the end of the output of a conda-build command without --no-locking
```
make[2]: Leaving directory `/reg/neh/home4/davidsch/conda-bld/szip_1490906972393/work/szip-2.1'
make[1]: Leaving directory `/reg/neh/home4/davidsch/conda-bld/szip_1490906972393/work/szip-2.1'
INFO conda_build.build:bundle_conda(864): Packaging szip-2.1-102
number of files: 8
Fixing permissions
Making absolute symlink lib/libsz.so.2 -> libsz.so.2.0.0 relative
Making absolute symlink lib/libsz.so -> libsz.so.2.0.0 relative
patchelf: file: /reg/neh/home/davidsch/conda-bld/szip_1490906972393/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placeh/lib/libsz.so.2.0.0
    setting rpath to: $ORIGIN/.
Fixing permissions
Traceback (most recent call last):
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/bin/conda-build", line 6, in <module>
    sys.exit(conda_build.cli.main_build.main())
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 334, in main
    execute(sys.argv[1:])
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 325, in execute
    noverify=args.no_verify)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/api.py", line 97, in build
    need_source_download=need_source_download, config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1502, in build_tree
    config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1145, in build
    built_package = bundlers[output_dict.get('type', 'conda')](output_dict, m, config, env)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 886, in bundle_conda
    create_info_files(metadata, files, config=config, prefix=config.build_prefix)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 493, in create_info_files
    copy_recipe(m, config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 205, in copy_recipe
    utils.copy_into(src_path, dst_path, timeout=config.timeout, locking=config.locking)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/utils.py", line 167, in copy_into
    with try_acquire_locks(locks, timeout):
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/contextlib.py", line 17, in __enter__
    return self.gen.next()
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/utils.py", line 105, in try_acquire_locks
    lock.acquire(timeout=0.1)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/filelock.py", line 238, in acquire
    self._acquire()
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/filelock.py", line 358, in _acquire
    fd = os.open(self._lock_file, open_mode)
OSError: [Errno 13] Permission denied: '/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv'
(ana-1.2.5) pslogin7d: /reg/neh/home4/davidsch $ cat /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv
(ana-1.2.5) pslogin7d: /reg/neh/home4/davidsch $ rm /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv
rm: remove write-protected regular empty file '/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv'? y
(ana-1.2.5) pslogin7d: /reg/neh/home4/davidsch $ 
```
The last few times I have tried this, it is working, it is strange.
I'm leaving this open for now - please update if you are able to come up with reproduction steps.  There should be a fallback path to the user's home folder.  See https://github.com/conda/conda-build/pull/1749 for more details.
That fix looks like it should fix it, I just reproduced it again, but I think it is because I made the central install locks subdir world writable. I got 
```
OSError: [Errno 13] Permission denied: '/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv'
```
but the file is there, but it is owned by another user:
```
(ana-1.2.5) pslogin7d: /reg/neh/home4/davidsch $ ls -lrth /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv
-rw-r--r-- 1 jscott xr 0 Mar 30 16:05 /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv
```
so I guess there is ont that much entropy in in the filenames, 

I just made that locks subdir unwritable by others. conda-build goes to the end, but then it is still trying  to reference a file in the central install
```
OSError: [Errno 13] Permission denied: '/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv'
```
With conda-build 2.1.9, we find that if the central install locks directory is empty, than users can run conda-build. However if there is something there, it often fails -- but not always. I as the admin did a conda-build recently, of 'szip', it finished fine, but it left several files in the locks directory:
```
(manage) (psreldev) psel701: /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7 $ ls -lrth locks/
total 2.0K
-rw-r--r-- 1 psreldev xs 0 Apr 11 20:45 L3JlZy9nL3BzZG0vc3cv
-rw-r--r-- 1 psreldev xs 0 Apr 11 20:50 L3RtcC90bXAyNlBOVjI=
-rw-r--r-- 1 psreldev xs 0 Apr 11 20:50 L3RtcC90bXBLQU4wZUQv
-rw-r--r-- 1 psreldev xs 0 Apr 11 21:05 L3RtcC90bXBXdFFnSWk=
```
now a user, specifically @tjlane, is getting an error that I can reproduce. If I as a user run conda-build to build that same package (szip), it works fine, but for a different package, namely a copy of TJ's, I get
```
OSError: [Errno 13] Permission denied: '/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/locks/L3JlZy9nL3BzZG0vc3cv'
```
This seems like a bug, but maybe the bug is that conda-build did not clean up those locks when the admin finished running it?
I wonder if you could explain these locks?

I think you could reproduce this by creating the lock file with that specific name maybe? That is do
```
touch /path/to/miniconda2/locks/L3JlZy9nL3BzZG0vc3cv
```
and then, from a account without write access, create a directory called conda-recipe and put this meta.yaml file in it: 
```
package:
  name:    pscache
  version: 0.0.1

source:
  git_url: https://github.com/lcls-psana/pscache

requirements:
  build:
    - python
    - setuptools
    - redis
    - numpy
    - psana-conda

  run:
    - python
    - redis
    - numpy
    - psana-conda
```
then do
```
conda-build conda-recipe
```
it should crash right away - with the OSError.

When I put on no-locking, I get other errors, but it might be a separate issue. I don't know if you can reproduce that, but you could try
```
conda-build --no-locking -c lcls-rhel7 -c defaults -c conda-forge conda-recipe
```
that is you need our special channel lcls-rhel7 for the psana-conda dependency, and then we pick things up from defaults and lastly conda-forge -- trying the above might be a waste of time since nobody has done that, it should work, but we have a setup here with a special .condarc and local file channels - that probably aren't up to date with lcls-rhel7.

Anyways, the final ouput is below, the issue might be that conda-build assumes there will be a git_rev given for a specific tag, we're just trying to get things working now, so no tag yet:

```
(ana-1.2.6) pslogin7d: ~/users/tjlane/pscache $ conda-build --no-locking conda-recipe
BUILD START: pscache-0.0.1-py27_0

The following NEW packages will be INSTALLED:

    backports:          1.0-py27_0             defaults                                         
    backports_abc:      0.5-py27_0             defaults                                         
    boost:              1.57.0-4               defaults                                         
    cycler:             0.10.0-py27_0          defaults                                         
    fontconfig:         2.12.1-4               conda-forge                                      
    freetype:           2.7-1                  conda-forge                                      
    functools32:        3.2.3.2-py27_0         defaults                                         
    h5py:               2.7.0rc2-py27_2        file:///reg/g/psdm/sw/conda/channels/system-rhel7
    hdf5:               1.8.18-openmpi_2       file:///reg/g/psdm/sw/conda/channels/system-rhel7
    icu:                58.1-1                 conda-forge                                      
    libgfortran:        3.0.0-1                defaults                                         
    libiconv:           1.14-0                 defaults                                         
    libpng:             1.6.28-0               conda-forge                                      
    libsodium:          1.0.10-0               defaults                                         
    libxml2:            2.9.4-0                defaults                                         
    matplotlib:         2.0.0-np112py27_2      conda-forge                                      
    mkl:                2017.0.1-0             defaults                                         
    mpi4py:             2.0.0-py27_openmpi_103 file:///reg/g/psdm/sw/conda/channels/system-rhel7
    mysql:              5.5.24-0               defaults                                         
    numexpr:            2.6.2-np112py27_0      defaults                                         
    numpy:              1.12.1-py27_0          defaults                                         
    openmpi:            2.0.2-lsf_verbs_1      file:///reg/g/psdm/sw/conda/channels/system-rhel7
    openssl:            1.0.2k-1               defaults                                         
    pip:                9.0.1-py27_1           defaults                                         
    psana-conda:        1.2.6-py27_1           file:///reg/g/psdm/sw/conda/channels/psana-rhel7 
    pyparsing:          2.1.4-py27_0           defaults                                         
    pyqt:               4.11.4-py27_4          defaults                                         
    pytables:           3.3.0-py27_105         file:///reg/g/psdm/sw/conda/channels/system-rhel7
    python:             2.7.12-1               defaults                                         
    python-dateutil:    2.6.0-py27_0           defaults                                         
    pytz:               2017.2-py27_0          defaults                                         
    pyzmq:              16.0.2-py27_0          defaults                                         
    qt:                 4.8.7-3                defaults                                         
    readline:           6.2-2                  defaults                                         
    redis:              3.2.0-0                defaults                                         
    scipy:              0.19.0-np112py27_0     defaults                                         
    setuptools:         27.2.0-py27_0          defaults                                         
    singledispatch:     3.4.0.3-py27_0         defaults                                         
    sip:                4.18-py27_0            defaults                                         
    six:                1.10.0-py27_0          defaults                                         
    sqlite:             3.13.0-0               defaults                                         
    ssl_match_hostname: 3.4.0.2-py27_1         defaults                                         
    subprocess32:       3.2.7-py27_0           defaults                                         
    szip:               2.1-102                local                                            
    tk:                 8.5.18-0               defaults                                         
    tornado:            4.4.2-py27_0           defaults                                         
    wheel:              0.29.0-py27_0          defaults                                         
    zeromq:             4.1.5-0                defaults                                         
    zlib:               1.2.8-3                defaults                                         

Cloning into bare repository '/reg/neh/home/davidsch/conda-bld/git_cache/github.com/lcls-psana/pscache'...
remote: Counting objects: 91, done.
remote: Compressing objects: 100% (31/31), done.
remote: Total 91 (delta 10), reused 0 (delta 0), pack-reused 60
Unpacking objects: 100% (91/91), done.
Cloning into '/reg/neh/home/davidsch/conda-bld/pscache_1492014497347/work'...
done.
checkout: 'HEAD'
Your branch is up-to-date with 'origin/master'.
==> git log -n1 <==

commit a7d004d9df61fadc4806ee08f157018cc61b7aea
Author: TJ Lane <tjlane@stanford.edu>
Date:   Sat Apr 8 10:14:34 2017 -0700

    fix no data bug

fatal: No names found, cannot describe anything.
==> git describe --tags --dirty <==

commit a7d004d9df61fadc4806ee08f157018cc61b7aea
Author: TJ Lane <tjlane@stanford.edu>
Date:   Sat Apr 8 10:14:34 2017 -0700

    fix no data bug

==> git status <==

On branch master
Your branch is up-to-date with 'origin/master'.
nothing to commit, working tree clean

fatal: No names found, cannot describe anything.
WARNING conda_build.environ:get_git_info(167): Failed to obtain git tag information.  Are you using annotated tags?
fatal: No names found, cannot describe anything.
WARNING conda_build.environ:get_git_info(167): Failed to obtain git tag information.  Are you using annotated tags?
fatal: No names found, cannot describe anything.
WARNING conda_build.environ:get_git_info(167): Failed to obtain git tag information.  Are you using annotated tags?
fatal: No names found, cannot describe anything.
WARNING conda_build.environ:get_git_info(167): Failed to obtain git tag information.  Are you using annotated tags?
fatal: No names found, cannot describe anything.
WARNING conda_build.environ:get_git_info(167): Failed to obtain git tag information.  Are you using annotated tags?
fatal: No names found, cannot describe anything.
WARNING conda_build.environ:get_git_info(167): Failed to obtain git tag information.  Are you using annotated tags?
Package: pscache-0.0.1-py27_0
source tree in: /reg/neh/home/davidsch/conda-bld/pscache_1492014497347/work
INFO conda_build.build:bundle_conda(864): Packaging pscache-0.0.1-py27_0
number of files: 0
Fixing permissions
Fixing permissions
Traceback (most recent call last):
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/bin/conda-build", line 6, in <module>
    sys.exit(conda_build.cli.main_build.main())
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 334, in main
    execute(sys.argv[1:])
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 325, in execute
    noverify=args.no_verify)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/api.py", line 97, in build
    need_source_download=need_source_download, config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1503, in build_tree
    config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1146, in build
    built_package = bundlers[output_dict.get('type', 'conda')](output_dict, m, config, env)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 887, in bundle_conda
    create_info_files(metadata, files, config=config, prefix=config.build_prefix)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 494, in create_info_files
    copy_license(m, config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 272, in copy_license
    locking=config.locking)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/utils.py", line 177, in copy_into
    _copy_with_shell_fallback(src, dst_fn)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/utils.py", line 135, in _copy_with_shell_fallback
    raise OSError("Failed to copy {} to {}.  Error was: {}".format(src, dst, e))
OSError: Failed to copy /reg/neh/home/davidsch/conda-bld/pscache_1492014497347/work/None to /reg/neh/home/davidsch/conda-bld/pscache_1492014497347/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/info/LICENSE.txt.  Error was: Command 'cp -a /reg/neh/home/davidsch/conda-bld/pscache_1492014497347/work/None /reg/neh/home/davidsch/conda-bld/pscache_1492014497347/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/info/LICENSE.txt' returned non-zero exit status 1
```

I just testing --no-locking with conda-build 2.1.9 on a working recipe, and it fails with 
```
Traceback (most recent call last):
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/bin/conda-build", line 6, in <module>
    sys.exit(conda_build.cli.main_build.main())
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 334, in main
    execute(sys.argv[1:])
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 325, in execute
    noverify=args.no_verify)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/api.py", line 97, in build
    need_source_download=need_source_download, config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1503, in build_tree
    config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1146, in build
    built_package = bundlers[output_dict.get('type', 'conda')](output_dict, m, config, env)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 941, in bundle_conda
    update_index(os.path.dirname(output_folder), config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/index.py", line 145, in update_index
    write_repodata(repodata, dir_path, lock=lock, config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/index.py", line 43, in write_repodata
    with try_acquire_locks(locks, config.timeout):
UnboundLocalError: local variable 'locks' referenced before assignment
```
This is a lightweight package you could test with, the recipe is at

https://github.com/slaclab/anarel-manage/blob/master/recipes/psana/ndarray/meta.yaml

do if you make a conda-recipe directory with the file
```
package:
  name: ndarray
  version: 1.1.8

source:
  git_url: https://github.com/lcls-psana/ndarray.git
  git_rev: V01-01-08

build:
  number: 0

requirements:
  build:
    - boost
  run:
    - boost
```
and do 
```
conda-build --no-locking recipe-dir
```
you should get output like
```
(ana-1.2.6) pslogin7d: ~/users/tjlane $ conda-build --no-locking /reg/g/psdm/sw/conda/manage/recipes/psana/ndarray
BUILD START: ndarray-1.1.8-0

The following NEW packages will be INSTALLED:

    boost:      1.61.0-py36_0 defaults
    icu:        54.1-0        defaults
    openssl:    1.0.2k-1      defaults
    pip:        9.0.1-py36_1  defaults
    python:     3.6.1-0       defaults
    readline:   6.2-2         defaults
    setuptools: 27.2.0-py36_0 defaults
    sqlite:     3.13.0-0      defaults
    tk:         8.5.18-0      defaults
    wheel:      0.29.0-py36_0 defaults
    xz:         5.2.2-1       defaults
    zlib:       1.2.8-3       defaults

^[[5;5~Cloning into bare repository '/reg/neh/home/davidsch/conda-bld/git_cache/github.com/lcls-psana/ndarray.git'...
remote: Counting objects: 10211, done.
remote: Total 10211 (delta 0), reused 0 (delta 0), pack-reused 10211
Receiving objects: 100% (10211/10211), 2.21 MiB | 3.56 MiB/s, done.
Resolving deltas: 100% (94/94), done.
Cloning into '/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/work'...
done.
checkout: u'V01-01-08'
Note: checking out 'V01-01-08'.

You are in 'detached HEAD' state. You can look around, make experimental
changes and commit them, and you can discard any commits you make in this
state without impacting any branches by performing another checkout.

If you want to create a new branch to retain commits you create, you may
do so (now or later) by using -b with the checkout command again. Example:

  git checkout -b <new-branch-name>

HEAD is now at 2d3832a... remove comment in deprecated attribute for rhel5/6
==> git log -n1 <==

commit 2d3832ac6e4356567fb9061f03a3dfee7254b54d
Author: Clemens Weninger <clemens@m-info.de>
Date:   Wed Mar 15 20:17:47 2017 -0700

    remove comment in deprecated attribute for rhel5/6

==> git describe --tags --dirty <==

V01-01-08

==> git status <==

HEAD detached at V01-01-08
nothing to commit, working tree clean

Package: ndarray-1.1.8-0
source tree in: /reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/work
+ source /reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/bin/activate /reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla
+ echo '######## env ############'
######## env ############
+ env
DIRTY=
CPU_COUNT=2
PYTHONNOUSERSITE=1
GIT_DESCRIBE_NUMBER=0
PERL=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/bin/perl
GIT_FULL_HASH=2d3832ac6e4356567fb9061f03a3dfee7254b54d
PERL_VER=5.20.3
R=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/bin/R
GIT_DESCRIBE_HASH=g2d3832a
CMAKE_GENERATOR=Unix Makefiles
LUA_INCLUDE_DIR=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/include
PY_VER=2.7
CONDA_BUILD_STATE=BUILD
GIT_DESCRIBE_TAG=V01-01-08
PKG_BUILDNUM=0
CONDA_BUILD=1
PATH=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/bin:/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla:/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/bin:/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/envs/ana-1.2.6/bin:/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/bin:/reg/g/psdm/sw/conda/manage/bin:/usr/lib64/qt-3.3/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/reg/neh/home/davidsch/bin:/reg/neh/home/davidsch/scripts:/reg/neh/home/davidsch/sw/external/install/valgrind/bin
CONDA_PATH_BACKUP=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla:/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/bin:/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/envs/ana-1.2.6/bin:/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/bin:/reg/g/psdm/sw/conda/manage/bin:/usr/lib64/qt-3.3/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/reg/neh/home/davidsch/bin:/reg/neh/home/davidsch/scripts:/reg/neh/home/davidsch/sw/external/install/valgrind/bin
SRC_DIR=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/work
CONDA_PREFIX=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla
LD_RUN_PATH=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/lib
PWD=/reg/neh/home4/davidsch/conda-bld/ndarray_1492016170221/work
LUA=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/bin/lua
PY3K=0
PKG_VERSION=1.1.8
LUA_VER=5.2
STDLIB_DIR=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/lib/python3.6
LANG=C
CONDA_PY=27
SHLIB_EXT=.so
PS1=(/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla) 
ROOT=/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7
HTTPS_PROXY=
SHLVL=1
HOME=/reg/neh/home/davidsch
PKG_BUILD_STRING=0
CONDA_PS1_BACKUP=
HTTP_PROXY=
SP_DIR=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/lib/python3.6/site-packages
SYS_PREFIX=/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7
SUBDIR=linux-64
PREFIX=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla
RECIPE_DIR=/reg/g/psdm/sw/conda/manage/recipes/psana/ndarray
PKG_CONFIG_PATH=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/lib/pkgconfig
CONDA_DEFAULT_ENV=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla
ARCH=64
GIT_BUILD_STR=0_g2d3832a
SYS_PYTHON=/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/bin/python
PKG_NAME=ndarray
PYTHON=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/bin/python
_=/usr/bin/env
+ echo '#########################'
#########################
+ export DESTDIR=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/include
+ DESTDIR=/reg/neh/home/davidsch/conda-bld/ndarray_1492016170221/_b_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/include
+ make
all is done
+ make install
INFO conda_build.build:bundle_conda(864): Packaging ndarray-1.1.8-0
number of files: 5
Fixing permissions
Fixing permissions
/reg/neh/home/davidsch/conda-bld/linux-64/ndarray-1.1.8-0.tar.bz2
updating index in: /reg/neh/home/davidsch/conda-bld
Traceback (most recent call last):
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/bin/conda-build", line 6, in <module>
    sys.exit(conda_build.cli.main_build.main())
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 334, in main
    execute(sys.argv[1:])
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/cli/main_build.py", line 325, in execute
    noverify=args.no_verify)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/api.py", line 97, in build
    need_source_download=need_source_download, config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1503, in build_tree
    config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 1146, in build
    built_package = bundlers[output_dict.get('type', 'conda')](output_dict, m, config, env)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/build.py", line 941, in bundle_conda
    update_index(os.path.dirname(output_folder), config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/index.py", line 145, in update_index
    write_repodata(repodata, dir_path, lock=lock, config=config)
  File "/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/lib/python2.7/site-packages/conda_build/index.py", line 43, in write_repodata
    with try_acquire_locks(locks, config.timeout):
UnboundLocalError: local variable 'locks' referenced before assignment
```


Resolving patch:
diff --git a/conda_build/build.py b/conda_build/build.py
--- a/conda_build/build.py
+++ b/conda_build/build.py
@@ -616,6 +616,12 @@ def create_info_files_json_v1(m, info_dir, prefix, files, files_with_prefix):
 def get_build_index(config, clear_cache=True):
     # priority: local by croot (can vary), then channels passed as args,
     #     then channels from config.
+    native_dirs = [os.path.join(config.croot, dirname) for dirname in (config.subdir, 'noarch')]
+    for dirname in native_dirs:
+        if not os.path.isdir(dirname):
+            os.makedirs(dirname)
+        if not os.path.isfile(os.path.join(dirname, 'repodata.json.bz2')):
+            update_index(dirname, config)
     urls = [url_path(config.croot)] + list(config.channel_urls)
     index = get_index(channel_urls=urls,
                       prepend=not config.override_channels,
@@ -662,18 +668,7 @@ def create_env(prefix, specs, config, clear_cache=True, retry=0):
                 locks = []
                 try:
                     if config.locking:
-                        _pkgs_dirs = pkgs_dirs[:1]
-                        locked_folders = _pkgs_dirs + list(config.bldpkgs_dirs)
-                        for folder in locked_folders:
-                            if not os.path.isdir(folder):
-                                os.makedirs(folder)
-                            lock = utils.get_lock(folder, timeout=config.timeout)
-                            if not folder.endswith('pkgs'):
-                                update_index(folder, config=config, lock=lock,
-                                             could_be_mirror=False)
-                            locks.append(lock)
-                        # lock used to generally indicate a conda operation occurring
-                        locks.append(utils.get_lock('conda-operation', timeout=config.timeout))
+                        locks = utils.get_conda_operation_locks(config)
 
                     with utils.try_acquire_locks(locks, timeout=config.timeout):
                         index = get_build_index(config=config, clear_cache=True)
@@ -1566,6 +1561,10 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False,
                                     .format(recipe) + error_str + "\n" + extra_help)
             retried_recipes.append(recipe)
             recipe_list.extendleft(add_recipes)
+        finally:
+            # clean up locks to avoid permission errors when they exist in central installs
+            for lock in utils.get_conda_operation_locks(metadata.config):
+                utils.rm_rf(lock.lock_file)
 
     if post in [True, None]:
         # TODO: could probably use a better check for pkg type than this...
@@ -1573,6 +1572,7 @@ def build_tree(recipe_list, config, build_only=False, post=False, notest=False,
         wheels = [f for f in built_packages if f.endswith('.whl')]
         handle_anaconda_upload(tarballs, config=config)
         handle_pypi_upload(wheels, config=config)
+
     return built_packages
 
 
diff --git a/conda_build/config.py b/conda_build/config.py
--- a/conda_build/config.py
+++ b/conda_build/config.py
@@ -15,7 +15,7 @@
 from .conda_interface import root_dir, root_writable, subdir, cc_platform, cc_conda_build
 from .conda_interface import string_types, binstar_upload
 
-from .utils import get_build_folders, rm_rf
+from .utils import get_build_folders, rm_rf, get_conda_operation_locks
 
 on_win = (sys.platform == 'win32')
 DEFAULT_PREFIX_LENGTH = 255
@@ -409,16 +409,22 @@ def test_dir(self):
         _ensure_dir(path)
         return path
 
-    def clean(self):
+    def clean(self, remove_folders=True):
         # build folder is the whole burrito containing envs and source folders
         #   It will only exist if we download source, or create a build or test environment
-        if self.build_id:
-            if os.path.isdir(self.build_folder):
-                rm_rf(self.build_folder)
-        else:
-            for path in [self.work_dir, self.test_dir, self.build_prefix, self.test_prefix]:
-                if os.path.isdir(path):
-                    rm_rf(path)
+        if remove_folders and not getattr(self, 'dirty'):
+            if self.build_id:
+                if os.path.isdir(self.build_folder):
+                    rm_rf(self.build_folder)
+            else:
+                for path in [self.work_dir, self.test_dir, self.build_prefix, self.test_prefix]:
+                    if os.path.isdir(path):
+                        rm_rf(path)
+            if os.path.isfile(os.path.join(self.build_folder, 'prefix_files')):
+                rm_rf(os.path.join(self.build_folder, 'prefix_files'))
+
+        for lock in get_conda_operation_locks(self):
+            rm_rf(lock.lock_file)
 
     def clean_pkgs(self):
         for folder in self.bldpkgs_dirs:
diff --git a/conda_build/utils.py b/conda_build/utils.py
--- a/conda_build/utils.py
+++ b/conda_build/utils.py
@@ -28,6 +28,7 @@
 from .conda_interface import PY3, iteritems
 from .conda_interface import root_dir
 from .conda_interface import string_types
+from .conda_interface import pkgs_dirs
 
 from conda_build.os_utils import external
 
@@ -285,6 +286,22 @@ def get_lock(folder, timeout=90, filename=".conda_lock"):
     return _locations[location]
 
 
+def get_conda_operation_locks(config=None):
+    locks = []
+    # locks enabled by default
+    if not config or config.locking:
+        _pkgs_dirs = pkgs_dirs[:1]
+        locked_folders = _pkgs_dirs + list(config.bldpkgs_dirs) if config else []
+        for folder in locked_folders:
+            if not os.path.isdir(folder):
+                os.makedirs(folder)
+            lock = get_lock(folder, timeout=config.timeout if config else 90)
+            locks.append(lock)
+        # lock used to generally indicate a conda operation occurring
+        locks.append(get_lock('conda-operation', timeout=config.timeout if config else 90))
+    return locks
+
+
 def relative(f, d='lib'):
     assert not f.startswith('/'), f
     assert not d.startswith('/'), d


Test patch:
diff --git a/tests/test_api_build.py b/tests/test_api_build.py
--- a/tests/test_api_build.py
+++ b/tests/test_api_build.py
@@ -26,7 +26,7 @@
 from conda_build import api, exceptions, __version__
 from conda_build.build import VersionOrder
 from conda_build.utils import (copy_into, on_win, check_call_env, convert_path_for_cygwin_or_msys2,
-                               package_has_file, check_output_env)
+                               package_has_file, check_output_env, get_conda_operation_locks)
 from conda_build.os_utils.external import find_executable
 
 from .utils import metadata_dir, fail_dir, is_valid_dir, add_mangling
@@ -881,6 +881,18 @@ def test_extract_tarball_with_unicode_filename(test_config):
     api.build(recipe, config=test_config)
 
 
+def test_failed_recipe_leaves_folders(test_config, testing_workdir):
+    recipe = os.path.join(fail_dir, 'recursive-build')
+    m = api.render(recipe, config=test_config)[0]
+    locks = get_conda_operation_locks(m.config)
+    with pytest.raises(RuntimeError):
+        api.build(m)
+    assert os.listdir(m.config.build_folder)
+    # make sure that it does not leave lock files, though, as these cause permission errors on
+    #    centralized installations
+    assert not any(os.path.isfile(lock.lock_file) for lock in locks)
+
+
 def test_only_r_env_vars_defined(test_config):
     recipe = os.path.join(metadata_dir, '_r_env_defined')
     test_config.channel_urls = ('r', )