Repository: conda/conda

Issue:
Permission denied for c_rehash in multi-user config
Tried 4.3.14 and 4.3.8. Also tried adding the `conda-canary` channel, but no luck. This seems to have coincided with an upgrade from 4.3.8 -> 4.3.14. 

Here's the trace:

```
INFO conda.common.io:captured(59): overtaking stderr and stdout
INFO conda.common.io:captured(65): stderr and stdout yielding back
Current conda install:

               platform : linux-64
          conda version : 4.3.8
       conda is private : False
      conda-env version : 4.3.8
    conda-build version : not installed
         python version : 3.6.0.final.0
       requests version : 2.12.4
       root environment : /usr/local/python3/anaconda3  (read only)
    default environment : /usr/local/python3/anaconda3
       envs directories : /home/fido/.conda/envs
                          /usr/local/python3/anaconda3/envs
          package cache : /home/fido/.conda/envs/.pkgs
                          /usr/local/python3/anaconda3/pkgs
           channel URLs : https://repo.continuum.io/pkgs/free/linux-64
                          https://repo.continuum.io/pkgs/free/noarch
                          https://repo.continuum.io/pkgs/r/linux-64
                          https://repo.continuum.io/pkgs/r/noarch
                          https://repo.continuum.io/pkgs/pro/linux-64
                          https://repo.continuum.io/pkgs/pro/noarch
            config file : None
           offline mode : False
             user-agent : conda/4.3.8 requests/2.12.4 CPython/3.6.0 Linux/2.6.32-642.6.2.el6.x86_64 CentOS/6.8 glibc/2.12
                UID:GID : 30100:36014



`$ /usr/local/python3/anaconda3/bin/conda create -n bar0 python=3.5 -y -vv`




    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 307, in _execute_actions
        action.execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/path_actions.py", line 272, in execute
        super(PrefixReplaceLinkAction, self).execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/path_actions.py", line 227, in execute
        force=context.force)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/gateways/disk/create.py", line 241, in create_link
        shutil.copy2(src, dst)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 252, in copy2
        copystat(src, dst, follow_symlinks=follow_symlinks)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 219, in copystat
        _copyxattr(src, dst, follow_symlinks=follow)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 159, in _copyxattr
        os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
    PermissionError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar0/bin/c_rehash'
    
    During handling of the above exception, another exception occurred:
    
    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 261, in execute
        pkg_data, actions)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 325, in _execute_actions
        reverse_excs,
    conda.CondaMultiError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar0/bin/c_rehash'
    
    
    During handling of the above exception, another exception occurred:
    
    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/exceptions.py", line 617, in conda_exception_handler
        return_value = func(*args, **kwargs)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/main.py", line 137, in _main
        exit_code = args.func(args, p)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/main_create.py", line 68, in execute
        install(args, parser, 'create')
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/install.py", line 347, in install
        execute_actions(actions, index, verbose=not context.quiet)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/plan.py", line 837, in execute_actions
        execute_instructions(plan, index, verbose)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/instructions.py", line 258, in execute_instructions
        cmd(state, arg)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/instructions.py", line 119, in UNLINKLINKTRANSACTION_CMD
        txn.execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 277, in execute
        rollback_excs,
    conda.CondaMultiError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar0/bin/c_rehash'
```
Permission denied for c_rehash in multi-user config
Tried 4.3.14 and 4.3.8. Also tried adding the `conda-canary` channel, but no luck. This seems to have coincided with an upgrade from 4.3.8 -> 4.3.14. 

Here's the trace:

```
INFO conda.common.io:captured(59): overtaking stderr and stdout
INFO conda.common.io:captured(65): stderr and stdout yielding back
Current conda install:

               platform : linux-64
          conda version : 4.3.8
       conda is private : False
      conda-env version : 4.3.8
    conda-build version : not installed
         python version : 3.6.0.final.0
       requests version : 2.12.4
       root environment : /usr/local/python3/anaconda3  (read only)
    default environment : /usr/local/python3/anaconda3
       envs directories : /home/fido/.conda/envs
                          /usr/local/python3/anaconda3/envs
          package cache : /home/fido/.conda/envs/.pkgs
                          /usr/local/python3/anaconda3/pkgs
           channel URLs : https://repo.continuum.io/pkgs/free/linux-64
                          https://repo.continuum.io/pkgs/free/noarch
                          https://repo.continuum.io/pkgs/r/linux-64
                          https://repo.continuum.io/pkgs/r/noarch
                          https://repo.continuum.io/pkgs/pro/linux-64
                          https://repo.continuum.io/pkgs/pro/noarch
            config file : None
           offline mode : False
             user-agent : conda/4.3.8 requests/2.12.4 CPython/3.6.0 Linux/2.6.32-642.6.2.el6.x86_64 CentOS/6.8 glibc/2.12
                UID:GID : 30100:36014



`$ /usr/local/python3/anaconda3/bin/conda create -n bar0 python=3.5 -y -vv`




    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 307, in _execute_actions
        action.execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/path_actions.py", line 272, in execute
        super(PrefixReplaceLinkAction, self).execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/path_actions.py", line 227, in execute
        force=context.force)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/gateways/disk/create.py", line 241, in create_link
        shutil.copy2(src, dst)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 252, in copy2
        copystat(src, dst, follow_symlinks=follow_symlinks)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 219, in copystat
        _copyxattr(src, dst, follow_symlinks=follow)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 159, in _copyxattr
        os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
    PermissionError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar0/bin/c_rehash'
    
    During handling of the above exception, another exception occurred:
    
    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 261, in execute
        pkg_data, actions)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 325, in _execute_actions
        reverse_excs,
    conda.CondaMultiError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar0/bin/c_rehash'
    
    
    During handling of the above exception, another exception occurred:
    
    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/exceptions.py", line 617, in conda_exception_handler
        return_value = func(*args, **kwargs)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/main.py", line 137, in _main
        exit_code = args.func(args, p)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/main_create.py", line 68, in execute
        install(args, parser, 'create')
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/install.py", line 347, in install
        execute_actions(actions, index, verbose=not context.quiet)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/plan.py", line 837, in execute_actions
        execute_instructions(plan, index, verbose)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/instructions.py", line 258, in execute_instructions
        cmd(state, arg)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/instructions.py", line 119, in UNLINKLINKTRANSACTION_CMD
        txn.execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 277, in execute
        rollback_excs,
    conda.CondaMultiError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar0/bin/c_rehash'
```


Discussion:
What do you get for

    ls -al /home/fido/.conda/envs/bar0/bin/c_rehash
Please update to 4.3.13 or later.  And add here the new stack trace if you still have problems.
Hi,

Sorry I opened a new ticket. Forgot I had this one open.

```
-rwxrwxr-x. 1 fido accre 5144 Mar  3 10:28 /home/fido/.conda/envs/baz/bin/c_rehash
```

As per #5251, I can confirm that creating the environment works on a local drive.
Unfortunately, there's nothing jumping out at me here.  You're going to have to provide more context I think.  More information about the operating environments where this works and where this doesn't, since you say it works locally.  What's "not local"?  You might end up having to do your own forensics here and help us figure out where we have a bug, if indeed we do have a bug.
Sure thing. Sorry if I was being ambiguous. We're running CentOS 6.8 on a cluster. Installing to `/tmp/bar` works and is on a local disk, i.e. not network mounted. However, on our system, `/home` is a GPFS-mounted drive. Anaconda is also installed on a GPFS-mounted drive with `root` as owner. 

If the problem is due to how GPFS is implemented, then I can start down that path, but I thought it might have cropped up before. 

I updated to 4.3.17 and tried again to no avail:
```
DEBUG conda.gateways.disk.test:prefix_is_writable(64): testing write access for prefix '/usr/local/python3/anaconda3' using path '/usr/local/python3/anaconda3/conda-meta/history'
DEBUG conda.gateways.disk.test:file_path_is_writable(29): [Errno 13] Permission denied: '/usr/local/python3/anaconda3/conda-meta/history'
DEBUG conda.gateways.disk.test:prefix_is_writable(64): testing write access for prefix '/usr/local/python3/anaconda3' using path '/usr/local/python3/anaconda3/conda-meta/history'
DEBUG conda.gateways.disk.test:file_path_is_writable(29): [Errno 13] Permission denied: '/usr/local/python3/anaconda3/conda-meta/history'
    Current conda install:

                   platform : linux-64
              conda version : 4.3.17
           conda is private : False
          conda-env version : 4.3.17
        conda-build version : not installed
             python version : 3.6.0.final.0
           requests version : 2.12.4
           root environment : /usr/local/python3/anaconda3  (read only)
        default environment : /usr/local/python3/anaconda3
           envs directories : /home/fido/.conda/envs
                          /usr/local/python3/anaconda3/envs
              package cache : /usr/local/python3/anaconda3/pkgs
                          /home/fido/.conda/pkgs
               channel URLs : https://repo.continuum.io/pkgs/free/linux-64
                          https://repo.continuum.io/pkgs/free/noarch
                          https://repo.continuum.io/pkgs/r/linux-64
                          https://repo.continuum.io/pkgs/r/noarch
                          https://repo.continuum.io/pkgs/pro/linux-64
                          https://repo.continuum.io/pkgs/pro/noarch
                config file : None
               offline mode : False
                 user-agent : conda/4.3.17 requests/2.12.4 CPython/3.6.0 Linux/2.6.32-642.6.2.el6.x86_64 CentOS/6.8 glibc/2.12    
                    UID:GID : 30100:36014
    

`$ /usr/local/python3/anaconda3/bin/conda create --debug -n bar python=3.6`




    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 324, in _execute_actions
        action.execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/path_actions.py", line 291, in execute
        super(PrefixReplaceLinkAction, self).execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/path_actions.py", line 246, in execute
        force=context.force)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/gateways/disk/create.py", line 252, in create_link
        _do_copy(src, dst)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/gateways/disk/create.py", line 214, in _do_copy
        shutil.copy2(src, dst)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 252, in copy2
        copystat(src, dst, follow_symlinks=follow_symlinks)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 219, in copystat
        _copyxattr(src, dst, follow_symlinks=follow)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 159, in _copyxattr
        os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
    PermissionError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar/bin/c_rehash'
    
    During handling of the above exception, another exception occurred:
    
    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 279, in execute
        pkg_data, actions)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 342, in _execute_actions
        reverse_excs,
    conda.CondaMultiError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar/bin/c_rehash'
    
    
    During handling of the above exception, another exception occurred:
    
    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/exceptions.py", line 632, in conda_exception_handler
        return_value = func(*args, **kwargs)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/main.py", line 134, in _main
        exit_code = args.func(args, p)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/main_create.py", line 68, in execute
        install(args, parser, 'create')
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/install.py", line 357, in install
        execute_actions(actions, index, verbose=not context.quiet)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/plan.py", line 830, in execute_actions
        execute_instructions(plan, index, verbose)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/instructions.py", line 247, in execute_instructions
        cmd(state, arg)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/instructions.py", line 108, in UNLINKLINKTRANSACTION_CMD
        txn.execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 295, in execute
        rollback_excs,
    conda.CondaMultiError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar/bin/c_rehash'
```

Thanks!
One very import point that I failed to mention is that installing to my `/home` directory *does work* for Anaconda2 v. 4.3.14. Anaconda3 worked as recently as 4.3.8, I believe.
Right now, the evidence points to this issue being caused by GPFS and not conda, as I've run into other permission denied errors using `pip` alone. 
This may be related to an error that I have related to selinux security and shutil.copy2. conda uses shutil.copy2 when link fails (e.g., when overwriting an existing link). shutil.copy2 is a wrapper to shutil.copy and shutil.copystat. The shutil.copy works fine, but the shutil.copystat gives a permission denied when using the os.setxattr function on the security.selinux property. 

So, in my case this is a system configuration peculiarity coupled with an intolerant copy method. It is also easily fixable. If conda were to, instead of using copy2, use copy and then copystat with a try except wrapper it fixes the problem for me.

In conda/gateways/disk/create.py
replace both instances of 
```
shutil.copy2(src, dst)
```
with
```
shutil.copy(src, dst)
try:
    shutil.copystat(src, dst)
except PermissionError as e:
    print(str(e))
    pass
```

lines circa 178 and 207

BTW - I believe that this is a reasonable method. The permission error did not occur on the copy, so obviously the user has sufficient permissions for the task at hand. The one problem is that we rely on copystat maintaining its order of operation (i.e., setting xattrs last).
That seems pretty easy.  I'll see what it looks like getting that into the 4.3.x branch today.  And if it looks good, will go in 4.3.18.  Am ready to tag and release to conda-canary 4.3.18 today or tomorrow.
What do you get for

    ls -al /home/fido/.conda/envs/bar0/bin/c_rehash
Please update to 4.3.13 or later.  And add here the new stack trace if you still have problems.
Hi,

Sorry I opened a new ticket. Forgot I had this one open.

```
-rwxrwxr-x. 1 fido accre 5144 Mar  3 10:28 /home/fido/.conda/envs/baz/bin/c_rehash
```

As per #5251, I can confirm that creating the environment works on a local drive.
Unfortunately, there's nothing jumping out at me here.  You're going to have to provide more context I think.  More information about the operating environments where this works and where this doesn't, since you say it works locally.  What's "not local"?  You might end up having to do your own forensics here and help us figure out where we have a bug, if indeed we do have a bug.
Sure thing. Sorry if I was being ambiguous. We're running CentOS 6.8 on a cluster. Installing to `/tmp/bar` works and is on a local disk, i.e. not network mounted. However, on our system, `/home` is a GPFS-mounted drive. Anaconda is also installed on a GPFS-mounted drive with `root` as owner. 

If the problem is due to how GPFS is implemented, then I can start down that path, but I thought it might have cropped up before. 

I updated to 4.3.17 and tried again to no avail:
```
DEBUG conda.gateways.disk.test:prefix_is_writable(64): testing write access for prefix '/usr/local/python3/anaconda3' using path '/usr/local/python3/anaconda3/conda-meta/history'
DEBUG conda.gateways.disk.test:file_path_is_writable(29): [Errno 13] Permission denied: '/usr/local/python3/anaconda3/conda-meta/history'
DEBUG conda.gateways.disk.test:prefix_is_writable(64): testing write access for prefix '/usr/local/python3/anaconda3' using path '/usr/local/python3/anaconda3/conda-meta/history'
DEBUG conda.gateways.disk.test:file_path_is_writable(29): [Errno 13] Permission denied: '/usr/local/python3/anaconda3/conda-meta/history'
    Current conda install:

                   platform : linux-64
              conda version : 4.3.17
           conda is private : False
          conda-env version : 4.3.17
        conda-build version : not installed
             python version : 3.6.0.final.0
           requests version : 2.12.4
           root environment : /usr/local/python3/anaconda3  (read only)
        default environment : /usr/local/python3/anaconda3
           envs directories : /home/fido/.conda/envs
                          /usr/local/python3/anaconda3/envs
              package cache : /usr/local/python3/anaconda3/pkgs
                          /home/fido/.conda/pkgs
               channel URLs : https://repo.continuum.io/pkgs/free/linux-64
                          https://repo.continuum.io/pkgs/free/noarch
                          https://repo.continuum.io/pkgs/r/linux-64
                          https://repo.continuum.io/pkgs/r/noarch
                          https://repo.continuum.io/pkgs/pro/linux-64
                          https://repo.continuum.io/pkgs/pro/noarch
                config file : None
               offline mode : False
                 user-agent : conda/4.3.17 requests/2.12.4 CPython/3.6.0 Linux/2.6.32-642.6.2.el6.x86_64 CentOS/6.8 glibc/2.12    
                    UID:GID : 30100:36014
    

`$ /usr/local/python3/anaconda3/bin/conda create --debug -n bar python=3.6`




    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 324, in _execute_actions
        action.execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/path_actions.py", line 291, in execute
        super(PrefixReplaceLinkAction, self).execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/path_actions.py", line 246, in execute
        force=context.force)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/gateways/disk/create.py", line 252, in create_link
        _do_copy(src, dst)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/gateways/disk/create.py", line 214, in _do_copy
        shutil.copy2(src, dst)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 252, in copy2
        copystat(src, dst, follow_symlinks=follow_symlinks)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 219, in copystat
        _copyxattr(src, dst, follow_symlinks=follow)
      File "/usr/local/python3/anaconda3/lib/python3.6/shutil.py", line 159, in _copyxattr
        os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
    PermissionError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar/bin/c_rehash'
    
    During handling of the above exception, another exception occurred:
    
    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 279, in execute
        pkg_data, actions)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 342, in _execute_actions
        reverse_excs,
    conda.CondaMultiError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar/bin/c_rehash'
    
    
    During handling of the above exception, another exception occurred:
    
    Traceback (most recent call last):
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/exceptions.py", line 632, in conda_exception_handler
        return_value = func(*args, **kwargs)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/main.py", line 134, in _main
        exit_code = args.func(args, p)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/main_create.py", line 68, in execute
        install(args, parser, 'create')
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/cli/install.py", line 357, in install
        execute_actions(actions, index, verbose=not context.quiet)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/plan.py", line 830, in execute_actions
        execute_instructions(plan, index, verbose)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/instructions.py", line 247, in execute_instructions
        cmd(state, arg)
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/instructions.py", line 108, in UNLINKLINKTRANSACTION_CMD
        txn.execute()
      File "/usr/local/python3/anaconda3/lib/python3.6/site-packages/conda/core/link.py", line 295, in execute
        rollback_excs,
    conda.CondaMultiError: [Errno 13] Permission denied: '/home/fido/.conda/envs/bar/bin/c_rehash'
```

Thanks!
One very import point that I failed to mention is that installing to my `/home` directory *does work* for Anaconda2 v. 4.3.14. Anaconda3 worked as recently as 4.3.8, I believe.
Right now, the evidence points to this issue being caused by GPFS and not conda, as I've run into other permission denied errors using `pip` alone. 
This may be related to an error that I have related to selinux security and shutil.copy2. conda uses shutil.copy2 when link fails (e.g., when overwriting an existing link). shutil.copy2 is a wrapper to shutil.copy and shutil.copystat. The shutil.copy works fine, but the shutil.copystat gives a permission denied when using the os.setxattr function on the security.selinux property. 

So, in my case this is a system configuration peculiarity coupled with an intolerant copy method. It is also easily fixable. If conda were to, instead of using copy2, use copy and then copystat with a try except wrapper it fixes the problem for me.

In conda/gateways/disk/create.py
replace both instances of 
```
shutil.copy2(src, dst)
```
with
```
shutil.copy(src, dst)
try:
    shutil.copystat(src, dst)
except PermissionError as e:
    print(str(e))
    pass
```

lines circa 178 and 207

BTW - I believe that this is a reasonable method. The permission error did not occur on the copy, so obviously the user has sufficient permissions for the task at hand. The one problem is that we rely on copystat maintaining its order of operation (i.e., setting xattrs last).
That seems pretty easy.  I'll see what it looks like getting that into the 4.3.x branch today.  And if it looks good, will go in 4.3.18.  Am ready to tag and release to conda-canary 4.3.18 today or tomorrow.

Resolving patch:
diff --git a/conda/gateways/disk/create.py b/conda/gateways/disk/create.py
--- a/conda/gateways/disk/create.py
+++ b/conda/gateways/disk/create.py
@@ -8,7 +8,7 @@
 import os
 from os import X_OK, access, makedirs
 from os.path import basename, isdir, isfile, join, lexists
-import shutil
+from shutil import copy as shutil_copy, copystat
 import sys
 import tarfile
 import traceback
@@ -25,10 +25,10 @@
 from ...base.context import context
 from ...common.compat import ensure_binary, on_win
 from ...common.path import win_path_ok
+from ...core.portability import replace_long_shebang
 from ...exceptions import BasicClobberError, CondaOSError, maybe_raise
 from ...models.dist import Dist
-from ...models.enums import LinkType, FileMode
-from ...core.portability import replace_long_shebang
+from ...models.enums import FileMode, LinkType
 
 log = getLogger(__name__)
 stdoutlog = getLogger('stdoutlog')
@@ -182,7 +182,7 @@ def create_hard_link_or_copy(src, dst):
         link(src, dst)
     except (IOError, OSError):
         log.info('hard link failed, so copying %s => %s', src, dst)
-        shutil.copy2(src, dst)
+        _do_copy(src, dst)
 
 
 def _is_unix_executable_using_ORIGIN(path):
@@ -198,13 +198,13 @@ def _do_softlink(src, dst):
         # We only need to do this copy for executables which have an RPATH containing $ORIGIN
         #   on Linux, so `is_executable()` is currently overly aggressive.
         # A future optimization will be to copy code from @mingwandroid's virtualenv patch.
-        _do_copy(src, dst)
+        copy(src, dst)
     else:
         log.trace("soft linking %s => %s", src, dst)
         symlink(src, dst)
 
 
-def _do_copy(src, dst):
+def copy(src, dst):
     # on unix, make sure relative symlinks stay symlinks
     if not on_win and islink(src):
         src_points_to = readlink(src)
@@ -213,8 +213,18 @@ def _do_copy(src, dst):
             log.trace("soft linking %s => %s", src, dst)
             symlink(src_points_to, dst)
             return
+    _do_copy(src, dst)
+
+
+def _do_copy(src, dst):
     log.trace("copying %s => %s", src, dst)
-    shutil.copy2(src, dst)
+    shutil_copy(src, dst)
+    try:
+        copystat(src, dst)
+    except (IOError, OSError) as e:  # pragma: no cover
+        # shutil.copystat gives a permission denied when using the os.setxattr function
+        # on the security.selinux property.
+        log.debug('%r', e)
 
 
 def create_link(src, dst, link_type=LinkType.hardlink, force=False):
@@ -249,11 +259,11 @@ def create_link(src, dst, link_type=LinkType.hardlink, force=False):
                       "  error: %r\n"
                       "  src: %s\n"
                       "  dst: %s", e, src, dst)
-            _do_copy(src, dst)
+            copy(src, dst)
     elif link_type == LinkType.softlink:
         _do_softlink(src, dst)
     elif link_type == LinkType.copy:
-        _do_copy(src, dst)
+        copy(src, dst)
     else:
         raise CondaError("Did not expect linktype=%r" % link_type)
 


Test patch: