Repository: conda/conda

Issue:
OSError: Too many levels of symlinks
This happened in a conda render operation, but does not appear to be readily reproducible - perhaps just a fluke, but something to watch out for if it happens often enough.

```
Traceback (most recent call last):
  File "/Users/msarahan/miniconda3/bin/c3i", line 11, in <module>
    load_entry_point('conda-concourse-ci', 'console_scripts', 'c3i')()
  File "/Users/msarahan/code/conda-concourse-ci/conda_concourse_ci/cli.py", line 123, in main
    execute.submit_one_off(**args.__dict__)
  File "/Users/msarahan/code/conda-concourse-ci/conda_concourse_ci/execute.py", line 647, in submit_one_off
    **kwargs)
  File "/Users/msarahan/code/conda-concourse-ci/conda_concourse_ci/execute.py", line 508, in compute_builds
    variant_config_files=kw.get('variant_config_files', []))
  File "/Users/msarahan/code/conda-concourse-ci/conda_concourse_ci/execute.py", line 72, in collect_tasks
    config=config)
  File "/Users/msarahan/code/conda-concourse-ci/conda_concourse_ci/compute_build_graph.py", line 379, in construct_graph
    recipes_dir, config=config)
  File "/Users/msarahan/code/conda-concourse-ci/conda_concourse_ci/compute_build_graph.py", line 206, in add_recipe_to_graph
    rendered = _get_or_render_metadata(recipe_dir, worker, config=config)
  File "/Users/msarahan/miniconda3/lib/python3.6/site-packages/conda/utils.py", line 38, in __call__
    return self.func(*args, **kw)
  File "/Users/msarahan/code/conda-concourse-ci/conda_concourse_ci/compute_build_graph.py", line 199, in _get_or_render_metadata
    bypass_env_check=True, config=config)
  File "/Users/msarahan/code/conda-build/conda_build/api.py", line 49, in render
    permit_undefined_jinja=not finalize):
  File "/Users/msarahan/code/conda-build/conda_build/metadata.py", line 1781, in get_output_metadata_set
    permit_unsatisfiable_variants=permit_unsatisfiable_variants)
  File "/Users/msarahan/code/conda-build/conda_build/metadata.py", line 645, in finalize_outputs_pass
    fm = finalize_metadata(om, permit_unsatisfiable_variants=permit_unsatisfiable_variants)
  File "/Users/msarahan/code/conda-build/conda_build/render.py", line 387, in finalize_metadata
    exclude_pattern)
  File "/Users/msarahan/code/conda-build/conda_build/render.py", line 306, in add_upstream_pins
    permit_unsatisfiable_variants, exclude_pattern)
  File "/Users/msarahan/code/conda-build/conda_build/render.py", line 289, in _read_upstream_pin_files
    extra_run_specs = get_upstream_pins(m, actions, env)
  File "/Users/msarahan/code/conda-build/conda_build/render.py", line 251, in get_upstream_pins
    pfe.execute()
  File "/Users/msarahan/miniconda3/lib/python3.6/site-packages/conda/core/package_cache.py", line 494, in execute
    self._execute_action(action)
  File "/Users/msarahan/miniconda3/lib/python3.6/site-packages/conda/core/package_cache.py", line 515, in _execute_action
    raise CondaMultiError(exceptions)
conda.CondaMultiError: OSError(62, 'Too many levels of symbolic links')
OSError(62, 'Too many levels of symbolic links')
OSError(62, 'Too many levels of symbolic links')
```


Discussion:
In an auto-upload report we got something similar.  Maybe more details here.

    "command": "/Users/boris/bin/miniconda/bin/conda install clangxx_osx-64"
    "user_agent": "conda/4.4.1 requests/2.18.4 CPython/2.7.14 Darwin/17.3.0 OSX/10.13.2"

```
Traceback (most recent call last):
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/exceptions.py\", line 721, in __call__
    return func(*args, **kwargs)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/cli/main.py\", line 78, in _main
    exit_code = do_call(args, p)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/cli/conda_argparse.py\", line 75, in do_call
    exit_code = getattr(module, func_name)(args, parser)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/cli/main_install.py\", line 11, in execute
    install(args, parser, 'install')
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/cli/install.py\", line 239, in install
    handle_txn(progressive_fetch_extract, unlink_link_transaction, prefix, args, newenv)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/cli/install.py\", line 256, in handle_txn
    unlink_link_transaction.display_actions(progressive_fetch_extract)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/link.py\", line 706, in display_actions
    legacy_action_groups = self.make_legacy_action_groups(pfe)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/link.py\", line 689, in make_legacy_action_groups
    pfe.prepare()
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/common/io.py\", line 402, in decorated
    return f(*args, **kwds)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/package_cache.py\", line 508, in prepare
    for prec in self.link_precs)
  File \"/Users/boris/bin/miniconda/lib/python2.7/_abcoll.py\", line 571, in update
    for key, value in other:
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/package_cache.py\", line 508, in <genexpr>
    for prec in self.link_precs)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/package_cache.py\", line 402, in make_actions_for_record
    ), None)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/package_cache.py\", line 399, in <genexpr>
    pcrec for pcrec in concat(PackageCache(pkgs_dir).query(pref_or_spec)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/package_cache.py\", line 400, in <genexpr>
    for pkgs_dir in context.pkgs_dirs)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/package_cache.py\", line 116, in query
    return (pcrec for pcrec in itervalues(self._package_cache_records) if pcrec == param)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/package_cache.py\", line 209, in _package_cache_records
    self.load()
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/package_cache.py\", line 88, in load
    package_cache_record = self._make_single_record(base_name)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/core/package_cache.py\", line 294, in _make_single_record
    extract_tarball(package_tarball_full_path, extracted_package_dir)
  File \"/Users/boris/bin/miniconda/lib/python2.7/site-packages/conda/gateways/disk/create.py\", line 145, in extract_tarball
    t.extractall(path=destination_directory, members=members_with_progress())
  File \"/Users/boris/bin/miniconda/lib/python2.7/tarfile.py\", line 2081, in extractall
    self.extract(tarinfo, path)
  File \"/Users/boris/bin/miniconda/lib/python2.7/tarfile.py\", line 2118, in extract
    self._extract_member(tarinfo, os.path.join(path, tarinfo.name))
  File \"/Users/boris/bin/miniconda/lib/python2.7/tarfile.py\", line 2194, in _extract_member
    self.makefile(tarinfo, targetpath)
  File \"/Users/boris/bin/miniconda/lib/python2.7/tarfile.py\", line 2234, in makefile
    with bltn_open(targetpath, \"wb\") as target:
IOError: [Errno 62] Too many levels of symbolic links: u'/Users/boris/bin/miniconda/pkgs/openssl-1.0.2l-h077ae2c_5/ssl/man/man3/mdc2.3'
```

Only `defaults` in the operation's channel list.
I'm running into the exact error above when trying to use the undocumented `CONDA_SUBDIR` setting on a fresh anaconda install.

System:
python 3.6.3
anaconda 5.0.1
macOS 10.13.2

All other config options (channels, environment, etc) are default.

I am able to replicate the error from the root environment of a clean install with the following command:
```
CONDA_SUBDIR=linux-64 \
$HOME/anaconda3/bin/conda install \
  --yes --mkdir --json -vv \
  --prefix $HOME/temp_conda_env \
  python=3.6 flask pyyaml yaml pandas nomkl
```

Traceback: https://gist.github.com/cscanlin/25fa05575918752252a071f352a7395a

For reference, I am trying to use `CONDA_SUBDIR` in an attempt at picking up the torch on conda support for Zappa: https://github.com/Miserlou/Zappa/pull/108

Please let me know if I can provide additional details, thanks!

Hey Chris, really appreciate someone digging into conda + Zappa. It’s been on my list for a *long* time, and I just haven’t had the bandwidth. Going to try to look into this particular issue later today. 

Sent from my iPhone

> On Jan 21, 2018, at 9:34 PM, Chris Scanlin <notifications@github.com> wrote:
> 
> I'm running into the exact error above when trying to use the undocumented CONDA_SUBDIR setting on a fresh anaconda install.
> 
> System:
> python 3.6.3
> anaconda 5.0.1
> macOS 10.13.2
> 
> All other config options (channels, environment, etc) are default.
> 
> I am able to replicate the error from the root environment of a clean install with the following command:
> 
> CONDA_SUBDIR=linux-64 \
> $HOME/anaconda3/bin/conda install \
>   --yes --mkdir --json -vv \
>   --prefix $HOME/temp_conda_env \
>   python=3.6 flask pyyaml yaml pandas nomkl
> Traceback: https://gist.github.com/cscanlin/25fa05575918752252a071f352a7395a
> 
> For reference, I am trying to use CONDA_SUBDIR in an attempt at picking up the torch on conda support for Zappa: Miserlou/Zappa#108
> 
> Please let me know if I can provide additional details, thanks!
> 
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub, or mute the thread.
> 

Issue here is with trying to use python's `tarfile.extractall()` on the linux-64 openssl package, while on macOS.  Relevant stack trace is

```
    Traceback (most recent call last):
      File "/Users/kfranz/continuum/conda/conda/exceptions.py", line 751, in __call__
        return func(*args, **kwargs)
      File "/Users/kfranz/continuum/conda/conda/cli/main.py", line 78, in _main
        exit_code = do_call(args, p)
      File "/Users/kfranz/continuum/conda/conda/cli/conda_argparse.py", line 76, in do_call
        exit_code = getattr(module, func_name)(args, parser)
      File "/Users/kfranz/continuum/conda/conda/cli/main_install.py", line 11, in execute
        install(args, parser, 'install')
      File "/Users/kfranz/continuum/conda/conda/cli/install.py", line 255, in install
        handle_txn(progressive_fetch_extract, unlink_link_transaction, prefix, args, newenv)
      File "/Users/kfranz/continuum/conda/conda/cli/install.py", line 272, in handle_txn
        unlink_link_transaction.display_actions(progressive_fetch_extract)
      File "/Users/kfranz/continuum/conda/conda/core/link.py", line 706, in display_actions
        legacy_action_groups = self.make_legacy_action_groups(pfe)
      File "/Users/kfranz/continuum/conda/conda/core/link.py", line 689, in make_legacy_action_groups
        pfe.prepare()
      File "/Users/kfranz/continuum/conda/conda/common/io.py", line 415, in decorated
        return f(*args, **kwds)
      File "/Users/kfranz/continuum/conda/conda/core/package_cache.py", line 540, in prepare
        for prec in self.link_precs)
      File "/Users/kfranz/continuum/conda/conda/core/package_cache.py", line 540, in <genexpr>
        for prec in self.link_precs)
      File "/Users/kfranz/continuum/conda/conda/core/package_cache.py", line 434, in make_actions_for_record
        ), None)
      File "/Users/kfranz/continuum/conda/conda/core/package_cache.py", line 431, in <genexpr>
        pcrec for pcrec in concat(PackageCache(pkgs_dir).query(pref_or_spec)
      File "/Users/kfranz/continuum/conda/conda/core/package_cache.py", line 432, in <genexpr>
        for pkgs_dir in context.pkgs_dirs)
      File "/Users/kfranz/continuum/conda/conda/core/package_cache.py", line 118, in query
        return (pcrec for pcrec in itervalues(self._package_cache_records) if pcrec == param)
      File "/Users/kfranz/continuum/conda/conda/core/package_cache.py", line 211, in _package_cache_records
        self.load()
      File "/Users/kfranz/continuum/conda/conda/core/package_cache.py", line 90, in load
        package_cache_record = self._make_single_record(base_name)
      File "/Users/kfranz/continuum/conda/conda/core/package_cache.py", line 308, in _make_single_record
        extract_tarball(package_tarball_full_path, extracted_package_dir)
      File "/Users/kfranz/continuum/conda/conda/gateways/disk/create.py", line 145, in extract_tarball
        t.extractall(path=destination_directory, members=members_with_progress())
      File "/usr/local/Cellar/python3/3.5.2_1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/tarfile.py", line 1996, in extractall
        numeric_owner=numeric_owner)
      File "/usr/local/Cellar/python3/3.5.2_1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/tarfile.py", line 2038, in extract
        numeric_owner=numeric_owner)
      File "/usr/local/Cellar/python3/3.5.2_1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/tarfile.py", line 2108, in _extract_member
        self.makefile(tarinfo, targetpath)
      File "/usr/local/Cellar/python3/3.5.2_1/Frameworks/Python.framework/Versions/3.5/lib/python3.5/tarfile.py", line 2148, in makefile
        with bltn_open(targetpath, "wb") as target:
    OSError: [Errno 62] Too many levels of symbolic links: '/Users/kfranz/linux-64/openssl-1.0.2n-hb7f436b_0/ssl/man/man3/mdc2.3'
```

The linux-64 package has

```
❯ ls -al /Users/kfranz/linux-64/openssl-1.0.2n-hb7f436b_0/ssl/man/man3/MDC*
lrwxr-xr-x  1 kfranz  staff     6B Jan 22 13:23 /Users/kfranz/linux-64/openssl-1.0.2n-hb7f436b_0/ssl/man/man3/MDC2.3@ -> mdc2.3
lrwxr-xr-x  1 kfranz  staff     6B Jan 22 13:23 /Users/kfranz/linux-64/openssl-1.0.2n-hb7f436b_0/ssl/man/man3/MDC2_Final.3@ -> mdc2.3
lrwxr-xr-x  1 kfranz  staff     6B Jan 22 13:23 /Users/kfranz/linux-64/openssl-1.0.2n-hb7f436b_0/ssl/man/man3/MDC2_Init.3@ -> mdc2.3
lrwxr-xr-x  1 kfranz  staff     6B Jan 22 13:23 /Users/kfranz/linux-64/openssl-1.0.2n-hb7f436b_0/ssl/man/man3/MDC2_Update.3@ -> mdc2.3
```

Specifically the problem is with

    ssl/man/man3/MDC2.3@ -> mdc2.3

So probably something going on with case sensitivity differences in the file system here.  Still figuring out what the correct fix is...

Resolving patch:
diff --git a/conda/exceptions.py b/conda/exceptions.py
--- a/conda/exceptions.py
+++ b/conda/exceptions.py
@@ -582,6 +582,21 @@ def __init__(self, message):
         super(CondaUpgradeError, self).__init__(msg)
 
 
+class CaseInsensitiveFileSystemError(CondaError):
+    def __init__(self, package_location, extract_location, **kwargs):
+        message = dals("""
+        Cannot extract package to a case-insensitive file system.
+          package location: %(package_location)s
+          extract location: %(extract_location)s
+        """)
+        super(CaseInsensitiveFileSystemError, self).__init__(
+            message,
+            package_location=package_location,
+            extract_location=extract_location,
+            **kwargs
+        )
+
+
 class CondaVerificationError(CondaError):
     def __init__(self, message):
         super(CondaVerificationError, self).__init__(message)
diff --git a/conda/gateways/disk/create.py b/conda/gateways/disk/create.py
--- a/conda/gateways/disk/create.py
+++ b/conda/gateways/disk/create.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from errno import EACCES, EPERM
+from errno import EACCES, ELOOP, EPERM
 from io import open
 from logging import getLogger
 import os
@@ -24,7 +24,8 @@
 from ...common.compat import ensure_binary, on_win
 from ...common.path import ensure_pad, expand, win_path_double_escape, win_path_ok
 from ...common.serialize import json_dump
-from ...exceptions import BasicClobberError, CondaOSError, maybe_raise
+from ...exceptions import (BasicClobberError, CaseInsensitiveFileSystemError, CondaOSError,
+                           maybe_raise)
 from ...models.enums import FileMode, LinkType
 
 log = getLogger(__name__)
@@ -142,7 +143,17 @@ def members_with_progress():
                     progress_update_callback(q / num_members)
                 yield member
 
-        t.extractall(path=destination_directory, members=members_with_progress())
+        try:
+            t.extractall(path=destination_directory, members=members_with_progress())
+        except EnvironmentError as e:
+            if e.errno == ELOOP:
+                raise CaseInsensitiveFileSystemError(
+                    package_location=tarball_full_path,
+                    extract_location=destination_directory,
+                    caused_by=e,
+                )
+            else:
+                raise
 
     if sys.platform.startswith('linux') and os.getuid() == 0:
         # When extracting as root, tarfile will by restore ownership


Test patch: