Repository: python/mypy

Issue:
INTERNAL ERROR report
Hi,

I get the following error when I run

    mypy --ignore-missing-imports --follow-imports=silent @typechecked-files --no-strict-optional --show-traceback

from the root of [Sockeye](https://github.com/awslabs/sockeye/). I am not sure how to fix it or what causes it. Can you help?

```
./contrib/sacrebleu/sacrebleu.py: error: INTERNAL ERROR -- please report a bug at https://github.com/python/mypy/issues version: 0.610
Traceback (most recent call last):
  File "/usr/local/bin/mypy", line 11, in <module>
    sys.exit(console_entry())
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/__main__.py", line 7, in console_entry
    main(None)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/main.py", line 91, in main
    res = type_check_only(sources, bin_dir, options, flush_errors, fscache)  # noqa
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/main.py", line 148, in type_check_only
    fscache=fscache)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/build.py", line 185, in build
    flush_errors, fscache)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/build.py", line 285, in _build
    graph = dispatch(sources, manager)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/build.py", line 2398, in dispatch
    process_graph(graph, manager)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/build.py", line 2691, in process_graph
    process_stale_scc(graph, scc, manager)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/build.py", line 2801, in process_stale_scc
    graph[id].semantic_analysis()
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/build.py", line 2009, in semantic_analysis
    self.manager.semantic_analyzer.visit_file(self.tree, self.xpath, self.options, patches)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/contextlib.py", line 99, in __exit__
    self.gen.throw(type, value, traceback)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/build.py", line 1818, in wrap_context
    yield
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/build.py", line 2009, in semantic_analysis
    self.manager.semantic_analyzer.visit_file(self.tree, self.xpath, self.options, patches)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/semanal.py", line 291, in visit_file
    v.type = self.anal_type(v.type)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/semanal.py", line 1578, in anal_type
    typ = t.accept(a)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/types.py", line 203, in accept
    return visitor.visit_unbound_type(self)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/typeanal.py", line 178, in visit_unbound_type
    typ = self.visit_unbound_type_nonoptional(t)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/typeanal.py", line 186, in visit_unbound_type_nonoptional
    sym = self.lookup(t.name, t, suppress_errors=self.third_pass)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/semanal.py", line 3058, in lookup_qualified
    n = self.lookup(parts[0], ctx, suppress_errors=suppress_errors)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/semanal.py", line 3038, in lookup
    self.name_not_defined(name, ctx)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/semanal.py", line 3246, in name_not_defined
    self.fail(message, ctx)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/semanal.py", line 3281, in fail
    self.errors.report(ctx.get_line(), ctx.get_column(), msg, blocker=blocker)
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/errors.py", line 260, in report
    target=self.current_target())
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/errors.py", line 208, in current_target
    return self.scope.current_target()
  File "/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/mypy/scope.py", line 30, in current_target
    assert self.module
AssertionError: 
./contrib/sacrebleu/sacrebleu.py: : note: use --pdb to drop into pdb
```



Discussion:
This usually happens when the mypy install is broken or site-packages is in the path. How did you install mypy? Can you look for `/usr/local/Cellar/python3/3.6.4/Frameworks/Python.framework/Versions/3.6/lib/mypy`? If it doesn't exist, your install is broken. Otherwise, you might try install mypy from git, I've added some better debugging about how site-packages got into the `PATH`.
Thanks — I installed through pip3, and no amount of uninstall/install would resolve the problem, but it installed fine through git as you suggested.
I'd like to reopen this, because we've seen this internally too -- a user has installed Python 3 via brew (/Cellar/ in the path is the giveaway here), installed mypy using pip3, and then somehow mypy can't find typeshed resulting in this crash. It doesn't happen to everyone who uses brew though, and IIRC there's a way to fix it that doesn't require installing from git master (something with `brew update` and/or upgrading Python 3 perhaps).
Sounds good. I’m happy to run anything if it would be helpful. I will note that after `pip3 install mypy` there was indeed no `mypy` file at the path noted by @ethanhs above. 
If you could uninstall the git version and re-install normally, is there any folder named `typeshed` on your computer?

Also, I am going to up the priority of #5307, since we keep running into issues with data_files.
Sorry to ask, but how do I uninstall from the git repo? Googling didn't turn up much. Is
````
rm $(which mypy)
rm -rf /usr/local/lib/python3.6/site-packages/*mypy*
````
sufficient? If so, after doing that, reinstalling mypy (`pip3 install mypy`), mypy still fails, and there is no typeshed folder, at least not where I looked:

```
find /usr/local/lib/python3.6/site-packages -name typshed
# → returns nothing
```
You can uninstall via `pip uninstall mypy` for future reference (sorry I probably should have mentioned that). 

The typeshed folder is usually located outside of the site-packages, so it might be more useful to do `find /usr/local -name typeshed`. Hopefully this isn't horribly slow, if it is, you might try narrowing it to `/usr/local/Cellar`, which should help. Thank you for working with me on debugging this!
Ok, I didn't realize I could uninstall with pip after installing with git. Though it makes sense.

I ran the following:
```
pip3 uninstall mypy
find /usr/local -name typeshed # → nothing
pip3 install mypy
find /usr/local -name typeshed # → /usr/local/lib/mypy/typeshed
```
But now the original error is back.
Aha! That confirms my theory. It looks like data files are installed in a different root than the mypy package (`/usr/local/lib/` rather than `/usr/local/Cellar/`). Thanks so much for debugging this! 

For now, I'd suggest using mypy from git again if you can, but hopefully we can figure out a fix before the next release (which is soon).
Cool, glad I could help! I'm happy to use the workaround. Thanks for helping me out.
I propose we hardcode `/usr/local/lib/mypy/typeshed` on MacOS, as it is quite a different location from the normal "walk up from the mypy package" route we take.
You mean as an extra place to look, right? But how do we know that what we
find there is what was installed with the current version of mypy?

On Fri, Jul 6, 2018 at 10:06 AM Ethan Smith <notifications@github.com>
wrote:

> I propose we hardcode /usr/local/lib/mypy/typeshed on MacOS, as it is
> quite a different location from the normal "walk up from the mypy package"
> route we take.
>
> —
> You are receiving this because you modified the open/close state.
>
>
> Reply to this email directly, view it on GitHub
> <https://github.com/python/mypy/issues/5319#issuecomment-403092795>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/ACwrMj8n734P6KBFGvjKjECKte8dWSK4ks5uD5kOgaJpZM4VC5ha>
> .
>
-- 
--Guido (mobile)

>You mean as an extra place to look, right? But how do we know that what we
find there is what was installed with the current version of mypy?

We don't which sucks :(

FWIW, I'm happy to implement whichever plan we end up with in #5307. That is definitely the better solution here.
I'm just curious through which combination of pip and brew features we end
up installing the data in /usr/local when the code is installed in
.../Cellar/...

Maybe a symbolic link is involved?


Resolving patch:
diff --git a/mypy/build.py b/mypy/build.py
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -20,6 +20,7 @@
 import hashlib
 import json
 import os.path
+import pathlib
 import re
 import site
 import stat
@@ -326,7 +327,7 @@ def _build(sources: List[BuildSource],
     # This seems the most reasonable place to tune garbage collection.
     gc.set_threshold(50000)
 
-    data_dir = default_data_dir(bin_dir)
+    data_dir = default_data_dir()
     fscache = fscache or FileSystemCache()
 
     search_paths = compute_search_paths(sources, options, data_dir, fscache, alt_lib_path)
@@ -365,68 +366,9 @@ def _build(sources: List[BuildSource],
         reports.finish()
 
 
-def default_data_dir(bin_dir: Optional[str]) -> str:
-    """Returns directory containing typeshed directory
-
-    Args:
-      bin_dir: directory containing the mypy script
-    """
-    if not bin_dir:
-        if os.name == 'nt':
-            prefixes = [os.path.join(sys.prefix, 'Lib')]
-            try:
-                prefixes.append(os.path.join(site.getuserbase(), 'lib'))
-            except AttributeError:
-                # getuserbase in not available in virtualenvs
-                prefixes.append(os.path.join(get_python_lib(), 'lib'))
-            for parent in prefixes:
-                    data_dir = os.path.join(parent, 'mypy')
-                    if os.path.exists(data_dir):
-                        return data_dir
-        mypy_package = os.path.dirname(__file__)
-        parent = os.path.dirname(mypy_package)
-        if (os.path.basename(parent) == 'site-packages' or
-                os.path.basename(parent) == 'dist-packages'):
-            # Installed in site-packages or dist-packages, but invoked with python3 -m mypy;
-            # __file__ is .../blah/lib/python3.N/site-packages/mypy/build.py
-            # or .../blah/lib/python3.N/dist-packages/mypy/build.py (Debian)
-            # or .../blah/lib64/python3.N/dist-packages/mypy/build.py (Gentoo)
-            # or .../blah/lib/site-packages/mypy/build.py (Windows)
-            # blah may be a virtualenv or /usr/local.  We want .../blah/lib/mypy.
-            lib = parent
-            for i in range(2):
-                lib = os.path.dirname(lib)
-                if os.path.basename(lib) in ('lib', 'lib32', 'lib64'):
-                    return os.path.join(os.path.dirname(lib), 'lib/mypy')
-        subdir = os.path.join(parent, 'lib', 'mypy')
-        if os.path.isdir(subdir):
-            # If installed via buildout, the __file__ is
-            # somewhere/mypy/__init__.py and what we want is
-            # somewhere/lib/mypy.
-            return subdir
-        # Default to directory containing this file's parent.
-        return parent
-    base = os.path.basename(bin_dir)
-    dir = os.path.dirname(bin_dir)
-    if (sys.platform == 'win32' and base.lower() == 'scripts'
-            and not os.path.isdir(os.path.join(dir, 'typeshed'))):
-        # Installed, on Windows.
-        return os.path.join(dir, 'Lib', 'mypy')
-    elif base == 'scripts':
-        # Assume that we have a repo check out or unpacked source tarball.
-        return dir
-    elif base == 'bin':
-        # Installed to somewhere (can be under /usr/local or anywhere).
-        return os.path.join(dir, 'lib', 'mypy')
-    elif base == 'python3':
-        # Assume we installed python3 with brew on os x
-        return os.path.join(os.path.dirname(dir), 'lib', 'mypy')
-    elif dir.endswith('python-exec'):
-        # Gentoo uses a python wrapper in /usr/lib to which mypy is a symlink.
-        return os.path.join(os.path.dirname(dir), 'mypy')
-    else:
-        # Don't know where to find the data files!
-        raise RuntimeError("Broken installation: can't determine base dir")
+def default_data_dir() -> str:
+    """Returns directory containing typeshed directory."""
+    return os.path.dirname(__file__)
 
 
 def mypy_path() -> List[str]:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -33,8 +33,8 @@
 '''.lstrip()
 
 
-def find_data_files(base, globs):
-    """Find all interesting data files, for setup(data_files=)
+def find_package_data(base, globs):
+    """Find all interesting data files, for setup(package_data=)
 
     Arguments:
       root:  The directory to search in.
@@ -49,9 +49,7 @@ def find_data_files(base, globs):
             files += glob.glob(os.path.join(rv_dir, pat))
         if not files:
             continue
-        target = os.path.join('lib', 'mypy', rv_dir)
-        rv.append((target, files))
-
+        rv.extend([f[5:] for f in files])
     return rv
 
 
@@ -67,11 +65,12 @@ def run(self):
         build_py.run(self)
 
 
-data_files = []
+package_data = ['py.typed']
+
+package_data += find_package_data(os.path.join('mypy', 'typeshed'), ['*.py', '*.pyi'])
 
-data_files += find_data_files('typeshed', ['*.py', '*.pyi'])
+package_data += find_package_data(os.path.join('mypy', 'xml'), ['*.xsd', '*.xslt', '*.css'])
 
-data_files += find_data_files('xml', ['*.xsd', '*.xslt', '*.css'])
 
 classifiers = [
     'Development Status :: 3 - Alpha',
@@ -95,12 +94,11 @@ def run(self):
       license='MIT License',
       py_modules=[],
       packages=['mypy', 'mypy.test', 'mypy.server', 'mypy.plugins'],
-      package_data={'mypy': ['py.typed']},
+      package_data={'mypy': package_data},
       entry_points={'console_scripts': ['mypy=mypy.__main__:console_entry',
                                         'stubgen=mypy.stubgen:main',
                                         'dmypy=mypy.dmypy:main',
                                         ]},
-      data_files=data_files,
       classifiers=classifiers,
       cmdclass={'build_py': CustomPythonBuild},
       install_requires = ['typed-ast >= 1.1.0, < 1.2.0',
@@ -110,4 +108,5 @@ def run(self):
           ':python_version < "3.5"': 'typing >= 3.5.3',
           'dmypy': 'psutil >= 5.4.0, < 5.5.0; sys_platform!="win32"',
       },
+      include_package_data=True,
       )


Test patch: