Repository: python/mypy

Issue:
`--install-types` suggests incompatible stub packages
<!--
  If you're new to mypy and you're not sure whether what you're experiencing is a mypy bug, please see the "Question and Help" form
  instead.
-->

**Bug Report**

<!--
Note: If the problem you are reporting is about a specific library function, then the typeshed tracker is better suited
for this report: https://github.com/python/typeshed/issues
-->

Running `mypy --install-types` suggest installation of stubs that are ot compatible with the current running version of python.
**To Reproduce**

(Write your steps here:)

1. Create a project containing `import OpenSSL.crypto`
2. `mypy --install-types`


**Expected Behavior**

<!--
  How did you expect your project to behave?
  It’s fine if you’re not sure your understanding is correct.
  Write down what you thought would happen. If you just expected no errors, you can delete this section.
-->

Does not suggest installing library stubs incompatible with python version.

**Actual Behavior**

<!--
  Did something go wrong?
  Is something broken, or not behaving as you expected?
-->

Suggests (and installs) stubs incompatible with the current version.

**Your Environment**

<!-- Include as many relevant details about the environment you experienced the bug in -->

- Mypy version used: 0.901
- Mypy command-line flags: none
- Mypy configuration options from `mypy.ini` (and other config files): none
- Python version used: 3.8, 3.9
- Operating system and version: Debian

<!--
You can freely edit this text, please remove all the lines
you believe are unnecessary.
-->



Discussion:
https://github.com/python/typeshed/issues/5593

Resolving patch:
diff --git a/mypy/build.py b/mypy/build.py
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -58,7 +58,7 @@
 from mypy.renaming import VariableRenameVisitor
 from mypy.config_parser import parse_mypy_comments
 from mypy.freetree import free_tree
-from mypy.stubinfo import legacy_bundled_packages
+from mypy.stubinfo import legacy_bundled_packages, is_legacy_bundled_package
 from mypy import errorcodes as codes
 
 
@@ -2449,7 +2449,9 @@ def find_module_and_diagnose(manager: BuildManager,
         # otherwise updating mypy can silently result in new false
         # negatives.
         global_ignore_missing_imports = manager.options.ignore_missing_imports
-        if ((top_level in legacy_bundled_packages or second_level in legacy_bundled_packages)
+        py_ver = options.python_version[0]
+        if ((is_legacy_bundled_package(top_level, py_ver)
+                or is_legacy_bundled_package(second_level, py_ver))
                 and global_ignore_missing_imports
                 and not options.ignore_missing_imports_per_module):
             ignore_missing_imports = False
@@ -2558,10 +2560,10 @@ def module_not_found(manager: BuildManager, line: int, caller_state: State,
             top_level = second_level
         for note in notes:
             if '{stub_dist}' in note:
-                note = note.format(stub_dist=legacy_bundled_packages[top_level])
+                note = note.format(stub_dist=legacy_bundled_packages[top_level].name)
             errors.report(line, 0, note, severity='note', only_once=True, code=codes.IMPORT)
         if reason is ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED:
-            manager.missing_stub_packages.add(legacy_bundled_packages[top_level])
+            manager.missing_stub_packages.add(legacy_bundled_packages[top_level].name)
     errors.set_import_context(save_import_context)
 
 
diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py
--- a/mypy/modulefinder.py
+++ b/mypy/modulefinder.py
@@ -17,7 +17,7 @@
 
 from mypy.fscache import FileSystemCache
 from mypy.options import Options
-from mypy.stubinfo import legacy_bundled_packages
+from mypy.stubinfo import is_legacy_bundled_package
 from mypy import sitepkgs
 
 # Paths to be searched in find_module().
@@ -136,7 +136,7 @@ def __init__(self,
         if options:
             custom_typeshed_dir = options.custom_typeshed_dir
         self.stdlib_py_versions = load_stdlib_py_versions(custom_typeshed_dir)
-        self.python2 = options and options.python_version[0] == 2
+        self.python_major_ver = 3 if options is None else options.python_version[0]
 
     def clear(self) -> None:
         self.results.clear()
@@ -187,7 +187,7 @@ def get_toplevel_possibilities(self, lib_path: Tuple[str, ...], id: str) -> List
                 name = os.path.splitext(name)[0]
                 components.setdefault(name, []).append(dir)
 
-        if self.python2:
+        if self.python_major_ver == 2:
             components = {id: filter_redundant_py2_dirs(dirs)
                           for id, dirs in components.items()}
 
@@ -230,8 +230,8 @@ def _find_module_non_stub_helper(self, components: List[str],
             elif not plausible_match and (self.fscache.isdir(dir_path)
                                           or self.fscache.isfile(dir_path + ".py")):
                 plausible_match = True
-        if (components[0] in legacy_bundled_packages
-                or '.'.join(components[:2]) in legacy_bundled_packages):
+        if (is_legacy_bundled_package(components[0], self.python_major_ver)
+                or is_legacy_bundled_package('.'.join(components[:2]), self.python_major_ver)):
             return ModuleNotFoundReason.APPROVED_STUBS_NOT_INSTALLED
         elif plausible_match:
             return ModuleNotFoundReason.FOUND_WITHOUT_TYPE_HINTS
@@ -280,7 +280,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
         for pkg_dir in self.search_paths.package_path:
             stub_name = components[0] + '-stubs'
             stub_dir = os.path.join(pkg_dir, stub_name)
-            if self.python2:
+            if self.python_major_ver == 2:
                 alt_stub_name = components[0] + '-python2-stubs'
                 alt_stub_dir = os.path.join(pkg_dir, alt_stub_name)
                 if fscache.isdir(alt_stub_dir):
@@ -348,7 +348,7 @@ def _find_module(self, id: str, use_typeshed: bool) -> ModuleSearchResult:
             for extension in PYTHON_EXTENSIONS:
                 path = base_path + sepinit + extension
                 suffix = '-stubs'
-                if self.python2:
+                if self.python_major_ver == 2:
                     if os.path.isdir(base_path + '-python2-stubs'):
                         suffix = '-python2-stubs'
                 path_stubs = base_path + suffix + sepinit + extension
@@ -432,7 +432,7 @@ def _is_compatible_stub_package(self, stub_dir: str) -> bool:
             import toml
             with open(metadata_fnam, 'r') as f:
                 metadata = toml.load(f)
-            if self.python2:
+            if self.python_major_ver == 2:
                 return bool(metadata.get('python2', False))
             else:
                 return bool(metadata.get('python3', True))
diff --git a/mypy/stubinfo.py b/mypy/stubinfo.py
--- a/mypy/stubinfo.py
+++ b/mypy/stubinfo.py
@@ -1,83 +1,100 @@
+from typing import Optional
+
+
+class StubInfo:
+    def __init__(self, name: str, py_version: Optional[int] = None) -> None:
+        self.name = name
+        # If None, compatible with py2+py3, if 2/3, only compatible with py2/py3
+        self.py_version = py_version
+
+
+def is_legacy_bundled_package(prefix: str, py_version: int) -> bool:
+    if prefix not in legacy_bundled_packages:
+        return False
+    package_ver = legacy_bundled_packages[prefix].py_version
+    return package_ver is None or package_ver == py_version
+
+
 # Stubs for these third-party packages used to be shipped with mypy.
 #
 # Map package name to PyPI stub distribution name.
 #
 # Package name can have one or two components ('a' or 'a.b').
 legacy_bundled_packages = {
-    'aiofiles': 'types-aiofiles',
-    'atomicwrites': 'types-atomicwrites',
-    'attr': 'types-attrs',
-    'backports': 'types-backports',
-    'backports_abc': 'types-backports_abc',
-    'bleach': 'types-bleach',
-    'boto': 'types-boto',
-    'cachetools': 'types-cachetools',
-    'certifi': 'types-certifi',
-    'characteristic': 'types-characteristic',
-    'chardet': 'types-chardet',
-    'click': 'types-click',
-    'click_spinner': 'types-click-spinner',
-    'concurrent': 'types-futures',
-    'contextvars': 'types-contextvars',
-    'croniter': 'types-croniter',
-    'cryptography': 'types-cryptography',
-    'dataclasses': 'types-dataclasses',
-    'dateparser': 'types-dateparser',
-    'datetimerange': 'types-DateTimeRange',
-    'dateutil': 'types-python-dateutil',
-    'decorator': 'types-decorator',
-    'deprecated': 'types-Deprecated',
-    'docutils': 'types-docutils',
-    'emoji': 'types-emoji',
-    'enum': 'types-enum34',
-    'fb303': 'types-fb303',
-    'filelock': 'types-filelock',
-    'first': 'types-first',
-    'flask': 'types-Flask',
-    'freezegun': 'types-freezegun',
-    'frozendict': 'types-frozendict',
-    'geoip2': 'types-geoip2',
-    'gflags': 'types-python-gflags',
-    'google.protobuf': 'types-protobuf',
-    'ipaddress': 'types-ipaddress',
-    'itsdangerous': 'types-itsdangerous',
-    'jinja2': 'types-Jinja2',
-    'jwt': 'types-jwt',
-    'kazoo': 'types-kazoo',
-    'markdown': 'types-Markdown',
-    'markupsafe': 'types-MarkupSafe',
-    'maxminddb': 'types-maxminddb',
-    'mock': 'types-mock',
-    'OpenSSL': 'types-openssl-python',
-    'orjson': 'types-orjson',
-    'paramiko': 'types-paramiko',
-    'pathlib2': 'types-pathlib2',
-    'pkg_resources': 'types-pkg_resources',
-    'polib': 'types-polib',
-    'pycurl': 'types-pycurl',
-    'pymssql': 'types-pymssql',
-    'pymysql': 'types-PyMySQL',
-    'pyrfc3339': 'types-pyRFC3339',
-    'python2': 'types-six',
-    'pytz': 'types-pytz',
-    'pyVmomi': 'types-pyvmomi',
-    'redis': 'types-redis',
-    'requests': 'types-requests',
-    'retry': 'types-retry',
-    'routes': 'types-Routes',
-    'scribe': 'types-scribe',
-    'simplejson': 'types-simplejson',
-    'singledispatch': 'types-singledispatch',
-    'six': 'types-six',
-    'slugify': 'types-python-slugify',
-    'tabulate': 'types-tabulate',
-    'termcolor': 'types-termcolor',
-    'toml': 'types-toml',
-    'tornado': 'types-tornado',
-    'typed_ast': 'types-typed-ast',
-    'tzlocal': 'types-tzlocal',
-    'ujson': 'types-ujson',
-    'waitress': 'types-waitress',
-    'werkzeug': 'types-Werkzeug',
-    'yaml': 'types-PyYAML',
+    'aiofiles': StubInfo('types-aiofiles', py_version=3),
+    'atomicwrites': StubInfo('types-atomicwrites'),
+    'attr': StubInfo('types-attrs'),
+    'backports': StubInfo('types-backports'),
+    'backports_abc': StubInfo('types-backports_abc'),
+    'bleach': StubInfo('types-bleach'),
+    'boto': StubInfo('types-boto'),
+    'cachetools': StubInfo('types-cachetools'),
+    'certifi': StubInfo('types-certifi'),
+    'characteristic': StubInfo('types-characteristic'),
+    'chardet': StubInfo('types-chardet'),
+    'click': StubInfo('types-click'),
+    'click_spinner': StubInfo('types-click-spinner'),
+    'concurrent': StubInfo('types-futures', py_version=2),
+    'contextvars': StubInfo('types-contextvars', py_version=3),
+    'croniter': StubInfo('types-croniter'),
+    'cryptography': StubInfo('types-cryptography'),
+    'dataclasses': StubInfo('types-dataclasses', py_version=3),
+    'dateparser': StubInfo('types-dateparser'),
+    'datetimerange': StubInfo('types-DateTimeRange'),
+    'dateutil': StubInfo('types-python-dateutil'),
+    'decorator': StubInfo('types-decorator'),
+    'deprecated': StubInfo('types-Deprecated'),
+    'docutils': StubInfo('types-docutils', py_version=3),
+    'emoji': StubInfo('types-emoji'),
+    'enum': StubInfo('types-enum34', py_version=2),
+    'fb303': StubInfo('types-fb303', py_version=2),
+    'filelock': StubInfo('types-filelock', py_version=3),
+    'first': StubInfo('types-first'),
+    'flask': StubInfo('types-Flask'),
+    'freezegun': StubInfo('types-freezegun', py_version=3),
+    'frozendict': StubInfo('types-frozendict', py_version=3),
+    'geoip2': StubInfo('types-geoip2'),
+    'gflags': StubInfo('types-python-gflags'),
+    'google.protobuf': StubInfo('types-protobuf'),
+    'ipaddress': StubInfo('types-ipaddress', py_version=2),
+    'itsdangerous': StubInfo('types-itsdangerous'),
+    'jinja2': StubInfo('types-Jinja2'),
+    'jwt': StubInfo('types-jwt'),
+    'kazoo': StubInfo('types-kazoo', py_version=2),
+    'markdown': StubInfo('types-Markdown'),
+    'markupsafe': StubInfo('types-MarkupSafe'),
+    'maxminddb': StubInfo('types-maxminddb'),
+    'mock': StubInfo('types-mock'),
+    'OpenSSL': StubInfo('types-openssl-python', py_version=2),
+    'orjson': StubInfo('types-orjson', py_version=3),
+    'paramiko': StubInfo('types-paramiko'),
+    'pathlib2': StubInfo('types-pathlib2', py_version=2),
+    'pkg_resources': StubInfo('types-pkg_resources', py_version=3),
+    'polib': StubInfo('types-polib'),
+    'pycurl': StubInfo('types-pycurl'),
+    'pymssql': StubInfo('types-pymssql', py_version=2),
+    'pymysql': StubInfo('types-PyMySQL'),
+    'pyrfc3339': StubInfo('types-pyRFC3339', py_version=3),
+    'python2': StubInfo('types-six'),
+    'pytz': StubInfo('types-pytz'),
+    'pyVmomi': StubInfo('types-pyvmomi'),
+    'redis': StubInfo('types-redis'),
+    'requests': StubInfo('types-requests'),
+    'retry': StubInfo('types-retry'),
+    'routes': StubInfo('types-Routes', py_version=2),
+    'scribe': StubInfo('types-scribe', py_version=2),
+    'simplejson': StubInfo('types-simplejson'),
+    'singledispatch': StubInfo('types-singledispatch'),
+    'six': StubInfo('types-six'),
+    'slugify': StubInfo('types-python-slugify'),
+    'tabulate': StubInfo('types-tabulate'),
+    'termcolor': StubInfo('types-termcolor'),
+    'toml': StubInfo('types-toml'),
+    'tornado': StubInfo('types-tornado', py_version=2),
+    'typed_ast': StubInfo('types-typed-ast', py_version=3),
+    'tzlocal': StubInfo('types-tzlocal'),
+    'ujson': StubInfo('types-ujson'),
+    'waitress': StubInfo('types-waitress', py_version=3),
+    'werkzeug': StubInfo('types-Werkzeug'),
+    'yaml': StubInfo('types-PyYAML'),
 }


Test patch:
diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py
--- a/mypy/test/testpythoneval.py
+++ b/mypy/test/testpythoneval.py
@@ -72,6 +72,10 @@ def test_python_evaluation(testcase: DataDrivenTestCase, cache_dir: str) -> None
         interpreter = python3_path
         mypy_cmdline.append('--python-version={}'.format('.'.join(map(str, PYTHON3_VERSION))))
 
+    m = re.search('# flags: (.*)$', '\n'.join(testcase.input), re.MULTILINE)
+    if m:
+        mypy_cmdline.extend(m.group(1).split())
+
     # Write the program to a file.
     program = '_' + testcase.name + '.py'
     program_path = os.path.join(test_temp_dir, program)
diff --git a/mypy/test/teststubinfo.py b/mypy/test/teststubinfo.py
new file mode 100644
--- /dev/null
+++ b/mypy/test/teststubinfo.py
@@ -0,0 +1,18 @@
+import unittest
+
+from mypy.stubinfo import is_legacy_bundled_package
+
+
+class TestStubInfo(unittest.TestCase):
+    def test_is_legacy_bundled_packages(self) -> None:
+        assert not is_legacy_bundled_package('foobar_asdf', 2)
+        assert not is_legacy_bundled_package('foobar_asdf', 3)
+
+        assert is_legacy_bundled_package('click', 2)
+        assert is_legacy_bundled_package('click', 3)
+
+        assert is_legacy_bundled_package('scribe', 2)
+        assert not is_legacy_bundled_package('scribe', 3)
+
+        assert not is_legacy_bundled_package('dataclasses', 2)
+        assert is_legacy_bundled_package('dataclasses', 3)
diff --git a/test-data/unit/python2eval.test b/test-data/unit/python2eval.test
--- a/test-data/unit/python2eval.test
+++ b/test-data/unit/python2eval.test
@@ -433,3 +433,17 @@ _testDefaultDictInference.py:5: note: Revealed type is "collections.defaultdict[
 from collections import abc
 [out]
 _testIgnorePython3StdlibStubs_python2.py:1: error: Module "collections" has no attribute "abc"
+
+[case testNoApprovedPython2StubInstalled_python2]
+# flags: --ignore-missing-imports
+import scribe
+from scribe import x
+import maxminddb
+import foobar_asdf
+[out]
+_testNoApprovedPython2StubInstalled_python2.py:2: error: Library stubs not installed for "scribe" (or incompatible with Python 2.7)
+_testNoApprovedPython2StubInstalled_python2.py:2: note: Hint: "python3 -m pip install types-scribe"
+_testNoApprovedPython2StubInstalled_python2.py:2: note: (or run "mypy --install-types" to install all missing stub packages)
+_testNoApprovedPython2StubInstalled_python2.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+_testNoApprovedPython2StubInstalled_python2.py:4: error: Library stubs not installed for "maxminddb" (or incompatible with Python 2.7)
+_testNoApprovedPython2StubInstalled_python2.py:4: note: Hint: "python3 -m pip install types-maxminddb"
diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test
--- a/test-data/unit/pythoneval.test
+++ b/test-data/unit/pythoneval.test
@@ -1508,3 +1508,26 @@ x = 0
 [out]
 mypy: "tmp/typing.py" shadows library module "typing"
 note: A user-defined top-level module with name "typing" is not supported
+
+[case testIgnoreImportIfNoPython3StubAvailable]
+# flags: --ignore-missing-imports
+import scribe  # No Python 3 stubs available for scribe
+from scribe import x
+import maxminddb  # Python 3 stubs available for maxminddb
+import foobar_asdf
+[out]
+_testIgnoreImportIfNoPython3StubAvailable.py:4: error: Library stubs not installed for "maxminddb" (or incompatible with Python 3.6)
+_testIgnoreImportIfNoPython3StubAvailable.py:4: note: Hint: "python3 -m pip install types-maxminddb"
+_testIgnoreImportIfNoPython3StubAvailable.py:4: note: (or run "mypy --install-types" to install all missing stub packages)
+_testIgnoreImportIfNoPython3StubAvailable.py:4: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+
+[case testNoPython3StubAvailable]
+import scribe
+from scribe import x
+import maxminddb
+[out]
+_testNoPython3StubAvailable.py:1: error: Cannot find implementation or library stub for module named "scribe"
+_testNoPython3StubAvailable.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+_testNoPython3StubAvailable.py:3: error: Library stubs not installed for "maxminddb" (or incompatible with Python 3.6)
+_testNoPython3StubAvailable.py:3: note: Hint: "python3 -m pip install types-maxminddb"
+_testNoPython3StubAvailable.py:3: note: (or run "mypy --install-types" to install all missing stub packages)