Repository: python/mypy

Issue:
Docs for --python-version are wrong
The docs for `--python-version` and the corresponding config option (`python_version`) state that they default to the Python version used to run mypy. This is not true and has never been true -- the default is fixed and taken from `defaults.PYTHON3_VERSION`.

In the [discussion about the PEP 561 implementation](https://github.com/python/mypy/pull/4403#discussion_r169826677) I've talked myself into believing that we should do what the docs state rather than what we currently implement.

@JukkaL Before we go ahead with this change I'd like your opinion, please.


Discussion:
My preference would be to make using PEP 561 installed stubs as simple as possible. The most straightforward way to use installed stubs would be to have both mypy and the stubs use the Python installation used to run mypy. Then no extra mypy options should be needed, if the same Python is also used to install the stub packages. This implies that `--python-version` should default to the Python version used to run mypy instead of the current behavior. So I'm in agreement here.

However, if we make this change, I'd like to update the errors generated when using unsupported syntax to document how to change the target Python version. For example, if the user is using Python 3.5 and tries to use the variable annotation syntax which was introduced in 3.6, the current error isn't necessarily very helpful:

```
foo.py:8: error: Variable annotation syntax is only supported in Python 3.6 and greater
```

Something like this would be better:

```
foo.py:8: error: Variable annotation syntax is only supported in Python 3.6 and greater
foo.py:8: note: Use Python 3.6+ to run mypy to enable 3.6 features (or use --python-version)
```

Great. I think improving the errors is an optional task; the situation isn't much different now (in fact it's worse now since simply running mypy with a different Python version doesn't help, and the docs are wrong about that). [CC: @ethanhs ]

Resolving patch:
diff --git a/mypy/options.py b/mypy/options.py
--- a/mypy/options.py
+++ b/mypy/options.py
@@ -52,7 +52,7 @@ def __init__(self) -> None:
 
         # -- build options --
         self.build_type = BuildType.STANDARD
-        self.python_version = defaults.PYTHON3_VERSION
+        self.python_version = sys.version_info[:2]  # type: Tuple[int, int]
         self.platform = sys.platform
         self.custom_typing_module = None  # type: Optional[str]
         self.custom_typeshed_dir = None  # type: Optional[str]
diff --git a/mypy/waiter.py b/mypy/waiter.py
--- a/mypy/waiter.py
+++ b/mypy/waiter.py
@@ -160,7 +160,7 @@ def load_log_file(self) -> Optional[List[Dict[str, Dict[str, Any]]]]:
                 test_log = json.load(fp)
         except FileNotFoundError:
             test_log = []
-        except json.JSONDecodeError:
+        except ValueError:
             print('corrupt test log file {}'.format(self.FULL_LOG_FILENAME), file=sys.stderr)
             test_log = []
         return test_log
diff --git a/runtests.py b/runtests.py
--- a/runtests.py
+++ b/runtests.py
@@ -78,9 +78,10 @@ def add_mypy_cmd(self, name: str, mypy_args: List[str], cwd: Optional[str] = Non
     def add_mypy(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
         self.add_mypy_cmd(name, list(args), cwd=cwd)
 
-    def add_mypy_modules(self, name: str, modules: Iterable[str],
-                         cwd: Optional[str] = None) -> None:
-        args = list(itertools.chain(*(['-m', mod] for mod in modules)))
+    def add_mypy_modules(self, name: str, modules: Iterable[str], cwd: Optional[str] = None,
+                         extra_args: Optional[List[str]] = None) -> None:
+        args = extra_args or []
+        args.extend(list(itertools.chain(*(['-m', mod] for mod in modules))))
         self.add_mypy_cmd(name, args, cwd=cwd)
 
     def add_mypy_package(self, name: str, packagename: str, *flags: str) -> None:
@@ -256,7 +257,8 @@ def add_stubs(driver: Driver) -> None:
                 module = file_to_module(f[len(stubdir) + 1:])
                 modules.add(module)
 
-    driver.add_mypy_modules('stubs', sorted(modules))
+    # these require at least 3.5 otherwise it will fail trying to import zipapp
+    driver.add_mypy_modules('stubs', sorted(modules), extra_args=['--python-version=3.5'])
 
 
 def add_stdlibsamples(driver: Driver) -> None:
@@ -276,7 +278,11 @@ def add_stdlibsamples(driver: Driver) -> None:
 
 def add_samples(driver: Driver) -> None:
     for f in find_files(os.path.join('test-data', 'samples'), suffix='.py'):
-        driver.add_mypy('file %s' % f, f)
+        if f == os.path.join('test-data', 'samples', 'crawl2.py'):
+            # This test requires 3.5 for async functions
+            driver.add_mypy_cmd('file {}'.format(f), ['--python-version=3.5', f])
+        else:
+            driver.add_mypy('file %s' % f, f)
 
 
 def usage(status: int) -> None:


Test patch:
diff --git a/mypy/test/testdiff.py b/mypy/test/testdiff.py
--- a/mypy/test/testdiff.py
+++ b/mypy/test/testdiff.py
@@ -5,6 +5,7 @@
 
 from mypy import build
 from mypy.build import BuildSource
+from mypy.defaults import PYTHON3_VERSION
 from mypy.errors import CompileError
 from mypy.nodes import MypyFile
 from mypy.options import Options
@@ -53,6 +54,7 @@ def build(self, source: str) -> Tuple[List[str], Optional[Dict[str, MypyFile]]]:
         options.use_builtins_fixtures = True
         options.show_traceback = True
         options.cache_dir = os.devnull
+        options.python_version = PYTHON3_VERSION
         try:
             result = build.build(sources=[BuildSource('main', None, source)],
                                  options=options,
diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py
--- a/mypy/test/testmerge.py
+++ b/mypy/test/testmerge.py
@@ -6,6 +6,7 @@
 
 from mypy import build
 from mypy.build import BuildManager, BuildSource, State, Graph
+from mypy.defaults import PYTHON3_VERSION
 from mypy.errors import Errors, CompileError
 from mypy.nodes import (
     Node, MypyFile, SymbolTable, SymbolTableNode, TypeInfo, Expression, Var, TypeVarExpr,
@@ -106,6 +107,7 @@ def build(self, source: str) -> Tuple[List[str], Optional[BuildManager], Dict[st
         options.fine_grained_incremental = True
         options.use_builtins_fixtures = True
         options.show_traceback = True
+        options.python_version = PYTHON3_VERSION
         main_path = os.path.join(test_temp_dir, 'main')
         with open(main_path, 'w') as f:
             f.write(source)
diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py
--- a/mypy/test/testpythoneval.py
+++ b/mypy/test/testpythoneval.py
@@ -19,6 +19,7 @@
 import pytest  # type: ignore  # no pytest in typeshed
 from typing import Dict, List, Tuple, Optional
 
+from mypy.defaults import PYTHON3_VERSION
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal
@@ -60,6 +61,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None:
             return
     else:
         interpreter = python3_path
+        mypy_cmdline.append('--python-version={}'.format('.'.join(map(str, PYTHON3_VERSION))))
 
     # Write the program to a file.
     program = '_' + testcase.name + '.py'
diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py
--- a/mypy/test/testsemanal.py
+++ b/mypy/test/testsemanal.py
@@ -6,6 +6,7 @@
 
 from mypy import build
 from mypy.build import BuildSource
+from mypy.defaults import PYTHON3_VERSION
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages, testfile_pyversion,
 )
@@ -38,6 +39,7 @@ def get_semanal_options() -> Options:
     options.use_builtins_fixtures = True
     options.semantic_analysis_only = True
     options.show_traceback = True
+    options.python_version = PYTHON3_VERSION
     return options
 
 
diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test
--- a/test-data/unit/cmdline.test
+++ b/test-data/unit/cmdline.test
@@ -581,7 +581,7 @@ m.py:6: error: Explicit "Any" is not allowed
 m.py:9: error: Explicit "Any" is not allowed
 
 [case testDisallowAnyExplicitVarDeclaration]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 
 [file mypy.ini]
 [[mypy]
@@ -601,7 +601,7 @@ m.py:3: error: Explicit "Any" is not allowed
 m.py:5: error: Explicit "Any" is not allowed
 
 [case testDisallowAnyExplicitGenericVarDeclaration]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 
 [file mypy.ini]
 [[mypy]
@@ -785,7 +785,7 @@ N = TypedDict('N', {'x': str, 'y': List})  # no error
 m.py:4: error: Explicit "Any" is not allowed
 
 [case testDisallowAnyGenericsTupleNoTypeParams]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
@@ -821,7 +821,7 @@ def g(s: List[Tuple[str, str]]) -> None: pass  # no error
 m.py:3: error: Missing type parameters for generic type
 
 [case testDisallowAnyGenericsTypeType]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
@@ -858,7 +858,7 @@ def g(l: L[str]) -> None: pass  # no error
 m.py:5: error: Missing type parameters for generic type
 
 [case testDisallowAnyGenericsGenericAlias]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
@@ -882,7 +882,7 @@ m.py:7: error: Missing type parameters for generic type
 m.py:11: error: Missing type parameters for generic type
 
 [case testDisallowAnyGenericsPlainList]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
@@ -906,7 +906,7 @@ m.py:8: error: Need type annotation for 'x'
 m.py:9: error: Missing type parameters for generic type
 
 [case testDisallowAnyGenericsCustomGenericClass]
-# cmd: mypy m.py
+# cmd: mypy --python-version=3.6 m.py
 [file mypy.ini]
 [[mypy]
 [[mypy-m]
diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test
--- a/test-data/unit/reports.test
+++ b/test-data/unit/reports.test
@@ -281,7 +281,7 @@ Total      0      16    100.00%
 
 
 [case testAnyExpressionsReportTypesOfAny]
-# cmd: mypy --any-exprs-report report n.py
+# cmd: mypy --python-version=3.6 --any-exprs-report report n.py
 
 [file n.py]
 from typing import Any, List