Repository: PyCQA/pyflakes

Issue:
python 3.8 SyntaxWarning: "is" with a literal. Did you mean "=="?
Hi there -

I searched a bit for this but couldn't find it.   This is a warning generated by Python 3.8 and I think it's new.  It's very hard to get pytest to raise for this, so it would be better if it were in pyflakes / flake8.

demo:

```
"""test a syntax warning."""

x = 5

if x is ():
    print("hi")

```

```
$ /opt/python3.8/bin/python3 -m py_compile test.py
test.py:5: SyntaxWarning: "is" with a literal. Did you mean "=="?
  if x is ():

```



Discussion:
bitglue: Pyflakes doesn't otherwise check for syntax errors, since those get raised when generating the AST. Why is this one necessary to check specially?
zzzeek: > Pyflakes doesn't otherwise check for syntax errors, since those get raised when generating the AST. Why is this one necessary to check specially?

this error is not "raised" by the interpreter, it's given out as a warning, which is very difficult to detect in a test suite.   We raise exceptions on warnings however this particular one is still sneaking through before the filter gets set, apparently, as the condition here is virtually identical to existing checks that pyflakes performs it's appropriate that this one be present as well.

asottile: > 
> 
> Pyflakes doesn't otherwise check for syntax errors, since those get raised when generating the AST. Why is this one necessary to check specially?

this isn't a `SyntaxError`, this is a warning in python3.8+ (afaict the check that's being augmented here was also in response to the new warning in python3.8)

Patch:
diff --git a/pyflakes/checker.py b/pyflakes/checker.py
index eca2002..351ab12 100644
--- a/pyflakes/checker.py
+++ b/pyflakes/checker.py
@@ -78,6 +78,51 @@ else:
     LOOP_TYPES = (ast.While, ast.For)
     FUNCTION_TYPES = (ast.FunctionDef,)
 
+
+if PY38_PLUS:
+    def _is_singleton(node):  # type: (ast.AST) -> bool
+        return (
+            isinstance(node, ast.Constant) and
+            isinstance(node.value, (bool, type(Ellipsis), type(None)))
+        )
+elif not PY2:
+    def _is_singleton(node):  # type: (ast.AST) -> bool
+        return isinstance(node, (ast.NameConstant, ast.Ellipsis))
+else:
+    def _is_singleton(node):  # type: (ast.AST) -> bool
+        return (
+            isinstance(node, ast.Name) and
+            node.id in {'True', 'False', 'Ellipsis', 'None'}
+        )
+
+
+def _is_tuple_constant(node):  # type: (ast.AST) -> bool
+    return (
+        isinstance(node, ast.Tuple) and
+        all(_is_constant(elt) for elt in node.elts)
+    )
+
+
+if PY38_PLUS:
+    def _is_constant(node):
+        return isinstance(node, ast.Constant) or _is_tuple_constant(node)
+else:
+    _const_tps = (ast.Str, ast.Num)
+    if not PY2:
+        _const_tps += (ast.Bytes,)
+
+    def _is_constant(node):
+        return (
+            isinstance(node, _const_tps) or
+            _is_singleton(node) or
+            _is_tuple_constant(node)
+        )
+
+
+def _is_const_non_singleton(node):  # type: (ast.AST) -> bool
+    return _is_constant(node) and not _is_singleton(node)
+
+
 # https://github.com/python/typed_ast/blob/1.4.0/ast27/Parser/tokenizer.c#L102-L104
 TYPE_COMMENT_RE = re.compile(r'^#\s*type:\s*')
 # https://github.com/python/typed_ast/blob/1.4.0/ast27/Parser/tokenizer.c#L1408-L1413
@@ -618,40 +663,55 @@ def getNodeName(node):
         return node.name
 
 
-def is_typing_overload(value, scope_stack):
-    def name_is_typing_overload(name):  # type: (str) -> bool
+def _is_typing(node, typing_attr, scope_stack):
+    def _bare_name_is_attr(name):
+        expected_typing_names = {
+            'typing.{}'.format(typing_attr),
+            'typing_extensions.{}'.format(typing_attr),
+        }
         for scope in reversed(scope_stack):
             if name in scope:
                 return (
                     isinstance(scope[name], ImportationFrom) and
-                    scope[name].fullName in (
-                        'typing.overload', 'typing_extensions.overload',
-                    )
+                    scope[name].fullName in expected_typing_names
                 )
 
         return False
 
-    def is_typing_overload_decorator(node):
-        return (
-            (
-                isinstance(node, ast.Name) and name_is_typing_overload(node.id)
-            ) or (
-                isinstance(node, ast.Attribute) and
-                isinstance(node.value, ast.Name) and
-                node.value.id == 'typing' and
-                node.attr == 'overload'
-            )
+    return (
+        (
+            isinstance(node, ast.Name) and
+            _bare_name_is_attr(node.id)
+        ) or (
+            isinstance(node, ast.Attribute) and
+            isinstance(node.value, ast.Name) and
+            node.value.id in {'typing', 'typing_extensions'} and
+            node.attr == typing_attr
         )
+    )
+
 
+def is_typing_overload(value, scope_stack):
     return (
         isinstance(value.source, FUNCTION_TYPES) and
         any(
-            is_typing_overload_decorator(dec)
+            _is_typing(dec, 'overload', scope_stack)
             for dec in value.source.decorator_list
         )
     )
 
 
+def in_annotation(func):
+    @functools.wraps(func)
+    def in_annotation_func(self, *args, **kwargs):
+        orig, self._in_annotation = self._in_annotation, True
+        try:
+            return func(self, *args, **kwargs)
+        finally:
+            self._in_annotation = orig
+    return in_annotation_func
+
+
 def make_tokens(code):
     # PY3: tokenize.tokenize requires readline of bytes
     if not isinstance(code, bytes):
@@ -738,6 +798,9 @@ class Checker(object):
     nodeDepth = 0
     offset = None
     traceTree = False
+    _in_annotation = False
+    _in_typing_literal = False
+    _in_deferred = False
 
     builtIns = set(builtin_vars).union(_MAGIC_GLOBALS)
     _customBuiltIns = os.environ.get('PYFLAKES_BUILTINS')
@@ -769,6 +832,7 @@ class Checker(object):
         for builtin in self.builtIns:
             self.addBinding(None, Builtin(builtin))
         self.handleChildren(tree)
+        self._in_deferred = True
         self.runDeferred(self._deferredFunctions)
         # Set _deferredFunctions to None so that deferFunction will fail
         # noisily if called after we've run through the deferred functions.
@@ -1009,12 +1073,30 @@ class Checker(object):
 
         self.scope[value.name] = value
 
+    def _unknown_handler(self, node):
+        # this environment variable configures whether to error on unknown
+        # ast types.
+        #
+        # this is silent by default but the error is enabled for the pyflakes
+        # testsuite.
+        #
+        # this allows new syntax to be added to python without *requiring*
+        # changes from the pyflakes side.  but will still produce an error
+        # in the pyflakes testsuite (so more specific handling can be added if
+        # needed).
+        if os.environ.get('PYFLAKES_ERROR_UNKNOWN'):
+            raise NotImplementedError('Unexpected type: {}'.format(type(node)))
+        else:
+            self.handleChildren(node)
+
     def getNodeHandler(self, node_class):
         try:
             return self._nodeHandlers[node_class]
         except KeyError:
             nodeType = getNodeType(node_class)
-        self._nodeHandlers[node_class] = handler = getattr(self, nodeType)
+        self._nodeHandlers[node_class] = handler = getattr(
+            self, nodeType, self._unknown_handler,
+        )
         return handler
 
     def handleNodeLoad(self, node):
@@ -1281,6 +1363,7 @@ class Checker(object):
         self.popScope()
         self.scopeStack = saved_stack
 
+    @in_annotation
     def handleStringAnnotation(self, s, node, ref_lineno, ref_col_offset, err):
         try:
             tree = ast.parse(s)
@@ -1304,6 +1387,7 @@ class Checker(object):
 
         self.handleNode(parsed_annotation, node)
 
+    @in_annotation
     def handleAnnotation(self, annotation, node):
         if isinstance(annotation, ast.Str):
             # Defer handling forward annotation.
@@ -1316,7 +1400,8 @@ class Checker(object):
                 messages.ForwardAnnotationSyntaxError,
             ))
         elif self.annotationsFutureEnabled:
-            self.deferFunction(lambda: self.handleNode(annotation, node))
+            fn = in_annotation(Checker.handleNode)
+            self.deferFunction(lambda: fn(self, annotation, node))
         else:
             self.handleNode(annotation, node)
 
@@ -1332,9 +1417,19 @@ class Checker(object):
 
     # "expr" type nodes
     BOOLOP = UNARYOP = IFEXP = SET = \
-        REPR = ATTRIBUTE = SUBSCRIPT = \
+        REPR = ATTRIBUTE = \
         STARRED = NAMECONSTANT = NAMEDEXPR = handleChildren
 
+    def SUBSCRIPT(self, node):
+        if _is_typing(node.value, 'Literal', self.scopeStack):
+            orig, self._in_typing_literal = self._in_typing_literal, True
+            try:
+                self.handleChildren(node)
+            finally:
+                self._in_typing_literal = orig
+        else:
+            self.handleChildren(node)
+
     def _handle_string_dot_format(self, node):
         try:
             placeholders = tuple(parse_format_string(node.func.value.s))
@@ -1575,7 +1670,27 @@ class Checker(object):
             self._handle_percent_format(node)
         self.handleChildren(node)
 
-    NUM = STR = BYTES = ELLIPSIS = CONSTANT = ignore
+    def STR(self, node):
+        if self._in_annotation and not self._in_typing_literal:
+            fn = functools.partial(
+                self.handleStringAnnotation,
+                node.s,
+                node,
+                node.lineno,
+                node.col_offset,
+                messages.ForwardAnnotationSyntaxError,
+            )
+            if self._in_deferred:
+                fn()
+            else:
+                self.deferFunction(fn)
+
+    if PY38_PLUS:
+        def CONSTANT(self, node):
+            if isinstance(node.value, str):
+                return self.STR(node)
+    else:
+        NUM = BYTES = ELLIPSIS = CONSTANT = ignore
 
     # "slice" type nodes
     SLICE = EXTSLICE = INDEX = handleChildren
@@ -1738,7 +1853,7 @@ class Checker(object):
                 break
             # Handle Try/TryFinally difference in Python < and >= 3.3
             if hasattr(n, 'finalbody') and isinstance(node, ast.Continue):
-                if n_child in n.finalbody:
+                if n_child in n.finalbody and not PY38_PLUS:
                     self.report(messages.ContinueInFinally, node)
                     return
         if isinstance(node, ast.Continue):
@@ -1799,6 +1914,10 @@ class Checker(object):
             addArgs(node.args.args)
             defaults = node.args.defaults
         else:
+            if PY38_PLUS:
+                for arg in node.args.posonlyargs:
+                    args.append(arg.arg)
+                    annotations.append(arg.annotation)
             for arg in node.args.args + node.args.kwonlyargs:
                 args.append(arg.arg)
                 annotations.append(arg.annotation)
@@ -2050,14 +2169,14 @@ class Checker(object):
             self.handleNode(node.value, node)
 
     def COMPARE(self, node):
-        literals = (ast.Str, ast.Num)
-        if not PY2:
-            literals += (ast.Bytes,)
-
         left = node.left
         for op, right in zip(node.ops, node.comparators):
-            if (isinstance(op, (ast.Is, ast.IsNot)) and
-                    (isinstance(left, literals) or isinstance(right, literals))):
+            if (
+                    isinstance(op, (ast.Is, ast.IsNot)) and (
+                        _is_const_non_singleton(left) or
+                        _is_const_non_singleton(right)
+                    )
+            ):
                 self.report(messages.IsLiteral, node)
             left = right
 
diff --git a/pyflakes/messages.py b/pyflakes/messages.py
index cf67cf8..e93a493 100644
--- a/pyflakes/messages.py
+++ b/pyflakes/messages.py
@@ -265,7 +265,7 @@ class InvalidPrintSyntax(Message):
 
 
 class IsLiteral(Message):
-    message = 'use ==/!= to compare str, bytes, and int literals'
+    message = 'use ==/!= to compare constant literals (str, bytes, int, float, tuple)'
 
 
 class FStringMissingPlaceholders(Message):
diff --git a/tox.ini b/tox.ini
index 5821483..f2256c8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,6 +5,7 @@ envlist =
 
 [testenv]
 deps = flake8==3.6.0
+setenv = PYFLAKES_ERROR_UNKNOWN=1
 commands =
     python -m unittest discover pyflakes
     flake8 pyflakes setup.py


Test patch:
diff --git a/pyflakes/test/test_is_literal.py b/pyflakes/test/test_is_literal.py
index 9280cda..fbbb205 100644
--- a/pyflakes/test/test_is_literal.py
+++ b/pyflakes/test/test_is_literal.py
@@ -198,3 +198,25 @@ class Test(TestCase):
         if 4 < x is 'foo':
             pass
         """, IsLiteral)
+
+    def test_is_tuple_constant(self):
+        self.flakes('''\
+            x = 5
+            if x is ():
+                pass
+        ''', IsLiteral)
+
+    def test_is_tuple_constant_containing_constants(self):
+        self.flakes('''\
+            x = 5
+            if x is (1, '2', True, (1.5, ())):
+                pass
+        ''', IsLiteral)
+
+    def test_is_tuple_containing_variables_ok(self):
+        # a bit nonsensical, but does not trigger a SyntaxWarning
+        self.flakes('''\
+            x = 5
+            if x is (x,):
+                pass
+        ''')
diff --git a/pyflakes/test/test_other.py b/pyflakes/test/test_other.py
index df2f790..282accb 100644
--- a/pyflakes/test/test_other.py
+++ b/pyflakes/test/test_other.py
@@ -493,8 +493,10 @@ class Test(TestCase):
                     continue
         ''')
 
+    @skipIf(version_info > (3, 8), "Python <= 3.8 only")
     def test_continueInFinally(self):
         # 'continue' inside 'finally' is a special syntax error
+        # that is removed in 3.8
         self.flakes('''
         while True:
             try:
@@ -2003,6 +2005,7 @@ class TestAsyncStatements(TestCase):
         ''', m.BreakOutsideLoop)
 
     @skipIf(version_info < (3, 5), 'new in Python 3.5')
+    @skipIf(version_info > (3, 8), "Python <= 3.8 only")
     def test_continueInAsyncForFinally(self):
         self.flakes('''
         async def read_data(db):
diff --git a/pyflakes/test/test_type_annotations.py b/pyflakes/test/test_type_annotations.py
index 289535d..15c658b 100644
--- a/pyflakes/test/test_type_annotations.py
+++ b/pyflakes/test/test_type_annotations.py
@@ -42,6 +42,7 @@ class TestTypeAnnotations(TestCase):
     def test_typingExtensionsOverload(self):
         """Allow intentional redefinitions via @typing_extensions.overload"""
         self.flakes("""
+        import typing_extensions
         from typing_extensions import overload
 
         @overload
@@ -54,6 +55,17 @@ class TestTypeAnnotations(TestCase):
 
         def f(s):
             return s
+
+        @typing_extensions.overload
+        def g(s):  # type: (None) -> None
+            pass
+
+        @typing_extensions.overload
+        def g(s):  # type: (int) -> int
+            pass
+
+        def g(s):
+            return s
         """)
 
     @skipIf(version_info < (3, 5), 'new in Python 3.5')
@@ -418,3 +430,72 @@ class TestTypeAnnotations(TestCase):
                 Y = 2
                 return Y
         """, m.UndefinedName)
+
+    @skipIf(version_info < (3, 8), 'new in Python 3.8')
+    def test_positional_only_argument_annotations(self):
+        self.flakes("""
+        from x import C
+
+        def f(c: C, /): ...
+        """)
+
+    @skipIf(version_info < (3,), 'new in Python 3')
+    def test_partially_quoted_type_annotation(self):
+        self.flakes("""
+        from queue import Queue
+        from typing import Optional
+
+        def f() -> Optional['Queue[str]']:
+            return None
+        """)
+
+    @skipIf(version_info < (3,), 'new in Python 3')
+    def test_literal_type_typing(self):
+        self.flakes("""
+        from typing import Literal
+
+        def f(x: Literal['some string']) -> None:
+            return None
+        """)
+
+    @skipIf(version_info < (3,), 'new in Python 3')
+    def test_literal_type_typing_extensions(self):
+        self.flakes("""
+        from typing_extensions import Literal
+
+        def f(x: Literal['some string']) -> None:
+            return None
+        """)
+
+    @skipIf(version_info < (3,), 'new in Python 3')
+    def test_literal_union_type_typing(self):
+        self.flakes("""
+        from typing import Literal
+
+        def f(x: Literal['some string', 'foo bar']) -> None:
+            return None
+        """)
+
+    @skipIf(version_info < (3,), 'new in Python 3')
+    def test_deferred_twice_annotation(self):
+        self.flakes("""
+            from queue import Queue
+            from typing import Optional
+
+
+            def f() -> "Optional['Queue[str]']":
+                return None
+        """)
+
+    @skipIf(version_info < (3, 7), 'new in Python 3.7')
+    def test_partial_string_annotations_with_future_annotations(self):
+        self.flakes("""
+            from __future__ import annotations
+
+            from queue import Queue
+            from typing import Optional
+
+
+            def f() -> Optional['Queue[str]']:
+                return None
+        """)