Repository: PyCQA/pyflakes
Pull request #438 (2019-03-02 06:15:09)

Shebang detection improvements


Patch:
diff --git a/NEWS.rst b/NEWS.rst
index 8bf9023..993a545 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -1,3 +1,7 @@
+2.1.1 (2019-02-28)
+- Fix reported line number for type comment errors
+- Fix typing.overload check to only check imported names
+
 2.1.0 (2019-01-23)
 
 - Allow intentional assignment to variables named ``_``
diff --git a/README.rst b/README.rst
index 4b1d3c2..ca802a8 100644
--- a/README.rst
+++ b/README.rst
@@ -86,4 +86,4 @@ All changes should include tests and pass flake8_.
 Changelog
 ---------
 
-Please see `NEWS.rst <NEWS.rst>`_.
+Please see `NEWS.rst <https://github.com/PyCQA/pyflakes/blob/master/NEWS.rst>`_.
diff --git a/pyflakes/__init__.py b/pyflakes/__init__.py
index a33997d..55fa725 100644
--- a/pyflakes/__init__.py
+++ b/pyflakes/__init__.py
@@ -1,1 +1,1 @@
-__version__ = '2.1.0'
+__version__ = '2.1.1'
diff --git a/pyflakes/api.py b/pyflakes/api.py
index 6226e5d..50e8ad8 100644
--- a/pyflakes/api.py
+++ b/pyflakes/api.py
@@ -14,7 +14,7 @@ from pyflakes import reporter as modReporter
 
 __all__ = ['check', 'checkPath', 'checkRecursive', 'iterSourceCode', 'main']
 
-PYTHON_SHEBANG_REGEX = re.compile(br'^#!.*\bpython[23w]?\b\s*$')
+PYTHON_SHEBANG_REGEX = re.compile(br'^#!.*\bpython([23](\.\d+)?|w)?[dmu]?\s')
 
 
 def check(codeString, filename, reporter=None):
@@ -118,8 +118,7 @@ def isPythonFile(filename):
     except IOError:
         return False
 
-    first_line = text.splitlines()[0]
-    return PYTHON_SHEBANG_REGEX.match(first_line)
+    return PYTHON_SHEBANG_REGEX.match(text)
 
 
 def iterSourceCode(paths):
diff --git a/pyflakes/checker.py b/pyflakes/checker.py
index 0e636c1..f9ae0aa 100644
--- a/pyflakes/checker.py
+++ b/pyflakes/checker.py
@@ -12,6 +12,7 @@ import doctest
 import functools
 import os
 import re
+import string
 import sys
 import tokenize
 
@@ -29,6 +30,8 @@ except AttributeError:
 
 builtin_vars = dir(__import__('__builtin__' if PY2 else 'builtins'))
 
+parse_format_string = string.Formatter().parse
+
 if PY2:
     tokenize_tokenize = tokenize.generate_tokens
 else:
@@ -81,6 +84,87 @@ TYPE_IGNORE_RE = re.compile(TYPE_COMMENT_RE.pattern + r'ignore\s*(#|$)')
 TYPE_FUNC_RE = re.compile(r'^(\(.*?\))\s*->\s*(.*)$')
 
 
+MAPPING_KEY_RE = re.compile(r'\(([^()]*)\)')
+CONVERSION_FLAG_RE = re.compile('[#0+ -]*')
+WIDTH_RE = re.compile(r'(?:\*|\d*)')
+PRECISION_RE = re.compile(r'(?:\.(?:\*|\d*))?')
+LENGTH_RE = re.compile('[hlL]?')
+# https://docs.python.org/3/library/stdtypes.html#old-string-formatting
+VALID_CONVERSIONS = frozenset('diouxXeEfFgGcrsa%')
+
+
+def _must_match(regex, string, pos):
+    # type: (Pattern[str], str, int) -> Match[str]
+    match = regex.match(string, pos)
+    assert match is not None
+    return match
+
+
+def parse_percent_format(s):  # type: (str) -> Tuple[PercentFormat, ...]
+    """Parses the string component of a `'...' % ...` format call
+
+    Copied from https://github.com/asottile/pyupgrade at v1.20.1
+    """
+
+    def _parse_inner():
+        # type: () -> Generator[PercentFormat, None, None]
+        string_start = 0
+        string_end = 0
+        in_fmt = False
+
+        i = 0
+        while i < len(s):
+            if not in_fmt:
+                try:
+                    i = s.index('%', i)
+                except ValueError:  # no more % fields!
+                    yield s[string_start:], None
+                    return
+                else:
+                    string_end = i
+                    i += 1
+                    in_fmt = True
+            else:
+                key_match = MAPPING_KEY_RE.match(s, i)
+                if key_match:
+                    key = key_match.group(1)  # type: Optional[str]
+                    i = key_match.end()
+                else:
+                    key = None
+
+                conversion_flag_match = _must_match(CONVERSION_FLAG_RE, s, i)
+                conversion_flag = conversion_flag_match.group() or None
+                i = conversion_flag_match.end()
+
+                width_match = _must_match(WIDTH_RE, s, i)
+                width = width_match.group() or None
+                i = width_match.end()
+
+                precision_match = _must_match(PRECISION_RE, s, i)
+                precision = precision_match.group() or None
+                i = precision_match.end()
+
+                # length modifier is ignored
+                i = _must_match(LENGTH_RE, s, i).end()
+
+                try:
+                    conversion = s[i]
+                except IndexError:
+                    raise ValueError('end-of-string while parsing format')
+                i += 1
+
+                fmt = (key, conversion_flag, width, precision, conversion)
+                yield s[string_start:string_end], fmt
+
+                in_fmt = False
+                string_start = i
+
+        if in_fmt:
+            raise ValueError('end-of-string while parsing format')
+
+    return tuple(_parse_inner())
+
+
 class _FieldsOrder(dict):
     """Fix order of AST node fields."""
 
@@ -822,22 +906,31 @@ class Checker(object):
     def getParent(self, node):
         # Lookup the first parent which is not Tuple, List or Starred
         while True:
-            node = node.parent
+            node = node._pyflakes_parent
             if not hasattr(node, 'elts') and not hasattr(node, 'ctx'):
                 return node
 
     def getCommonAncestor(self, lnode, rnode, stop):
-        if stop in (lnode, rnode) or not (hasattr(lnode, 'parent') and
-                                          hasattr(rnode, 'parent')):
+        if (
+                stop in (lnode, rnode) or
+                not (
+                    hasattr(lnode, '_pyflakes_parent') and
+                    hasattr(rnode, '_pyflakes_parent')
+                )
+        ):
             return None
         if lnode is rnode:
             return lnode
 
-        if (lnode.depth > rnode.depth):
-            return self.getCommonAncestor(lnode.parent, rnode, stop)
-        if (lnode.depth < rnode.depth):
-            return self.getCommonAncestor(lnode, rnode.parent, stop)
-        return self.getCommonAncestor(lnode.parent, rnode.parent, stop)
+        if (lnode._pyflakes_depth > rnode._pyflakes_depth):
+            return self.getCommonAncestor(lnode._pyflakes_parent, rnode, stop)
+        if (lnode._pyflakes_depth < rnode._pyflakes_depth):
+            return self.getCommonAncestor(lnode, rnode._pyflakes_parent, stop)
+        return self.getCommonAncestor(
+            lnode._pyflakes_parent,
+            rnode._pyflakes_parent,
+            stop,
+        )
 
     def descendantOf(self, node, ancestors, stop):
         for a in ancestors:
@@ -875,7 +968,7 @@ class Checker(object):
         - `node` is the statement responsible for the change
         - `value` is the new value, a Binding instance
         """
-        # assert value.source in (node, node.parent):
+        # assert value.source in (node, node._pyflakes_parent):
         for scope in self.scopeStack[::-1]:
             if value.name in scope:
                 break
@@ -1014,11 +1107,11 @@ class Checker(object):
 
         parent_stmt = self.getParent(node)
         if isinstance(parent_stmt, (FOR_TYPES, ast.comprehension)) or (
-                parent_stmt != node.parent and
+                parent_stmt != node._pyflakes_parent and
                 not self.isLiteralTupleUnpacking(parent_stmt)):
             binding = Binding(name, node)
         elif name == '__all__' and isinstance(self.scope, ModuleScope):
-            binding = ExportBinding(name, node.parent, self.scope)
+            binding = ExportBinding(name, node._pyflakes_parent, self.scope)
         elif isinstance(getattr(node, 'ctx', None), ast.Param):
             binding = Argument(name, self.getScopeNode(node))
         else:
@@ -1031,11 +1124,11 @@ class Checker(object):
             """
             Return `True` if node is part of a conditional body.
             """
-            current = getattr(node, 'parent', None)
+            current = getattr(node, '_pyflakes_parent', None)
             while current:
                 if isinstance(current, (ast.If, ast.While, ast.IfExp)):
                     return True
-                current = getattr(current, 'parent', None)
+                current = getattr(current, '_pyflakes_parent', None)
             return False
 
         name = getNodeName(node)
@@ -1122,8 +1215,8 @@ class Checker(object):
                                         self.isDocstring(node)):
             self.futuresAllowed = False
         self.nodeDepth += 1
-        node.depth = self.nodeDepth
-        node.parent = parent
+        node._pyflakes_depth = self.nodeDepth
+        node._pyflakes_parent = parent
         try:
             handler = self.getNodeHandler(node.__class__)
             handler(node)
@@ -1162,7 +1255,8 @@ class Checker(object):
         self.scopeStack = [self.scopeStack[0]]
         node_offset = self.offset or (0, 0)
         self.pushScope(DoctestScope)
-        self.addBinding(None, Builtin('_'))
+        if '_' not in self.scopeStack[0]:
+            self.addBinding(None, Builtin('_'))
         for example in examples:
             try:
                 tree = ast.parse(example.source, "<doctest>")
@@ -1231,10 +1325,250 @@ class Checker(object):
     PASS = ignore
 
     # "expr" type nodes
-    BOOLOP = BINOP = UNARYOP = IFEXP = SET = \
-        CALL = REPR = ATTRIBUTE = SUBSCRIPT = \
+    BOOLOP = UNARYOP = IFEXP = SET = \
+        REPR = ATTRIBUTE = SUBSCRIPT = \
         STARRED = NAMECONSTANT = handleChildren
 
+    def _handle_string_dot_format(self, node):
+        try:
+            placeholders = tuple(parse_format_string(node.func.value.s))
+        except ValueError as e:
+            self.report(messages.StringDotFormatInvalidFormat, node, e)
+            return
+
+        class state:  # py2-compatible `nonlocal`
+            auto = None
+            next_auto = 0
+
+        placeholder_positional = set()
+        placeholder_named = set()
+
+        def _add_key(fmtkey):
+            """Returns True if there is an error which should early-exit"""
+            if fmtkey is None:  # end of string or `{` / `}` escapes
+                return False
+
+            # attributes / indices are allowed in `.format(...)`
+            fmtkey, _, _ = fmtkey.partition('.')
+            fmtkey, _, _ = fmtkey.partition('[')
+
+            try:
+                fmtkey = int(fmtkey)
+            except ValueError:
+                pass
+            else:  # fmtkey was an integer
+                if state.auto is True:
+                    self.report(messages.StringDotFormatMixingAutomatic, node)
+                    return True
+                else:
+                    state.auto = False
+
+            if fmtkey == '':
+                if state.auto is False:
+                    self.report(messages.StringDotFormatMixingAutomatic, node)
+                    return True
+                else:
+                    state.auto = True
+
+                fmtkey = state.next_auto
+                state.next_auto += 1
+
+            if isinstance(fmtkey, int):
+                placeholder_positional.add(fmtkey)
+            else:
+                placeholder_named.add(fmtkey)
+
+            return False
+
+        for _, fmtkey, spec, _ in placeholders:
+            if _add_key(fmtkey):
+                return
+
+            # spec can also contain format specifiers
+            if spec is not None:
+                try:
+                    spec_placeholders = tuple(parse_format_string(spec))
+                except ValueError as e:
+                    self.report(messages.StringDotFormatInvalidFormat, node, e)
+                    return
+
+                for _, spec_fmtkey, spec_spec, _ in spec_placeholders:
+                    # can't recurse again
+                    if spec_spec is not None and '{' in spec_spec:
+                        self.report(
+                            messages.StringDotFormatInvalidFormat,
+                            node,
+                            'Max string recursion exceeded',
+                        )
+                        return
+                    if _add_key(spec_fmtkey):
+                        return
+
+        # bail early if there is *args or **kwargs
+        if (
+                # python 2.x *args / **kwargs
+                getattr(node, 'starargs', None) or
+                getattr(node, 'kwargs', None) or
+                # python 3.x *args
+                any(
+                    isinstance(arg, getattr(ast, 'Starred', ()))
+                    for arg in node.args
+                ) or
+                # python 3.x **kwargs
+                any(kwd.arg is None for kwd in node.keywords)
+        ):
+            return
+
+        substitution_positional = set(range(len(node.args)))
+        substitution_named = {kwd.arg for kwd in node.keywords}
+
+        extra_positional = substitution_positional - placeholder_positional
+        extra_named = substitution_named - placeholder_named
+
+        missing_arguments = (
+            (placeholder_positional | placeholder_named) -
+            (substitution_positional | substitution_named)
+        )
+
+        if extra_positional:
+            self.report(
+                messages.StringDotFormatExtraPositionalArguments,
+                node,
+                ', '.join(sorted(str(x) for x in extra_positional)),
+            )
+        if extra_named:
+            self.report(
+                messages.StringDotFormatExtraNamedArguments,
+                node,
+                ', '.join(sorted(extra_named)),
+            )
+        if missing_arguments:
+            self.report(
+                messages.StringDotFormatMissingArgument,
+                node,
+                ', '.join(sorted(str(x) for x in missing_arguments)),
+            )
+
+    def CALL(self, node):
+        if (
+                isinstance(node.func, ast.Attribute) and
+                isinstance(node.func.value, ast.Str) and
+                node.func.attr == 'format'
+        ):
+            self._handle_string_dot_format(node)
+        self.handleChildren(node)
+
+    def _handle_percent_format(self, node):
+        try:
+            placeholders = parse_percent_format(node.left.s)
+        except ValueError:
+            self.report(
+                messages.PercentFormatInvalidFormat,
+                node,
+                'incomplete format',
+            )
+            return
+
+        named = set()
+        positional_count = 0
+        positional = None
+        for _, placeholder in placeholders:
+            if placeholder is None:
+                continue
+            name, _, width, precision, conversion = placeholder
+
+            if conversion == '%':
+                continue
+
+            if conversion not in VALID_CONVERSIONS:
+                self.report(
+                    messages.PercentFormatUnsupportedFormatCharacter,
+                    node,
+                    conversion,
+                )
+
+            if positional is None and conversion:
+                positional = name is None
+
+            for part in (width, precision):
+                if part is not None and '*' in part:
+                    if not positional:
+                        self.report(
+                            messages.PercentFormatStarRequiresSequence,
+                            node,
+                        )
+                    else:
+                        positional_count += 1
+
+            if positional and name is not None:
+                self.report(
+                    messages.PercentFormatMixedPositionalAndNamed,
+                    node,
+                )
+                return
+            elif not positional and name is None:
+                self.report(
+                    messages.PercentFormatMixedPositionalAndNamed,
+                    node,
+                )
+                return
+
+            if positional:
+                positional_count += 1
+            else:
+                named.add(name)
+
+        if (
+                isinstance(node.right, (ast.List, ast.Tuple)) and
+                # does not have any *splats (py35+ feature)
+                not any(
+                    isinstance(elt, getattr(ast, 'Starred', ()))
+                    for elt in node.right.elts
+                )
+        ):
+            substitution_count = len(node.right.elts)
+            if positional and positional_count != substitution_count:
+                self.report(
+                    messages.PercentFormatPositionalCountMismatch,
+                    node,
+                    positional_count,
+                    substitution_count,
+                )
+            elif not positional:
+                self.report(messages.PercentFormatExpectedMapping, node)
+
+        if (
+                isinstance(node.right, ast.Dict) and
+                all(isinstance(k, ast.Str) for k in node.right.keys)
+        ):
+            if positional and positional_count > 1:
+                self.report(messages.PercentFormatExpectedSequence, node)
+                return
+
+            substitution_keys = {k.s for k in node.right.keys}
+            extra_keys = substitution_keys - named
+            missing_keys = named - substitution_keys
+            if not positional and extra_keys:
+                self.report(
+                    messages.PercentFormatExtraNamedArguments,
+                    node,
+                    ', '.join(sorted(extra_keys)),
+                )
+            if not positional and missing_keys:
+                self.report(
+                    messages.PercentFormatMissingArgument,
+                    node,
+                    ', '.join(sorted(missing_keys)),
+                )
+
+    def BINOP(self, node):
+        if (
+                isinstance(node.op, ast.Mod) and
+                isinstance(node.left, ast.Str)
+        ):
+            self._handle_percent_format(node)
+        self.handleChildren(node)
+
     NUM = STR = BYTES = ELLIPSIS = CONSTANT = ignore
 
     # "slice" type nodes
@@ -1263,7 +1597,24 @@ class Checker(object):
             self.report(messages.RaiseNotImplemented, node)
 
     # additional node types
-    COMPREHENSION = KEYWORD = FORMATTEDVALUE = JOINEDSTR = handleChildren
+    COMPREHENSION = KEYWORD = FORMATTEDVALUE = handleChildren
+
+    _in_fstring = False
+
+    def JOINEDSTR(self, node):
+        if (
+                # the conversion / etc. flags are parsed as f-strings without
+                # placeholders
+                not self._in_fstring and
+                not any(isinstance(x, ast.FormattedValue) for x in node.values)
+        ):
+            self.report(messages.FStringMissingPlaceholders, node)
+
+        self._in_fstring, orig = True, self._in_fstring
+        try:
+            self.handleChildren(node)
+        finally:
+            self._in_fstring = orig
 
     def DICT(self, node):
         # Complain if there are duplicate keys with different values
@@ -1355,7 +1706,7 @@ class Checker(object):
         if isinstance(node.ctx, (ast.Load, ast.AugLoad)):
             self.handleNodeLoad(node)
             if (node.id == 'locals' and isinstance(self.scope, FunctionScope) and
-                    isinstance(node.parent, ast.Call)):
+                    isinstance(node._pyflakes_parent, ast.Call)):
                 # we are doing locals() call in current scope
                 self.scope.usesLocals = True
         elif isinstance(node.ctx, (ast.Store, ast.AugStore, ast.Param)):
@@ -1371,8 +1722,8 @@ class Checker(object):
         # definition (not OK), for 'continue', a finally block (not OK), or
         # the top module scope (not OK)
         n = node
-        while hasattr(n, 'parent'):
-            n, n_child = n.parent, n
+        while hasattr(n, '_pyflakes_parent'):
+            n, n_child = n._pyflakes_parent, n
             if isinstance(n, LOOP_TYPES):
                 # Doesn't apply unless it's in the loop itself
                 if n_child not in n.orelse:
@@ -1480,7 +1831,7 @@ class Checker(object):
 
             self.pushScope()
 
-            self.handleChildren(node, omit='decorator_list')
+            self.handleChildren(node, omit=['decorator_list', 'returns'])
 
             def checkUnusedAssignments():
                 """
diff --git a/pyflakes/messages.py b/pyflakes/messages.py
index 8ae545f..cf67cf8 100644
--- a/pyflakes/messages.py
+++ b/pyflakes/messages.py
@@ -266,3 +266,99 @@ class InvalidPrintSyntax(Message):
 
 class IsLiteral(Message):
     message = 'use ==/!= to compare str, bytes, and int literals'
+
+
+class FStringMissingPlaceholders(Message):
+    message = 'f-string is missing placeholders'
+
+
+class StringDotFormatExtraPositionalArguments(Message):
+    message = "'...'.format(...) has unused arguments at position(s): %s"
+
+    def __init__(self, filename, loc, extra_positions):
+        Message.__init__(self, filename, loc)
+        self.message_args = (extra_positions,)
+
+
+class StringDotFormatExtraNamedArguments(Message):
+    message = "'...'.format(...) has unused named argument(s): %s"
+
+    def __init__(self, filename, loc, extra_keywords):
+        Message.__init__(self, filename, loc)
+        self.message_args = (extra_keywords,)
+
+
+class StringDotFormatMissingArgument(Message):
+    message = "'...'.format(...) is missing argument(s) for placeholder(s): %s"
+
+    def __init__(self, filename, loc, missing_arguments):
+        Message.__init__(self, filename, loc)
+        self.message_args = (missing_arguments,)
+
+
+class StringDotFormatMixingAutomatic(Message):
+    message = "'...'.format(...) mixes automatic and manual numbering"
+
+
+class StringDotFormatInvalidFormat(Message):
+    message = "'...'.format(...) has invalid format string: %s"
+
+    def __init__(self, filename, loc, error):
+        Message.__init__(self, filename, loc)
+        self.message_args = (error,)
+
+
+class PercentFormatInvalidFormat(Message):
+    message = "'...' %% ... has invalid format string: %s"
+
+    def __init__(self, filename, loc, error):
+        Message.__init__(self, filename, loc)
+        self.message_args = (error,)
+
+
+class PercentFormatMixedPositionalAndNamed(Message):
+    message = "'...' %% ... has mixed positional and named placeholders"
+
+
+class PercentFormatUnsupportedFormatCharacter(Message):
+    message = "'...' %% ... has unsupported format character %r"
+
+    def __init__(self, filename, loc, c):
+        Message.__init__(self, filename, loc)
+        self.message_args = (c,)
+
+
+class PercentFormatPositionalCountMismatch(Message):
+    message = "'...' %% ... has %d placeholder(s) but %d substitution(s)"
+
+    def __init__(self, filename, loc, n_placeholders, n_substitutions):
+        Message.__init__(self, filename, loc)
+        self.message_args = (n_placeholders, n_substitutions)
+
+
+class PercentFormatExtraNamedArguments(Message):
+    message = "'...' %% ... has unused named argument(s): %s"
+
+    def __init__(self, filename, loc, extra_keywords):
+        Message.__init__(self, filename, loc)
+        self.message_args = (extra_keywords,)
+
+
+class PercentFormatMissingArgument(Message):
+    message = "'...' %% ... is missing argument(s) for placeholder(s): %s"
+
+    def __init__(self, filename, loc, missing_arguments):
+        Message.__init__(self, filename, loc)
+        self.message_args = (missing_arguments,)
+
+
+class PercentFormatExpectedMapping(Message):
+    message = "'...' %% ... expected mapping but got sequence"
+
+
+class PercentFormatExpectedSequence(Message):
+    message = "'...' %% ... expected sequence but got mapping"
+
+
+class PercentFormatStarRequiresSequence(Message):
+    message = "'...' %% ... `*` specifier requires sequence"


Test patch:
diff --git a/pyflakes/test/test_api.py b/pyflakes/test/test_api.py
index ee205f9..93b3886 100644
--- a/pyflakes/test/test_api.py
+++ b/pyflakes/test/test_api.py
@@ -214,9 +214,35 @@ class TestIterSourceCode(TestCase):
         with open(pythonw, 'w') as fd:
             fd.write('#!/usr/bin/env pythonw\n')
 
+        python3args = os.path.join(self.tempdir, 'g')
+        with open(python3args, 'w') as fd:
+            fd.write('#!/usr/bin/python3 -u\n')
+
+        python2u = os.path.join(self.tempdir, 'h')
+        with open(python2u, 'w') as fd:
+            fd.write('#!/usr/bin/python2u\n')
+
+        python3d = os.path.join(self.tempdir, 'i')
+        with open(python3d, 'w') as fd:
+            fd.write('#!/usr/local/bin/python3d\n')
+
+        python38m = os.path.join(self.tempdir, 'j')
+        with open(python38m, 'w') as fd:
+            fd.write('#! /usr/bin/env python3.8m\n')
+
+        python27 = os.path.join(self.tempdir, 'k')
+        with open(python27, 'w') as fd:
+            fd.write('#!/usr/bin/python2.7   \n')
+
+        # Should NOT be treated as Python source
+        notfirst = os.path.join(self.tempdir, 'l')
+        with open(notfirst, 'w') as fd:
+            fd.write('#!/bin/sh\n#!/usr/bin/python\n')
+
         self.assertEqual(
             sorted(iterSourceCode([self.tempdir])),
-            sorted([python, python2, python3, pythonw]))
+            sorted([python, python2, python3, pythonw, python3args, python2u,
+                    python3d, python38m, python27]))
 
     def test_multipleDirectories(self):
         """
diff --git a/pyflakes/test/test_doctests.py b/pyflakes/test/test_doctests.py
index 0825ffe..5565226 100644
--- a/pyflakes/test/test_doctests.py
+++ b/pyflakes/test/test_doctests.py
@@ -433,6 +433,16 @@ class Test(TestCase):
             return 1
         ''')
 
+    def test_globalUnderscoreInDoctest(self):
+        self.flakes("""
+        from gettext import ugettext as _
+
+        def doctest_stuff():
+            '''
+                >>> pass
+            '''
+        """, m.UnusedImport)
+
 
 class TestOther(_DoctestMixin, TestOther):
     """Run TestOther with each test wrapped in a doctest."""
diff --git a/pyflakes/test/test_other.py b/pyflakes/test/test_other.py
index 1b10e02..775fdde 100644
--- a/pyflakes/test/test_other.py
+++ b/pyflakes/test/test_other.py
@@ -1755,6 +1755,159 @@ class TestUnusedAssignment(TestCase):
         ''')
 
 
+class TestStringFormatting(TestCase):
+
+    @skipIf(version_info < (3, 6), 'new in Python 3.6')
+    def test_f_string_without_placeholders(self):
+        self.flakes("f'foo'", m.FStringMissingPlaceholders)
+        self.flakes('''
+            f"""foo
+            bar
+            """
+        ''', m.FStringMissingPlaceholders)
+        self.flakes('''
+            print(
+                f'foo'
+                f'bar'
+            )
+        ''', m.FStringMissingPlaceholders)
+        # this is an "escaped placeholder" but not a placeholder
+        self.flakes("f'{{}}'", m.FStringMissingPlaceholders)
+        # ok: f-string with placeholders
+        self.flakes('''
+            x = 5
+            print(f'{x}')
+        ''')
+        # ok: f-string with format specifiers
+        self.flakes('''
+            x = 'a' * 90
+            print(f'{x:.8}')
+        ''')
+        # ok: f-string with multiple format specifiers
+        self.flakes('''
+            x = y = 5
+            print(f'{x:>2} {y:>2}')
+        ''')
+
+    def test_invalid_dot_format_calls(self):
+        self.flakes('''
+            '{'.format(1)
+        ''', m.StringDotFormatInvalidFormat)
+        self.flakes('''
+            '{} {1}'.format(1, 2)
+        ''', m.StringDotFormatMixingAutomatic)
+        self.flakes('''
+            '{0} {}'.format(1, 2)
+        ''', m.StringDotFormatMixingAutomatic)
+        self.flakes('''
+            '{}'.format(1, 2)
+        ''', m.StringDotFormatExtraPositionalArguments)
+        self.flakes('''
+            '{}'.format(1, bar=2)
+        ''', m.StringDotFormatExtraNamedArguments)
+        self.flakes('''
+            '{} {}'.format(1)
+        ''', m.StringDotFormatMissingArgument)
+        self.flakes('''
+            '{2}'.format()
+        ''', m.StringDotFormatMissingArgument)
+        self.flakes('''
+            '{bar}'.format()
+        ''', m.StringDotFormatMissingArgument)
+        # too much string recursion (placeholder-in-placeholder)
+        self.flakes('''
+            '{:{:{}}}'.format(1, 2, 3)
+        ''', m.StringDotFormatInvalidFormat)
+        # ok: dotted / bracketed names need to handle the param differently
+        self.flakes("'{.__class__}'.format('')")
+        self.flakes("'{foo[bar]}'.format(foo={'bar': 'barv'})")
+        # ok: placeholder-placeholders
+        self.flakes('''
+            print('{:{}} {}'.format(1, 15, 2))
+        ''')
+        # ok: not a placeholder-placeholder
+        self.flakes('''
+            print('{:2}'.format(1))
+        ''')
+        # ok: not mixed automatic
+        self.flakes('''
+            '{foo}-{}'.format(1, foo=2)
+        ''')
+        # ok: we can't determine statically the format args
+        self.flakes('''
+            a = ()
+            "{}".format(*a)
+        ''')
+        self.flakes('''
+            k = {}
+            "{foo}".format(**k)
+        ''')
+
+    def test_invalid_percent_format_calls(self):
+        self.flakes('''
+            '%(foo)' % {'foo': 'bar'}
+        ''', m.PercentFormatInvalidFormat)
+        self.flakes('''
+            '%s %(foo)s' % {'foo': 'bar'}
+        ''', m.PercentFormatMixedPositionalAndNamed)
+        self.flakes('''
+            '%(foo)s %s' % {'foo': 'bar'}
+        ''', m.PercentFormatMixedPositionalAndNamed)
+        self.flakes('''
+            '%j' % (1,)
+        ''', m.PercentFormatUnsupportedFormatCharacter)
+        self.flakes('''
+            '%s %s' % (1,)
+        ''', m.PercentFormatPositionalCountMismatch)
+        self.flakes('''
+            '%s %s' % (1, 2, 3)
+        ''', m.PercentFormatPositionalCountMismatch)
+        self.flakes('''
+            '%(bar)s' % {}
+        ''', m.PercentFormatMissingArgument,)
+        self.flakes('''
+            '%(bar)s' % {'bar': 1, 'baz': 2}
+        ''', m.PercentFormatExtraNamedArguments)
+        self.flakes('''
+            '%(bar)s' % (1, 2, 3)
+        ''', m.PercentFormatExpectedMapping)
+        self.flakes('''
+            '%s %s' % {'k': 'v'}
+        ''', m.PercentFormatExpectedSequence)
+        self.flakes('''
+            '%(bar)*s' % {'bar': 'baz'}
+        ''', m.PercentFormatStarRequiresSequence)
+        # ok: single %s with mapping
+        self.flakes('''
+            '%s' % {'foo': 'bar', 'baz': 'womp'}
+        ''')
+        # ok: does not cause a MemoryError (the strings aren't evaluated)
+        self.flakes('''
+            "%1000000000000f" % 1
+        ''')
+        # ok: %% should not count towards placeholder count
+        self.flakes('''
+            '%% %s %% %s' % (1, 2)
+        ''')
+        # ok: * consumes one positional argument
+        self.flakes('''
+            '%.*f' % (2, 1.1234)
+            '%*.*f' % (5, 2, 3.1234)
+        ''')
+
+    @skipIf(version_info < (3, 5), 'new in Python 3.5')
+    def test_ok_percent_format_cannot_determine_element_count(self):
+        self.flakes('''
+            a = []
+            '%s %s' % [*a]
+            '%s %s' % (*a,)
+        ''')
+        self.flakes('''
+            k = {}
+            '%(k)s' % {**k}
+        ''')
+
+
 class TestAsyncStatements(TestCase):
 
     @skipIf(version_info < (3, 5), 'new in Python 3.5')
diff --git a/pyflakes/test/test_type_annotations.py b/pyflakes/test/test_type_annotations.py
index b8876cb..9c34dcf 100644
--- a/pyflakes/test/test_type_annotations.py
+++ b/pyflakes/test/test_type_annotations.py
@@ -77,24 +77,16 @@ class TestTypeAnnotations(TestCase):
     def test_not_a_typing_overload(self):
         """regression test for @typing.overload detection bug in 2.1.0"""
         self.flakes("""
-            x = lambda f: f
+            def foo(x):
+                return x
 
-            @x
-            def t():
+            @foo
+            def bar():
                 pass
 
-            y = lambda f: f
-
-            @x
-            @y
-            def t():
-                pass
-
-            @x
-            @y
-            def t():
+            def bar():
                 pass
-        """, m.RedefinedWhileUnused, m.RedefinedWhileUnused)
+        """, m.RedefinedWhileUnused)
 
     @skipIf(version_info < (3, 6), 'new in Python 3.6')
     def test_variable_annotations(self):
@@ -350,3 +342,23 @@ class TestTypeAnnotations(TestCase):
         x = 1
         # type: F
         """)
+
+    @skipIf(version_info < (3,), 'new in Python 3')
+    def test_return_annotation_is_class_scope_variable(self):
+        self.flakes("""
+        from typing import TypeVar
+        class Test:
+            Y = TypeVar('Y')
+
+            def t(self, x: Y) -> Y:
+                return x
+        """)
+
+    @skipIf(version_info < (3,), 'new in Python 3')
+    def test_return_annotation_is_function_body_variable(self):
+        self.flakes("""
+        class Test:
+            def t(self) -> Y:
+                Y = 2
+                return Y
+        """, m.UndefinedName)