Repository: PyCQA/pyflakes
Pull request #76 (2016-07-21 15:37:28)

Fix PyPy2 Windows IntegrationTests
IntegrationTests.test_errors is failing on Windows under PyPy2 as its stderr emits `\r\r\n` as the line separator.

Add AppVeyor testing for three PyPy releases.

Also add a test for the other more complex stderr message emitted by the Reporter.

sigmavirus24: This has merge conflicts. Mind rebasing before I review much more?

Patch:
diff --git a/.appveyor.yml b/.appveyor.yml
index 4a1d13c..149be37 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -1,8 +1,46 @@
 # To activate, change the Appveyor settings to use `.appveyor.yml`.
 install:
-  - python -m pip install tox
+  - python -m pip install --upgrade virtualenv pip setuptools tox
+
+  # Fetch the three main PyPy releases
+  - ps: (New-Object Net.WebClient).DownloadFile('https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-win32.zip', "$env:appveyor_build_folder\pypy-2.6.1-win32.zip")
+  - ps: 7z x pypy-2.6.1-win32.zip | Out-Null
+  - move pypy-2.6.1-win32 C:\
+
+  - ps: (New-Object Net.WebClient).DownloadFile('https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.3.1-win32.zip', "$env:appveyor_build_folder\pypy2-v5.3.1-win32.zip")
+  - ps: 7z x pypy2-v5.3.1-win32.zip | Out-Null
+  - move pypy2-v5.3.1-win32 C:\
+
+  - ps: (New-Object Net.WebClient).DownloadFile('https://bitbucket.org/pypy/pypy/downloads/pypy3-2.4.0-win32.zip', "$env:appveyor_build_folder\pypy3-2.4.0-win32.zip")
+  - ps: 7z x pypy3-2.4.0-win32.zip | Out-Null
+  - move pypy3-2.4.0-win32 C:\
+
+  # pypy3 installer provides 'pypy.exe', not pypy3.exe.
+  - copy C:\pypy3-2.4.0-win32\pypy.exe C:\pypy3-2.4.0-win32\pypy3.exe
+
+  # workaround https://github.com/pypa/virtualenv/issues/93
+  - mkdir C:\python33\tcl\tcl8.6
+  - mkdir C:\python33\tcl\tk8.6
+  - mkdir C:\pypy3-2.4.0-win32\tcl\tcl8.6
+  - mkdir C:\pypy3-2.4.0-win32\tcl\tk8.6
+
+  # Only pypy2-5.3.1 is integrated into tox, as pypy3-2.4.0 fails and
+  # a Windows distribution of pypy3-5.2 isnt available yet.
+  - ps: $env:path = "$env:path;C:\pypy2-v5.3.1-win32"
+
+  # pypy3-2.4.0 and pypy-2.6.1 are manually bootstrapped and tested
+  - ps: (New-Object Net.WebClient).DownloadFile('https://bootstrap.pypa.io/get-pip.py', "$env:appveyor_build_folder\get-pip.py")
+  - git clone https://github.com/pypa/setuptools/
+  - cd setuptools
+  - C:\pypy3-2.4.0-win32\pypy3 bootstrap.py
+  - C:\pypy3-2.4.0-win32\pypy3 setup.py install
+  - C:\pypy-2.6.1-win32\pypy bootstrap.py
+  - C:\pypy-2.6.1-win32\pypy setup.py install
+  - cd ..
 
 build: off
 
 test_script:
   - python -m tox
+  - C:\pypy3-2.4.0-win32\pypy3 setup.py test -q
+  - C:\pypy-2.6.1-win32\pypy setup.py test -q
diff --git a/NEWS.txt b/NEWS.txt
index 8396109..d887c67 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -5,7 +5,7 @@
   - Avoid traceback when exception is del-ed in except
 
 1.2.1 (2015-05-05):
-  - Fix false RedefinedWhileUnesed for submodule imports
+  - Fix false RedefinedWhileUnused for submodule imports
 
 1.2.0 (2016-05-03):
   - Warn against reusing exception names after the except: block on Python 3
diff --git a/pyflakes/checker.py b/pyflakes/checker.py
index 9f354fd..1902f13 100644
--- a/pyflakes/checker.py
+++ b/pyflakes/checker.py
@@ -81,6 +81,17 @@ class _FieldsOrder(dict):
         return fields
 
 
+def counter(items):
+    """
+    Simplest required implementation of collections.Counter. Required as 2.6
+    does not have Counter in collections.
+    """
+    results = {}
+    for item in items:
+        results[item] = results.get(item, 0) + 1
+    return results
+
+
 def iter_child_nodes(node, omit=None, _fields_order=_FieldsOrder()):
     """
     Yield all direct child nodes of *node*, that is, all fields that
@@ -97,6 +108,33 @@ def iter_child_nodes(node, omit=None, _fields_order=_FieldsOrder()):
                 yield item
 
 
+def convert_to_value(item):
+    if isinstance(item, ast.Str):
+        return item.s
+    elif hasattr(ast, 'Bytes') and isinstance(item, ast.Bytes):
+        return item.s
+    elif isinstance(item, ast.Tuple):
+        return tuple(convert_to_value(i) for i in item.elts)
+    elif isinstance(item, ast.Num):
+        return item.n
+    elif isinstance(item, ast.Name):
+        result = VariableKey(item=item)
+        constants_lookup = {
+            'True': True,
+            'False': False,
+            'None': None,
+        }
+        return constants_lookup.get(
+            result.name,
+            result,
+        )
+    elif (not PY33) and isinstance(item, ast.NameConstant):
+        # None, True, False are nameconstants in python3, but names in 2
+        return item.value
+    else:
+        return UnhandledKeyType()
+
+
 class Binding(object):
     """
     Represents the binding of a value to a name.
@@ -133,6 +171,31 @@ class Definition(Binding):
     """
 
 
+class UnhandledKeyType(object):
+    """
+    A dictionary key of a type that we cannot or do not check for duplicates.
+    """
+
+
+class VariableKey(object):
+    """
+    A dictionary key which is a variable.
+
+    @ivar item: The variable AST object.
+    """
+    def __init__(self, item):
+        self.name = item.id
+
+    def __eq__(self, compare):
+        return (
+            compare.__class__ == self.__class__
+            and compare.name == self.name
+        )
+
+    def __hash__(self):
+        return hash(self.name)
+
+
 class Importation(Definition):
     """
     A binding created by an import statement.
@@ -241,7 +304,7 @@ class ImportationFrom(Importation):
 
 
 class StarImportation(Importation):
-    """A binding created by an 'from x import *' statement."""
+    """A binding created by a 'from x import *' statement."""
 
     def __init__(self, name, source):
         super(StarImportation, self).__init__('*', source)
@@ -736,7 +799,7 @@ class Checker(object):
             return
 
         if on_conditional_branch():
-            # We can not predict if this conditional branch is going to
+            # We cannot predict if this conditional branch is going to
             # be executed.
             return
 
@@ -855,7 +918,7 @@ class Checker(object):
     PASS = ignore
 
     # "expr" type nodes
-    BOOLOP = BINOP = UNARYOP = IFEXP = DICT = SET = \
+    BOOLOP = BINOP = UNARYOP = IFEXP = SET = \
         COMPARE = CALL = REPR = ATTRIBUTE = SUBSCRIPT = \
         STARRED = NAMECONSTANT = handleChildren
 
@@ -876,6 +939,42 @@ class Checker(object):
     # additional node types
     COMPREHENSION = KEYWORD = FORMATTEDVALUE = handleChildren
 
+    def DICT(self, node):
+        # Complain if there are duplicate keys with different values
+        # If they have the same value it's not going to cause potentially
+        # unexpected behaviour so we'll not complain.
+        keys = [
+            convert_to_value(key) for key in node.keys
+        ]
+
+        key_counts = counter(keys)
+        duplicate_keys = [
+            key for key, count in key_counts.items()
+            if count > 1
+        ]
+
+        for key in duplicate_keys:
+            key_indices = [i for i, i_key in enumerate(keys) if i_key == key]
+
+            values = counter(
+                convert_to_value(node.values[index])
+                for index in key_indices
+            )
+            if any(count == 1 for value, count in values.items()):
+                for key_index in key_indices:
+                    key_node = node.keys[key_index]
+                    if isinstance(key, VariableKey):
+                        self.report(messages.MultiValueRepeatedKeyVariable,
+                                    key_node,
+                                    key.name)
+                    else:
+                        self.report(
+                            messages.MultiValueRepeatedKeyLiteral,
+                            key_node,
+                            key,
+                        )
+        self.handleChildren(node)
+
     def ASSERT(self, node):
         if isinstance(node.test, ast.Tuple) and node.test.elts != []:
             self.report(messages.AssertTuple, node)
diff --git a/pyflakes/messages.py b/pyflakes/messages.py
index 05db5bf..58bb6a2 100644
--- a/pyflakes/messages.py
+++ b/pyflakes/messages.py
@@ -116,6 +116,22 @@ class DuplicateArgument(Message):
         self.message_args = (name,)
 
 
+class MultiValueRepeatedKeyLiteral(Message):
+    message = 'dictionary key %r repeated with different values'
+
+    def __init__(self, filename, loc, key):
+        Message.__init__(self, filename, loc)
+        self.message_args = (key,)
+
+
+class MultiValueRepeatedKeyVariable(Message):
+    message = 'dictionary key variable %s repeated with different values'
+
+    def __init__(self, filename, loc, key):
+        Message.__init__(self, filename, loc)
+        self.message_args = (key,)
+
+
 class LateFutureImport(Message):
     message = 'from __future__ imports must occur at the beginning of the file'
 


Test patch:
diff --git a/pyflakes/test/test_api.py b/pyflakes/test/test_api.py
index 4ffe242..77ee33c 100644
--- a/pyflakes/test/test_api.py
+++ b/pyflakes/test/test_api.py
@@ -8,6 +8,7 @@ import shutil
 import subprocess
 import tempfile
 
+from pyflakes.checker import PY2
 from pyflakes.messages import UnusedImport
 from pyflakes.reporter import Reporter
 from pyflakes.api import (
@@ -30,6 +31,12 @@ try:
 except AttributeError:
     PYPY = False
 
+try:
+    WindowsError
+    WIN = True
+except NameError:
+    WIN = False
+
 ERROR_HAS_COL_NUM = ERROR_HAS_LAST_LINE = sys.version_info >= (3, 2) or PYPY
 
 
@@ -661,6 +668,9 @@ class IntegrationTests(TestCase):
         if sys.version_info >= (3,):
             stdout = stdout.decode('utf-8')
             stderr = stderr.decode('utf-8')
+        # Workaround https://bitbucket.org/pypy/pypy/issues/2350
+        if PYPY and PY2 and WIN:
+            stderr = stderr.replace('\r\r\n', '\r\n')
         return (stdout, stderr, rv)
 
     def test_goodFile(self):
@@ -685,7 +695,7 @@ class IntegrationTests(TestCase):
         expected = UnusedImport(self.tempfilepath, Node(1), 'contraband')
         self.assertEqual(d, ("%s%s" % (expected, os.linesep), '', 1))
 
-    def test_errors(self):
+    def test_errors_io(self):
         """
         When pyflakes finds errors with the files it's given, (if they don't
         exist, say), then the return code is non-zero and the errors are
@@ -696,6 +706,20 @@ class IntegrationTests(TestCase):
                                                          os.linesep)
         self.assertEqual(d, ('', error_msg, 1))
 
+    def test_errors_syntax(self):
+        """
+        When pyflakes finds errors with the files it's given, (if they don't
+        exist, say), then the return code is non-zero and the errors are
+        printed to stderr.
+        """
+        fd = open(self.tempfilepath, 'wb')
+        fd.write("import".encode('ascii'))
+        fd.close()
+        d = self.runPyflakes([self.tempfilepath])
+        error_msg = '{0}:1:{2}: invalid syntax{1}import{1}    {3}^{1}'.format(
+            self.tempfilepath, os.linesep, 5 if PYPY else 7, '' if PYPY else '  ')
+        self.assertEqual(d, ('', error_msg, True))
+
     def test_readFromStdin(self):
         """
         If no arguments are passed to C{pyflakes} then it reads from stdin.
diff --git a/pyflakes/test/test_dict.py b/pyflakes/test/test_dict.py
new file mode 100644
index 0000000..628ec0c
--- /dev/null
+++ b/pyflakes/test/test_dict.py
@@ -0,0 +1,217 @@
+"""
+Tests for dict duplicate keys Pyflakes behavior.
+"""
+
+from sys import version_info
+
+from pyflakes import messages as m
+from pyflakes.test.harness import TestCase, skipIf
+
+
+class Test(TestCase):
+
+    def test_duplicate_keys(self):
+        self.flakes(
+            "{'yes': 1, 'yes': 2}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    @skipIf(version_info < (3,),
+            "bytes and strings with same 'value' are not equal in python3")
+    @skipIf(version_info[0:2] == (3, 2),
+            "python3.2 does not allow u"" literal string definition")
+    def test_duplicate_keys_bytes_vs_unicode_py3(self):
+        self.flakes("{b'a': 1, u'a': 2}")
+
+    @skipIf(version_info < (3,),
+            "bytes and strings with same 'value' are not equal in python3")
+    @skipIf(version_info[0:2] == (3, 2),
+            "python3.2 does not allow u"" literal string definition")
+    def test_duplicate_values_bytes_vs_unicode_py3(self):
+        self.flakes(
+            "{1: b'a', 1: u'a'}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    @skipIf(version_info >= (3,),
+            "bytes and strings with same 'value' are equal in python2")
+    def test_duplicate_keys_bytes_vs_unicode_py2(self):
+        self.flakes(
+            "{b'a': 1, u'a': 2}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    @skipIf(version_info >= (3,),
+            "bytes and strings with same 'value' are equal in python2")
+    def test_duplicate_values_bytes_vs_unicode_py2(self):
+        self.flakes("{1: b'a', 1: u'a'}")
+
+    def test_multiple_duplicate_keys(self):
+        self.flakes(
+            "{'yes': 1, 'yes': 2, 'no': 2, 'no': 3}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_keys_in_function(self):
+        self.flakes(
+            '''
+            def f(thing):
+                pass
+            f({'yes': 1, 'yes': 2})
+            ''',
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_keys_in_lambda(self):
+        self.flakes(
+            "lambda x: {(0,1): 1, (0,1): 2}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_keys_tuples(self):
+        self.flakes(
+            "{(0,1): 1, (0,1): 2}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_keys_tuples_int_and_float(self):
+        self.flakes(
+            "{(0,1): 1, (0,1.0): 2}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_keys_ints(self):
+        self.flakes(
+            "{1: 1, 1: 2}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_keys_bools(self):
+        self.flakes(
+            "{True: 1, True: 2}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_keys_bools_false(self):
+        # Needed to ensure 2.x correctly coerces these from variables
+        self.flakes(
+            "{False: 1, False: 2}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_keys_none(self):
+        self.flakes(
+            "{None: 1, None: 2}",
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_variable_keys(self):
+        self.flakes(
+            '''
+            a = 1
+            {a: 1, a: 2}
+            ''',
+            m.MultiValueRepeatedKeyVariable,
+            m.MultiValueRepeatedKeyVariable,
+        )
+
+    def test_duplicate_variable_values(self):
+        self.flakes(
+            '''
+            a = 1
+            b = 2
+            {1: a, 1: b}
+            ''',
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_variable_values_same_value(self):
+        # Current behaviour is not to look up variable values. This is to
+        # confirm that.
+        self.flakes(
+            '''
+            a = 1
+            b = 1
+            {1: a, 1: b}
+            ''',
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_duplicate_key_float_and_int(self):
+        """
+        These do look like different values, but when it comes to their use as
+        keys, they compare as equal and so are actually duplicates.
+        The literal dict {1: 1, 1.0: 1} actually becomes {1.0: 1}.
+        """
+        self.flakes(
+            '''
+            {1: 1, 1.0: 2}
+            ''',
+            m.MultiValueRepeatedKeyLiteral,
+            m.MultiValueRepeatedKeyLiteral,
+        )
+
+    def test_no_duplicate_key_error_same_value(self):
+        self.flakes('''
+        {'yes': 1, 'yes': 1}
+        ''')
+
+    def test_no_duplicate_key_errors(self):
+        self.flakes('''
+        {'yes': 1, 'no': 2}
+        ''')
+
+    def test_no_duplicate_keys_tuples_same_first_element(self):
+        self.flakes("{(0,1): 1, (0,2): 1}")
+
+    def test_no_duplicate_key_errors_func_call(self):
+        self.flakes('''
+        def test(thing):
+            pass
+        test({True: 1, None: 2, False: 1})
+        ''')
+
+    def test_no_duplicate_key_errors_bool_or_none(self):
+        self.flakes("{True: 1, None: 2, False: 1}")
+
+    def test_no_duplicate_key_errors_ints(self):
+        self.flakes('''
+        {1: 1, 2: 1}
+        ''')
+
+    def test_no_duplicate_key_errors_vars(self):
+        self.flakes('''
+        test = 'yes'
+        rest = 'yes'
+        {test: 1, rest: 2}
+        ''')
+
+    def test_no_duplicate_key_errors_tuples(self):
+        self.flakes('''
+        {(0,1): 1, (0,2): 1}
+        ''')
+
+    def test_no_duplicate_key_errors_instance_attributes(self):
+        self.flakes('''
+        class Test():
+            pass
+        f = Test()
+        f.a = 1
+        {f.a: 1, f.a: 1}
+        ''')
diff --git a/pyflakes/test/test_imports.py b/pyflakes/test/test_imports.py
index f1bb5cd..3e3be88 100644
--- a/pyflakes/test/test_imports.py
+++ b/pyflakes/test/test_imports.py
@@ -180,7 +180,7 @@ class Test(TestCase):
 
     def test_redefinedTry(self):
         """
-        Test that importing a module twice in an try block
+        Test that importing a module twice in a try block
         does raise a warning.
         """
         self.flakes('''
@@ -193,7 +193,7 @@ class Test(TestCase):
 
     def test_redefinedTryExcept(self):
         """
-        Test that importing a module twice in an try
+        Test that importing a module twice in a try
         and except block does not raise a warning.
         """
         self.flakes('''
@@ -1149,7 +1149,7 @@ class TestSpecialAll(TestCase):
 
 class Python26Tests(TestCase):
     """
-    Tests for checking of syntax which is valid in PYthon 2.6 and newer.
+    Tests for checking of syntax which is valid in Python 2.6 and newer.
     """
 
     @skipIf(version_info < (2, 6), "Python >= 2.6 only")
diff --git a/pyflakes/test/test_other.py b/pyflakes/test/test_other.py
index 7efc1c5..90b980b 100644
--- a/pyflakes/test/test_other.py
+++ b/pyflakes/test/test_other.py
@@ -1181,7 +1181,7 @@ class TestUnusedAssignment(TestCase):
                 return
         ''', m.UnusedVariable)
 
-    @skip("todo: Difficult because it does't apply in the context of a loop")
+    @skip("todo: Difficult because it doesn't apply in the context of a loop")
     def test_unusedReassignedVariable(self):
         """
         Shadowing a used variable can still raise an UnusedVariable warning.
@@ -1489,7 +1489,7 @@ class TestUnusedAssignment(TestCase):
 
     def test_withStatementTupleNamesUndefined(self):
         """
-        An undefined name warning is emitted if a name first defined by a the
+        An undefined name warning is emitted if a name first defined by the
         tuple-unpacking form of the C{with} statement is used before the
         C{with} statement.
         """