Repository: pantsbuild/pants

Issue:
Add line number to unowned_dependency_behavior warn/error
**Is your feature request related to a problem? Please describe.**
The output now is:

```
08:27:12.67 [ERROR] The following imports in path/to/file.py:address have no owners:

  * mod.name

```

**Describe the solution you'd like**
```
08:27:12.67 [ERROR] The following imports in path/to/file.py:address have no owners:

  * mod.name (line 255)

```

or if we want editors to allow auto-click:

```
08:27:12.67 [ERROR] The following imports in path/to/file.py:address have no owners:

  * mod.name (path/to/file.py:255)

```

**Describe alternatives you've considered**
N/A

**Additional context**
N/A


Discussion:
I love it! The auto-click format is nice.

Resolving patch:
diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_imports.py b/src/python/pants/backend/python/dependency_inference/parse_python_imports.py
--- a/src/python/pants/backend/python/dependency_inference/parse_python_imports.py
+++ b/src/python/pants/backend/python/dependency_inference/parse_python_imports.py
@@ -1,6 +1,7 @@
 # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import json
 import pkgutil
 from dataclasses import dataclass
 
@@ -9,15 +10,15 @@
 from pants.backend.python.util_rules.pex_environment import PythonExecutable
 from pants.core.util_rules.source_files import SourceFilesRequest
 from pants.core.util_rules.stripped_source_files import StrippedSourceFiles
-from pants.engine.collection import DeduplicatedCollection
 from pants.engine.fs import CreateDigest, Digest, FileContent, MergeDigests
 from pants.engine.process import Process, ProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
+from pants.util.frozendict import FrozenDict
 from pants.util.logging import LogLevel
 
 
-class ParsedPythonImports(DeduplicatedCollection[str]):
-    """All the discovered imports from a Python source file.
+class ParsedPythonImports(FrozenDict[str, int]):
+    """All the discovered imports from a Python source file mapped to the first line they appear.
 
     May include string imports if the request specified to include them.
     """
@@ -67,7 +68,8 @@ async def parse_python_imports(request: ParsePythonImportsRequest) -> ParsedPyth
     )
     # See above for where we explicitly encoded as utf8. Even though utf8 is the
     # default for decode(), we make that explicit here for emphasis.
-    return ParsedPythonImports(process_result.stdout.decode("utf8").strip().splitlines())
+    process_output = process_result.stdout.decode("utf8")
+    return ParsedPythonImports(json.loads(process_output) if process_output else {})
 
 
 def rules():
diff --git a/src/python/pants/backend/python/dependency_inference/rules.py b/src/python/pants/backend/python/dependency_inference/rules.py
--- a/src/python/pants/backend/python/dependency_inference/rules.py
+++ b/src/python/pants/backend/python/dependency_inference/rules.py
@@ -187,7 +187,7 @@ async def infer_python_dependencies_via_imports(
 
     owners_per_import = await MultiGet(
         Get(PythonModuleOwners, PythonModule(imported_module))
-        for imported_module in detected_imports
+        for imported_module in detected_imports.keys()
     )
 
     merged_result: set[Address] = set()
@@ -210,10 +210,14 @@ async def infer_python_dependencies_via_imports(
 
     unowned_dependency_behavior = python_infer_subsystem.unowned_dependency_behavior
     if unowned_imports and unowned_dependency_behavior is not UnownedDependencyUsage.DoNothing:
+        unowned_imports_with_lines = [
+            f"{module_name} ({request.sources_field.file_path}:{detected_imports[module_name]})"
+            for module_name in sorted(unowned_imports)
+        ]
         raise_error = unowned_dependency_behavior is UnownedDependencyUsage.RaiseError
         log = logger.error if raise_error else logger.warning
         log(
-            f"The following imports in {address} have no owners:\n\n{bullet_list(unowned_imports)}\n\n"
+            f"The following imports in {address} have no owners:\n\n{bullet_list(unowned_imports_with_lines)}\n\n"
             "If you are expecting this import to be provided by your own firstparty code, ensure that it is contained within a source root. "
             "Otherwise if you are using a requirements file, consider adding the relevant package.\n"
             "Otherwise consider declaring a `python_requirement_library` target, which can then be inferred.\n"
diff --git a/src/python/pants/backend/python/dependency_inference/scripts/import_parser.py b/src/python/pants/backend/python/dependency_inference/scripts/import_parser.py
--- a/src/python/pants/backend/python/dependency_inference/scripts/import_parser.py
+++ b/src/python/pants/backend/python/dependency_inference/scripts/import_parser.py
@@ -3,11 +3,14 @@
 # -*- coding: utf-8 -*-
 
 # NB: This must be compatible with Python 2.7 and 3.5+.
+# NB: If you're needing to debug this, an easy way is to just invoke it on a file.
+#   E.g. `MIN_DOTS=1 STRING_IMPORTS=N python3 src/python/pants/backend/python/dependency_inference/scripts/import_parser.py FILENAME`
 
 from __future__ import print_function, unicode_literals
 
 import ast
 import itertools
+import json
 import os
 import re
 import sys
@@ -28,11 +31,12 @@ class AstVisitor(ast.NodeVisitor):
     def __init__(self, package_parts, contents):
         self._package_parts = package_parts
         self._contents_lines = contents.decode(errors="ignore").splitlines()
-        self.imports = set()
+        # N.B. use `setdefault` when adding imports
+        self.imports = {}  # maps module_name to first lineno of occurance
 
-    def maybe_add_string_import(self, s):
+    def maybe_add_string_import(self, node, s):
         if STRING_IMPORT_REGEX.match(s):
-            self.imports.add(s)
+            self.imports.setdefault(s, node.lineno)
 
     @staticmethod
     def _is_pragma_ignored(line):
@@ -47,23 +51,23 @@ def _visit_import_stmt(self, node, import_prefix):
         token_iter = tokenize.generate_tokens(lambda: next(node_lines_iter))
 
         def consume_until(string):
-            return list(itertools.takewhile(lambda t: t[1] != string, token_iter))
+            for token in token_iter:
+                if token[1] == string:
+                    return token
 
         consume_until("import")
 
         # N.B. The names in this list are in the same order as the import statement
         for alias in node.names:
-            consume_until(alias.name.split(".")[-1])
+            token = consume_until(alias.name.split(".")[-1])
 
             # N.B. Keep consuming lines while they end in a line-continuation
             #   (unfortunately `tokenize` doesn't capture this)
-            line = "\\"
-            while line.endswith("\\"):
+            while token[4].endswith("\\"):
                 token = next(token_iter)
-                line = token[4]
 
-            if not self._is_pragma_ignored(line):
-                self.imports.add(import_prefix + alias.name)
+            if not self._is_pragma_ignored(token[4]):
+                self.imports.setdefault(import_prefix + alias.name, token[3][0] + node.lineno - 1)
             if alias.asname and token[1] != alias.asname:
                 consume_until(alias.asname)
 
@@ -94,8 +98,9 @@ def visit_Call(self, node):
                 name = str(node.args[0].value)
 
             if name is not None:
-                if not self._is_pragma_ignored(self._contents_lines[node.args[0].lineno - 1]):
-                    self.imports.add(name)
+                lineno = node.args[0].lineno
+                if not self._is_pragma_ignored(self._contents_lines[lineno - 1]):
+                    self.imports.setdefault(name, lineno)
                 return
 
         self.generic_visit(node)
@@ -109,7 +114,7 @@ def visit_Call(self, node):
         def visit_Str(self, node):
             try:
                 val = node.s.decode("utf8") if isinstance(node.s, bytes) else node.s
-                self.maybe_add_string_import(val)
+                self.maybe_add_string_import(node, val)
             except UnicodeError:
                 pass
 
@@ -118,7 +123,7 @@ def visit_Str(self, node):
     elif sys.version_info[0:2] < (3, 8):
 
         def visit_Str(self, node):
-            self.maybe_add_string_import(node.s)
+            self.maybe_add_string_import(node, node.s)
 
         setattr(AstVisitor, "visit_Str", visit_Str)
 
@@ -126,7 +131,7 @@ def visit_Str(self, node):
 
         def visit_Constant(self, node):
             if isinstance(node.value, str):
-                self.maybe_add_string_import(node.value)
+                self.maybe_add_string_import(node, node.value)
 
         setattr(AstVisitor, "visit_Constant", visit_Constant)
 
@@ -146,7 +151,8 @@ def main(filename):
     # We have to be careful to set the encoding explicitly and write raw bytes ourselves.
     # See below for where we explicitly decode.
     buffer = sys.stdout if sys.version_info[0:2] == (2, 7) else sys.stdout.buffer
-    buffer.write("\n".join(sorted(visitor.imports)).encode("utf8"))
+    output = json.dumps(visitor.imports)
+    buffer.write(output.encode("utf8"))
 
 
 if __name__ == "__main__":


Test patch:
diff --git a/src/python/pants/backend/python/dependency_inference/parse_python_imports_test.py b/src/python/pants/backend/python/dependency_inference/parse_python_imports_test.py
--- a/src/python/pants/backend/python/dependency_inference/parse_python_imports_test.py
+++ b/src/python/pants/backend/python/dependency_inference/parse_python_imports_test.py
@@ -42,7 +42,7 @@ def assert_imports_parsed(
     rule_runner: RuleRunner,
     content: str,
     *,
-    expected: list[str],
+    expected: dict[str, int],
     filename: str = "project/foo.py",
     constraints: str = ">=3.6",
     string_imports: bool = True,
@@ -67,7 +67,7 @@ def assert_imports_parsed(
             )
         ],
     )
-    assert list(imports) == sorted(expected)
+    assert dict(imports) == expected
 
 
 def test_normal_imports(rule_runner: RuleRunner) -> None:
@@ -76,6 +76,7 @@ def test_normal_imports(rule_runner: RuleRunner) -> None:
         from __future__ import print_function
 
         import os
+        import os  # repeated to test the line number
         import os.path
         from typing import TYPE_CHECKING
 
@@ -123,25 +124,25 @@ def test_normal_imports(rule_runner: RuleRunner) -> None:
     assert_imports_parsed(
         rule_runner,
         content,
-        expected=[
-            "__future__.print_function",
-            "also_not_ignored_but_looks_like_it_could_be",
-            "demo",
-            "multiline_import1.not_ignored1",
-            "multiline_import1.not_ignored2",
-            "multiline_import2.not_ignored",
-            "not_ignored_but_looks_like_it_could_be",
-            "os",
-            "os.path",
-            "requests",
-            "project.circular_dep.CircularDep",
-            "project.demo.Demo",
-            "project.demo.OriginalName",
-            "pkg_resources",
-            "subprocess",
-            "subprocess23",
-            "typing.TYPE_CHECKING",
-        ],
+        expected={
+            "__future__.print_function": 1,
+            "os": 3,
+            "os.path": 5,
+            "typing.TYPE_CHECKING": 6,
+            "requests": 8,
+            "demo": 10,
+            "project.demo.Demo": 11,
+            "project.demo.OriginalName": 12,
+            "multiline_import1.not_ignored1": 16,
+            "multiline_import1.not_ignored2": 23,
+            "multiline_import2.not_ignored": 26,
+            "project.circular_dep.CircularDep": 29,
+            "subprocess": 32,
+            "subprocess23": 34,
+            "pkg_resources": 36,
+            "not_ignored_but_looks_like_it_could_be": 39,
+            "also_not_ignored_but_looks_like_it_could_be": 45,
+        },
     )
 
 
@@ -163,14 +164,14 @@ def test_relative_imports(rule_runner: RuleRunner, basename: str) -> None:
         rule_runner,
         content,
         filename=f"project/util/{basename}",
-        expected=[
-            "project.util.sibling",
-            "project.util.sibling.Nibling",
-            "project.util.subdir.child.Child",
-            "project.parent.Parent",
-            "project.parent.Parent1",
-            "project.parent.Guardian",
-        ],
+        expected={
+            "project.util.sibling": 1,
+            "project.util.sibling.Nibling": 2,
+            "project.util.subdir.child.Child": 3,
+            "project.parent.Parent": 4,
+            "project.parent.Parent1": 6,
+            "project.parent.Guardian": 7,
+        },
     )
 
 
@@ -208,31 +209,31 @@ def test_imports_from_strings(rule_runner: RuleRunner, min_dots: int) -> None:
         """
     )
 
-    potentially_valid = [
-        "a.b",
-        "a.Foo",
-        "a.b.d",
-        "a.b2.d",
-        "a.b.c.Foo",
-        "a.b.c.d.Foo",
-        "a.b.c.d.FooBar",
-        "a.b.c.d.e.f.g.Baz",
-        "a.b_c.d._bar",
-        "a.b2.c.D",
-        "a.b.c_狗",
-    ]
-    expected = [sym for sym in potentially_valid if sym.count(".") >= min_dots]
+    potentially_valid = {
+        "a.b": 3,
+        "a.Foo": 4,
+        "a.b.d": 5,
+        "a.b2.d": 6,
+        "a.b.c.Foo": 7,
+        "a.b.c.d.Foo": 8,
+        "a.b.c.d.FooBar": 9,
+        "a.b.c.d.e.f.g.Baz": 10,
+        "a.b_c.d._bar": 11,
+        "a.b2.c.D": 12,
+        "a.b.c_狗": 13,
+    }
+    expected = {sym: line for sym, line in potentially_valid.items() if sym.count(".") >= min_dots}
 
     assert_imports_parsed(rule_runner, content, expected=expected, string_imports_min_dots=min_dots)
-    assert_imports_parsed(rule_runner, content, string_imports=False, expected=[])
+    assert_imports_parsed(rule_runner, content, string_imports=False, expected={})
 
 
 def test_gracefully_handle_syntax_errors(rule_runner: RuleRunner) -> None:
-    assert_imports_parsed(rule_runner, "x =", expected=[])
+    assert_imports_parsed(rule_runner, "x =", expected={})
 
 
 def test_handle_unicode(rule_runner: RuleRunner) -> None:
-    assert_imports_parsed(rule_runner, "x = 'äbç'", expected=[])
+    assert_imports_parsed(rule_runner, "x = 'äbç'", expected={})
 
 
 @skip_unless_python27_present
@@ -259,15 +260,15 @@ def test_works_with_python2(rule_runner: RuleRunner) -> None:
         rule_runner,
         content,
         constraints="==2.7.*",
-        expected=[
-            "demo",
-            "dep.from.bytes",
-            "dep.from.str",
-            "dep.from.str_狗",
-            "project.demo.Demo",
-            "pkg_resources",
-            "treat.as.a.regular.import.not.a.string.import",
-        ],
+        expected={
+            "demo": 4,
+            "project.demo.Demo": 5,
+            "pkg_resources": 7,
+            "treat.as.a.regular.import.not.a.string.import": 8,
+            "dep.from.bytes": 10,
+            "dep.from.str": 11,
+            "dep.from.str_狗": 12,
+        },
     )
 
 
@@ -292,13 +293,13 @@ def test_works_with_python38(rule_runner: RuleRunner) -> None:
         rule_runner,
         content,
         constraints=">=3.8",
-        expected=[
-            "demo",
-            "dep.from.str",
-            "project.demo.Demo",
-            "pkg_resources",
-            "treat.as.a.regular.import.not.a.string.import",
-        ],
+        expected={
+            "demo": 5,
+            "project.demo.Demo": 6,
+            "pkg_resources": 8,
+            "treat.as.a.regular.import.not.a.string.import": 9,
+            "dep.from.str": 11,
+        },
     )
 
 
@@ -325,11 +326,11 @@ def test_works_with_python39(rule_runner: RuleRunner) -> None:
         rule_runner,
         content,
         constraints=">=3.9",
-        expected=[
-            "demo",
-            "dep.from.str",
-            "project.demo.Demo",
-            "pkg_resources",
-            "treat.as.a.regular.import.not.a.string.import",
-        ],
+        expected={
+            "demo": 7,
+            "project.demo.Demo": 8,
+            "pkg_resources": 10,
+            "treat.as.a.regular.import.not.a.string.import": 11,
+            "dep.from.str": 13,
+        },
     )
diff --git a/src/python/pants/backend/python/dependency_inference/rules_test.py b/src/python/pants/backend/python/dependency_inference/rules_test.py
--- a/src/python/pants/backend/python/dependency_inference/rules_test.py
+++ b/src/python/pants/backend/python/dependency_inference/rules_test.py
@@ -298,7 +298,7 @@ def run_dep_inference(
     run_dep_inference(Address("src/python", relative_file_path="cheesey.py"), "warning")
     assert len(caplog.records) == 1
     assert "The following imports in src/python/cheesey.py have no owners:" in caplog.text
-    assert "  * venezuelan_beaver_cheese" in caplog.text
+    assert "  * venezuelan_beaver_cheese (src/python/cheesey.py:1)" in caplog.text
 
     # Now test with "error"
     caplog.clear()
@@ -308,7 +308,7 @@ def run_dep_inference(
     assert isinstance(exc_info.value.wrapped_exceptions[0], UnownedDependencyError)
     assert len(caplog.records) == 2  # one for the error being raised and one for our message
     assert "The following imports in src/python/cheesey.py have no owners:" in caplog.text
-    assert "  * venezuelan_beaver_cheese" in caplog.text
+    assert "  * venezuelan_beaver_cheese (src/python/cheesey.py:1)" in caplog.text
 
     caplog.clear()