Repository: ipython/ipython

Issue:
Completion in indented lines incorrectly strips the prefix when jedi is disabled
Completion in indented lines incorrectly strips the prefix when jedi is disabled.

Originally reported in Notebook (https://github.com/jupyter/notebook/issues/7397) then JupyterLab (https://github.com/jupyterlab/jupyterlab/issues/16522), but narrowed down to IPython 8.8 / 8.11; https://github.com/ipython/ipython/pull/13943 fixed a related issue but did not cover this one.




Discussion:
bisected to https://github.com/ipython/ipython/pull/13943
So It was trading one issue for another.
FYI, we're also affected by this in Spyder because we disable Jedi support by default (until recently it had serious performance issues).
@ccordoba12 would be great if you have time to confirm if https://github.com/ipython/ipython/pull/14474 helps (and does not introduce regressions elsewhere)



Patch:
diff --git a/IPython/core/completer.py b/IPython/core/completer.py
index 4eef2b55880..440953d0690 100644
--- a/IPython/core/completer.py
+++ b/IPython/core/completer.py
@@ -898,7 +898,7 @@ def rectify_completions(text: str, completions: _IC, *, _debug: bool = False) ->
         new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
         if c._origin == 'jedi':
             seen_jedi.add(new_text)
-        elif c._origin == 'IPCompleter.python_matches':
+        elif c._origin == "IPCompleter.python_matcher":
             seen_python_matches.add(new_text)
         yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin, signature=c.signature)
     diff = seen_python_matches.difference(seen_jedi)
@@ -1139,15 +1139,18 @@ def attr_matches(self, text):
         with a __getattr__ hook is evaluated.
 
         """
+        return self._attr_matches(text)[0]
+
+    def _attr_matches(self, text, include_prefix=True) -> Tuple[Sequence[str], str]:
         m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
         if not m2:
-            return []
+            return [], ""
         expr, attr = m2.group(1, 2)
 
         obj = self._evaluate_expr(expr)
 
         if obj is not_found:
-            return []
+            return [], ""
 
         if self.limit_to__all__ and hasattr(obj, '__all__'):
             words = get__all__entries(obj)
@@ -1170,28 +1173,36 @@ def attr_matches(self, text):
         # reconciliator would know that we intend to append to rather than
         # replace the input text; this requires refactoring to return range
         # which ought to be replaced (as does jedi).
-        tokens = _parse_tokens(expr)
-        rev_tokens = reversed(tokens)
-        skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE}
-        name_turn = True
-
-        parts = []
-        for token in rev_tokens:
-            if token.type in skip_over:
-                continue
-            if token.type == tokenize.NAME and name_turn:
-                parts.append(token.string)
-                name_turn = False
-            elif token.type == tokenize.OP and token.string == "." and not name_turn:
-                parts.append(token.string)
-                name_turn = True
-            else:
-                # short-circuit if not empty nor name token
-                break
+        if include_prefix:
+            tokens = _parse_tokens(expr)
+            rev_tokens = reversed(tokens)
+            skip_over = {tokenize.ENDMARKER, tokenize.NEWLINE}
+            name_turn = True
+
+            parts = []
+            for token in rev_tokens:
+                if token.type in skip_over:
+                    continue
+                if token.type == tokenize.NAME and name_turn:
+                    parts.append(token.string)
+                    name_turn = False
+                elif (
+                    token.type == tokenize.OP and token.string == "." and not name_turn
+                ):
+                    parts.append(token.string)
+                    name_turn = True
+                else:
+                    # short-circuit if not empty nor name token
+                    break
 
-        prefix_after_space = "".join(reversed(parts))
+            prefix_after_space = "".join(reversed(parts))
+        else:
+            prefix_after_space = ""
 
-        return ["%s.%s" % (prefix_after_space, w) for w in words if w[:n] == attr]
+        return (
+            ["%s.%s" % (prefix_after_space, w) for w in words if w[:n] == attr],
+            "." + attr,
+        )
 
     def _evaluate_expr(self, expr):
         obj = not_found
@@ -1973,9 +1984,8 @@ def matchers(self) -> List[Matcher]:
                 *self.magic_arg_matchers,
                 self.custom_completer_matcher,
                 self.dict_key_matcher,
-                # TODO: convert python_matches to v2 API
                 self.magic_matcher,
-                self.python_matches,
+                self.python_matcher,
                 self.file_matcher,
                 self.python_func_kw_matcher,
             ]
@@ -2316,9 +2326,42 @@ def _jedi_matches(
             else:
                 return iter([])
 
+    @context_matcher()
+    def python_matcher(self, context: CompletionContext) -> SimpleMatcherResult:
+        """Match attributes or global python names"""
+        text = context.line_with_cursor
+        if "." in text:
+            try:
+                matches, fragment = self._attr_matches(text, include_prefix=False)
+                if text.endswith(".") and self.omit__names:
+                    if self.omit__names == 1:
+                        # true if txt is _not_ a __ name, false otherwise:
+                        no__name = lambda txt: re.match(r".*\.__.*?__", txt) is None
+                    else:
+                        # true if txt is _not_ a _ name, false otherwise:
+                        no__name = (
+                            lambda txt: re.match(r"\._.*?", txt[txt.rindex(".") :])
+                            is None
+                        )
+                    matches = filter(no__name, matches)
+                return _convert_matcher_v1_result_to_v2(
+                    matches, type="attribute", fragment=fragment
+                )
+            except NameError:
+                # catches <undefined attributes>.<tab>
+                matches = []
+                return _convert_matcher_v1_result_to_v2(matches, type="attribute")
+        else:
+            matches = self.global_matches(context.token)
+            # TODO: maybe distinguish between functions, modules and just "variables"
+            return _convert_matcher_v1_result_to_v2(matches, type="variable")
+
     @completion_matcher(api_version=1)
     def python_matches(self, text: str) -> Iterable[str]:
-        """Match attributes or global python names"""
+        """Match attributes or global python names.
+
+        .. deprecated:: 8.27
+            You can use :meth:`python_matcher` instead."""
         if "." in text:
             try:
                 matches = self.attr_matches(text)


Test patch:
diff --git a/IPython/core/tests/test_completer.py b/IPython/core/tests/test_completer.py
index 87c561dc06b..e20d9b88a89 100644
--- a/IPython/core/tests/test_completer.py
+++ b/IPython/core/tests/test_completer.py
@@ -462,7 +462,10 @@ def test_all_completions_dups(self):
                 matches = c.all_completions("TestClass.")
                 assert len(matches) > 2, (jedi_status, matches)
                 matches = c.all_completions("TestClass.a")
-                assert matches == ['TestClass.a', 'TestClass.a1'], jedi_status
+                if jedi_status:
+                    assert matches == ["TestClass.a", "TestClass.a1"], jedi_status
+                else:
+                    assert matches == [".a", ".a1"], jedi_status
 
     @pytest.mark.xfail(
         sys.version_info.releaselevel in ("alpha",),
@@ -594,7 +597,7 @@ def _(line, cursor_pos, expect, message, completion):
                 ip.Completer.use_jedi = True
                 with provisionalcompleter():
                     completions = ip.Completer.completions(line, cursor_pos)
-                self.assertIn(completion, completions)
+                self.assertIn(completion, list(completions))
 
         with provisionalcompleter():
             _(
@@ -622,7 +625,7 @@ def _(line, cursor_pos, expect, message, completion):
             _(
                 "assert str.star",
                 14,
-                "str.startswith",
+                ".startswith",
                 "Should have completed on `assert str.star`: %s",
                 Completion(11, 14, "startswith"),
             )
@@ -633,6 +636,13 @@ def _(line, cursor_pos, expect, message, completion):
                 "Should have completed on `d['a b'].str`: %s",
                 Completion(9, 12, "strip"),
             )
+            _(
+                "a.app",
+                4,
+                ".append",
+                "Should have completed on `a.app`: %s",
+                Completion(2, 4, "append"),
+            )
 
     def test_omit__names(self):
         # also happens to test IPCompleter as a configurable
@@ -647,8 +657,8 @@ def test_omit__names(self):
         with provisionalcompleter():
             c.use_jedi = False
             s, matches = c.complete("ip.")
-            self.assertIn("ip.__str__", matches)
-            self.assertIn("ip._hidden_attr", matches)
+            self.assertIn(".__str__", matches)
+            self.assertIn("._hidden_attr", matches)
 
             # c.use_jedi = True
             # completions = set(c.completions('ip.', 3))
@@ -661,7 +671,7 @@ def test_omit__names(self):
         with provisionalcompleter():
             c.use_jedi = False
             s, matches = c.complete("ip.")
-            self.assertNotIn("ip.__str__", matches)
+            self.assertNotIn(".__str__", matches)
             # self.assertIn('ip._hidden_attr', matches)
 
             # c.use_jedi = True
@@ -675,8 +685,8 @@ def test_omit__names(self):
         with provisionalcompleter():
             c.use_jedi = False
             s, matches = c.complete("ip.")
-            self.assertNotIn("ip.__str__", matches)
-            self.assertNotIn("ip._hidden_attr", matches)
+            self.assertNotIn(".__str__", matches)
+            self.assertNotIn("._hidden_attr", matches)
 
             # c.use_jedi = True
             # completions = set(c.completions('ip.', 3))
@@ -686,7 +696,7 @@ def test_omit__names(self):
         with provisionalcompleter():
             c.use_jedi = False
             s, matches = c.complete("ip._x.")
-            self.assertIn("ip._x.keys", matches)
+            self.assertIn(".keys", matches)
 
             # c.use_jedi = True
             # completions = set(c.completions('ip._x.', 6))
@@ -697,7 +707,7 @@ def test_omit__names(self):
 
     def test_limit_to__all__False_ok(self):
         """
-        Limit to all is deprecated, once we remove it this test can go away. 
+        Limit to all is deprecated, once we remove it this test can go away.
         """
         ip = get_ipython()
         c = ip.Completer
@@ -708,7 +718,7 @@ def test_limit_to__all__False_ok(self):
         cfg.IPCompleter.limit_to__all__ = False
         c.update_config(cfg)
         s, matches = c.complete("d.")
-        self.assertIn("d.x", matches)
+        self.assertIn(".x", matches)
 
     def test_get__all__entries_ok(self):
         class A: