Repository: numba/numba

Issue:
"Unexpected" indentation can break DI emission
<!--

Thanks for opening an issue! To help the Numba team handle your information
efficiently, please first ensure that there is no other issue present that
already describes the issue you have
(search at https://github.com/numba/numba/issues?&q=is%3Aissue).

-->

## Reporting a bug

<!--

Before submitting a bug report please ensure that you can check off these boxes:

-->

- [x] I have tried using the RC1 version of Numba 0.55
- [x] I have included a self contained code sample to reproduce the problem.
  i.e. it's possible to run as 'python bug.py'.

This:
```python
from numba import njit
import numpy as np

def gen():
    @njit(debug=True)
    def foo():
# comment
        impl = 1
        return impl
    return foo

gen()()
```
does this:
```
numba/core/lowering.py", line 112, in _compute_def_location
    src_ast = ast.parse(source_str)
  File "<pypath>/lib/python3.9/ast.py", line 50, in parse
    return compile(source, filename, mode, flags,
  File "<unknown>", line 1
    @njit(debug=True)
IndentationError: unexpected indent
```
the issue is the `#comment` at a different indent level.



Discussion:
Other variants:
```python
from numba import njit
import numpy as np

def gen():
    @njit(debug=True)
    def foo():
        somestr = ("start..."
"continue")
        impl = 1
        return impl
    return foo

gen()()
```
and
```python
from numba import njit
import numpy as np

def gen():
    @njit(debug=True)
    def foo():
"""docstring"""
        impl = 1
        return impl
    return foo

gen()()
```
I suspect any sort of line continuation at a lesser indent level can trigger this issue.

Resolving patch:
diff --git a/numba/core/errors.py b/numba/core/errors.py
--- a/numba/core/errors.py
+++ b/numba/core/errors.py
@@ -97,6 +97,12 @@ class NumbaIRAssumptionWarning(NumbaPedanticWarning):
     Warning category for reporting an IR assumption violation.
     """
 
+
+class NumbaDebugInfoWarning(NumbaWarning):
+    """
+    Warning category for an issue with the emission of debug information.
+    """
+
 # These are needed in the color formatting of errors setup
 
 
diff --git a/numba/core/lowering.py b/numba/core/lowering.py
--- a/numba/core/lowering.py
+++ b/numba/core/lowering.py
@@ -13,7 +13,7 @@
                         targetconfig)
 from numba.core.errors import (LoweringError, new_error_context, TypingError,
                                LiteralTypingError, UnsupportedError,
-                               NumbaWarning)
+                               NumbaDebugInfoWarning)
 from numba.core.funcdesc import default_mangler
 from numba.core.environment import Environment
 from numba.core.analysis import compute_use_defs
@@ -100,19 +100,32 @@ def _compute_def_location(self):
                 raw_source_str, _ = inspect.getsourcelines(fn)
             except OSError:
                 msg = ("Could not find source for function: "
-                       f"{self.func_ir.func_id.func}")
-                warnings.warn(NumbaWarning(msg))
+                       f"{self.func_ir.func_id.func}. Debug line information "
+                       "may be inaccurate.")
+                warnings.warn(NumbaDebugInfoWarning(msg))
             else:
                 # Parse the source and find the line with `def <func>` in it, it
                 # is assumed that if the compilation has made it this far that
                 # the source is at least legal and has valid syntax.
 
-                # join the source as a block and dedent it
+                # Join the source as a block and dedent it.
                 source_str = textwrap.dedent(''.join(raw_source_str))
-                src_ast = ast.parse(source_str)
+                # Deal with unparsable source (see #7730), this can be caused
+                # by continuation lines/comments at indent levels that are
+                # invalid when the just function source is parsed in isolation.
+                src_ast = None
+                try:
+                    src_ast = ast.parse(source_str)
+                except IndentationError:
+                    msg = ("Could not parse the source for function: "
+                           f"{self.func_ir.func_id.func}. Debug line "
+                           "information may be inaccurate. This is often "
+                           "caused by comments/docstrings/line continuation "
+                           "that is at a lesser indent level than the source.")
+                    warnings.warn(NumbaDebugInfoWarning(msg))
                 # pull the definition out of the AST, only if it seems valid
                 # i.e. one thing in the body
-                if len(src_ast.body) == 1:
+                if src_ast is not None and len(src_ast.body) == 1:
                     pydef = src_ast.body.pop()
                     # -1 as lines start at 1 and this is an offset.
                     pydef_offset = pydef.lineno - 1


Test patch:
diff --git a/numba/tests/test_debuginfo.py b/numba/tests/test_debuginfo.py
--- a/numba/tests/test_debuginfo.py
+++ b/numba/tests/test_debuginfo.py
@@ -3,12 +3,16 @@
 import re
 import numpy as np
 import math
+from textwrap import dedent
+import unittest
+import warnings
 
-from numba.tests.support import TestCase, override_config, needs_subprocess
+from numba.tests.support import (TestCase, override_config, needs_subprocess,
+                                 ignore_internal_warnings)
 from numba import jit, njit
 from numba.core import types, utils
 from numba.core.datamodel import default_manager
-import unittest
+from numba.core.errors import NumbaDebugInfoWarning
 import llvmlite.binding as llvm
 
 #NOTE: These tests are potentially sensitive to changes in SSA or lowering
@@ -637,6 +641,50 @@ def foo(missing=None):
         # expect ir.LiteralStructType([])
         self.assertEqual(base_ty, ('!{}'))
 
+    def test_missing_source(self):
+        strsrc = """
+        def foo():
+            return 1
+        """
+        l = dict()
+        exec(dedent(strsrc), {}, l)
+        foo = njit(debug=True)(l['foo'])
+
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter('always', NumbaDebugInfoWarning)
+            ignore_internal_warnings()
+            foo()
+
+        self.assertEqual(len(w), 1)
+        found = w[0]
+        self.assertEqual(found.category, NumbaDebugInfoWarning)
+        msg = str(found.message)
+        # make sure the warning contains the right message
+        self.assertIn('Could not find source for function', msg)
+        # and refers to the offending function
+        self.assertIn(str(foo.py_func), msg)
+
+    def test_unparsable_indented_source(self):
+
+        @njit(debug=True)
+        def foo():
+# NOTE: THIS COMMENT MUST START AT COLUMN 0 FOR THIS SAMPLE CODE TO BE VALID # noqa: E115, E501
+            return 1
+
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter('always', NumbaDebugInfoWarning)
+            ignore_internal_warnings()
+            foo()
+
+        self.assertEqual(len(w), 1)
+        found = w[0]
+        self.assertEqual(found.category, NumbaDebugInfoWarning)
+        msg = str(found.message)
+        # make sure the warning contains the right message
+        self.assertIn('Could not parse the source for function', msg)
+        # and refers to the offending function
+        self.assertIn(str(foo.py_func), msg)
+
 
 if __name__ == '__main__':
     unittest.main()